def test_prepare_data_inputs_parameters(self):
variable = cwt.Variable('file:///test.nc', 'tas', name='v0')
domain = cwt.Domain([
cwt.Dimension('time', 0, 365),
], name='d0')
process = cwt.Process('CDAT.subset', name='subset')
process.description = mock.MagicMock()
process.description.metadata.return_value = {}
client = cwt.WPSClient('http://idontexist/wps')
data_inputs = client.prepare_data_inputs_str(process, [variable],
domain,
axes=['lats', 'lons'],
weightoptions='generated',
test=cwt.NamedParameter(
'test', 'True'))
self.assertIn('"axes": "lats|lons"', data_inputs)
self.assertIn('"weightoptions": "generated"', data_inputs)
self.assertIn('"test": "True"', data_inputs)
def test_execute(self):
process = models.Process.objects.create(identifier='CDSpark.max',
backend='EDAS')
job = models.Job.objects.create(server=self.server,
user=self.user,
process=process)
domain = cwt.Domain([cwt.Dimension('time', 0, 200)], name='d0')
domains = {'d0': domain}
var = cwt.Variable('file:///test.nc', 'tas', name='v0')
variables = {'v0': var}
proc = cwt.Process(identifier='CDSpark.max', name='max')
proc.domain = 'd0'
proc.set_inputs('v0')
operations = {'max': proc}
task = self.backend.execute('CDSpark.max',
variables,
domains,
operations,
user=self.user,
job=job)
self.assertIsNotNone(task)
def test_script_generator(self):
user = models.User.objects.first()
variables = {
'v0': cwt.Variable('file:///test.nc', 'tas', name='v0'),
'v1': cwt.Variable('file:///test.nc', 'tas', name='v1'),
}
domains = {
'd0': cwt.Domain([cwt.Dimension('time', 0, 200)], name='d0')
}
gridder = cwt.Gridder(grid='gaussian~32')
op = cwt.Process(identifier='CDAT.subset')
op.domain = domains.values()[0]
op.add_inputs(*variables.values())
op.parameters['gridder'] = gridder
op.parameters['axes'] = cwt.NamedParameter('axes', 'time')
operations = {'subset': op}
sg = wps_service.WPSScriptGenerator(variables, domains, operations,
user)
data = sg.generate()
self.assertIsNotNone(data)
def setUp(self):
self.client = cwt.WPSClient('https://0.0.0.0:10000/wps')
# Mock owslib.WebProcessingService
self.client.client = mock.MagicMock()
subset = mock.MagicMock()
type(subset).identifier = mock.PropertyMock(return_value='CDAT.subset')
type(subset).title = mock.PropertyMock(return_value='CDAT.subset')
type(subset).processVersion = mock.PropertyMock(return_value='1.0.0')
metrics = mock.MagicMock()
type(metrics).identifier = mock.PropertyMock(
return_value='CDAT.metrics')
type(metrics).title = mock.PropertyMock(return_value='CDAT.metrics')
type(metrics).processVersion = mock.PropertyMock(return_value='1.0.0')
type(self.client.client).processes = mock.PropertyMock(return_value=[
subset,
metrics,
])
self.process = cwt.Process.from_dict({
'name': 'CDAT.subset',
'input': [],
'domain': None,
'result': 'p0',
})
self.domain = cwt.Domain(time=(0, 365), name='d0')
self.variable = cwt.Variable('file:///test.nc', 'tas', name='v0')
def test_map_domain(self):
domain = cwt.Domain([
cwt.Dimension('time', 20, 200, cwt.INDICES),
cwt.Dimension('lat', 45, -45),
])
mock_var = mock.MagicMock()
mock_var.getAxisIndex.side_effect = [0, 1]
mock_axis1 = mock.MagicMock()
mock_axis1.isTime.return_value = True
mock_axis1.id = 'time'
mock_axis1.shape = (200,)
mock_axis2 = mock.MagicMock()
mock_axis2.isTime.return_value = False
mock_axis2.id = 'lat'
mock_axis2.shape = (180,)
mock_var.getAxis.side_effect = [
mock_axis1,
mock_axis2
]
self.ds.variable = mock_var
self.ds.map_domain(domain, 'days since 1990')
self.assertEqual(self.ds.temporal, slice(20, 200, 1))
self.assertIn('lat', self.ds.spatial)
self.assertEqual(self.ds.spatial['lat'], (45, -45))
def test_map_domain_missing_axis(self):
domain = cwt.Domain([
cwt.Dimension('time', 20, 200, cwt.INDICES),
cwt.Dimension('lat', 45, -45),
])
mock_var = mock.MagicMock()
mock_var.getAxisIndex.side_effect = [0, -1]
mock_axis1 = mock.MagicMock()
mock_axis1.isTime.return_value = True
mock_axis1.id = 'time'
mock_axis1.shape = (200,)
mock_axis2 = mock.MagicMock()
mock_axis2.isTime.return_value = False
mock_axis2.id = 'lat'
mock_axis2.shape = (180,)
mock_var.getAxis.side_effect = [
mock_axis1,
mock_axis2
]
self.ds.variable = mock_var
with self.assertRaises(tasks.WPSError):
self.ds.map_domain(domain, 'days since 1990')
def test_execute_block(self, mock_request):
mock_request.return_value.status_code = 200
mock_request.return_value.text = self.execute.toxml(bds=cwt.bds)
client = cwt.WPSClient('http://idontexist/wps')
process = cwt.Process.from_identifier('CDAT.subset')
with mock.patch('cwt.process.Process.output',
new_callable=mock.PropertyMock) as output:
output.return_value = 'test output'
process.wait = mock.MagicMock()
process.description = mock.MagicMock()
result = client.execute(process,
[cwt.Variable('file:///test.nc', 'tas')],
cwt.Domain([cwt.Dimension('time', 0,
365)]),
block=True)
process.wait.assert_called()
self.assertEqual(result, 'test output')
def test_add_dimension_timestamps(self):
dom = cwt.Domain()
dom.add_dimension('lat', ('1980-01-01', '2000-01-01'))
self.assertEqual(len(dom.dimensions), 1)
self.assertEqual(dom.dimensions['lat'].crs, cwt.TIMESTAMPS)
def test_add_dimension_values(self):
dom = cwt.Domain()
dom.add_dimension('lat', (0, 20))
self.assertEqual(len(dom.dimensions), 1)
self.assertEqual(dom.dimensions['lat'].crs, cwt.VALUES)
def test_add_dimension_indices(self):
dom = cwt.Domain()
dom.add_dimension('lat', slice(0, 20))
self.assertEqual(len(dom.dimensions), 1)
self.assertEqual(dom.dimensions['lat'].crs, cwt.INDICES)
def setUp(self):
self.data = {
'uri': 'file:///tas.nc',
'id': 'tas|tas1',
'domain': 'd0',
'mime_type': 'application/netcdf',
}
self.d0 = cwt.Domain(time=(1980, 2000), name='d0')
def test_prepare_data_inputs(self):
proc = cwt.Process(type('Process', (object,), dict(identifier='CDAT.avg')), name='avg')
tas = cwt.Variable('file:///data/tas_6h.nc', 'tas', name='tas1')
d0 = cwt.Domain(name='d0')
data_inputs = self.wps.prepare_data_inputs(proc, [tas], d0)
self.assertEqual(self.data_inputs, data_inputs)
def test_partitions(self):
domain = cwt.Domain([
cwt.Dimension('time', 20, 100),
cwt.Dimension('lat', 0, 90),
])
mock_cache = mock.MagicMock()
mock_cache_obj = mock.MagicMock()
collection = file_manager.DataSetCollection()
collection.check_cache = mock.MagicMock(return_value=(mock_cache, mock_cache_obj))
mock_dataset1 = mock.MagicMock()
mock_dataset1.variable_name = 'tas'
mock_dataset1.get_time.return_value.units = 'days since 2000'
mock_dataset1.partitions.return_value = [
mock.MagicMock(),
]
mock_dataset2 = mock.MagicMock()
mock_dataset2.variable_name = 'tas'
mock_dataset2.get_time.return_value.units = 'days since 1990'
mock_dataset2.partitions.return_value = [
mock.MagicMock(),
]
collection.datasets = [
mock_dataset1,
mock_dataset2,
]
result = [x for x in collection.partitions(domain, False)]
mock_dataset1.partitions.return_value[0].getTime.return_value.toRelativeTime.assert_called_with('days since 1990')
mock_dataset2.partitions.return_value[0].getTime.return_value.toRelativeTime.assert_called_with('days since 1990')
mock_cache_obj.write.assert_has_calls([
mock.call(mock_dataset2.partitions.return_value[0], id='tas'),
mock.call(mock_dataset1.partitions.return_value[0], id='tas'),
])
mock_cache_obj.sync.assert_called()
mock_cache.set_size.assert_called()
mock_dataset1.map_domain.assert_called_with(domain, 'days since 1990')
mock_dataset2.map_domain.assert_called_with(domain, 'days since 1990')
self.assertEqual(collection.datasets[0], mock_dataset2)
self.assertEqual(collection.datasets[1], mock_dataset1)
self.assertEqual(result[0], (mock_dataset2, mock_dataset2.partitions.return_value[0]))
self.assertEqual(result[1], (mock_dataset1, mock_dataset1.partitions.return_value[0]))
def test_parameterize(self):
process = cwt.Process.from_identifier('CDAT.subset')
process.set_domain(cwt.Domain([
cwt.Dimension('time', 0, 365),
]))
process.add_parameters(test=['value1'])
process.add_inputs(cwt.Variable('file:///test.nc', 'tas'))
data = process.parameterize()
def test_execute_failed(self, mock_request):
mock_request.return_value.status_code = 200
mock_request.return_value.text = self.execute_failed.toxml(bds=cwt.bds)
client = cwt.WPSClient('http://idontexist/wps')
process = cwt.Process.from_identifier('CDAT.subset')
with self.assertRaises(Exception):
client.execute(process, [cwt.Variable('file:///test.nc', 'tas')],
cwt.Domain([cwt.Dimension('time', 0, 365)]))
def execute(self, context, request, client, identifier, files, variable,
domain, **kwargs):
process = client.processes(identifier)[0]
domain = None if domain is None else cwt.Domain(**domain)
inputs = [cwt.Variable(x, variable) for x in files]
client.execute(process, inputs, domain)
context.set_data_inputs(request, inputs, domain, process)
return process
def test_parameterize(self):
expected = {
'id': 'd0',
'lat': {
'start': 0,
'end': 90,
'step': 1,
'crs': 'values'
}
}
dom = cwt.Domain([cwt.Dimension('lat', 0, 90)], name='d0')
self.assertDictContainsSubset(expected, dom.parameterize())
def test_api_key(self, mock_request):
mock_request.return_value.status_code = 200
mock_request.return_value.text = self.execute.toxml(bds=cwt.bds)
client = cwt.WPSClient('http://idontexist/wps', api_key='api_key_7')
process = cwt.Process.from_identifier('CDAT.subset')
process.description = mock.MagicMock()
response = client.execute(process,
[cwt.Variable('file:///test.nc', 'tas')],
cwt.Domain([cwt.Dimension('time', 0, 365)]))
def test_execute_no_inputs(self, mock_request):
mock_request.return_value.status_code = 200
mock_request.return_value.text = self.execute.toxml(bds=cwt.bds)
client = cwt.WPSClient('http://idontexist/wps')
process = cwt.Process.from_identifier('CDAT.subset')
process.description = mock.MagicMock()
client.execute(process, None,
cwt.Domain([cwt.Dimension('time', 0, 365)]))
self.assertIsNotNone(process.response)
def test_execute_missing_operation(self):
mock_job = mock.MagicMock()
variables = {
'v0': cwt.Variable('file:///test.nc', 'tas', name='v0'),
}
domains = {'d0': cwt.Domain([cwt.Dimension('time', 0, 200)])}
with self.assertRaises(WPSError) as e:
self.backend.execute('Oph.max',
variables,
domains, {},
job=mock_job,
user=self.user)
def test_execute_get(self, mock_request):
mock_request.return_value.status_code = 200
mock_request.return_value.text = self.execute.toxml(bds=cwt.bds)
client = cwt.WPSClient('http://idontexist/wps')
process = cwt.Process.from_identifier('CDAT.subset')
process.description = mock.MagicMock()
client.execute(process, [cwt.Variable('file:///test.nc', 'tas')],
cwt.Domain([cwt.Dimension('time', 0, 365)]),
method='GET')
self.assertIsNotNone(process.response)
def test_execute_no_duplicates(self, mock_request):
mock_request.return_value.status_code = 200
mock_request.return_value.text = self.execute.toxml(bds=cwt.bds)
client = cwt.WPSClient('http://idontexist/wps')
process = cwt.Process.from_identifier('CDAT.subset')
process.description = mock.MagicMock()
client.execute(process, [cwt.Variable('file:///test.nc', 'tas')],
cwt.Domain([cwt.Dimension('time', 0, 365)]),
axes=['time'])
self.assertEqual(len(process.parameters), 0)
self.assertIsNone(process.domain)
self.assertEqual(len(process.inputs), 0)
self.assertIsNotNone(process.response)
def run( self ):
d0 = cwt.Domain([], name="d0")
op1 = cwt.Operation.from_dict( { 'name': "CDSpark.multiAverage" } )
op1.add_input( cwt.Variable("file:///dass/nobackup/tpmaxwel/.edas/cache/collections/NCML/MERRA_TAS1hr.ncml", "tas" ) )
op3 = cwt.Operation.from_dict( { 'name': 'CDSpark.regrid', 'crs':'gaussian~128' } )
op3.add_input( op1 )
op2 = cwt.Operation.from_dict( { 'name': "CDSpark.multiAverage" } )
for i in range(1,3): op2.add_input( cwt.Variable('collection:/GISS-E2-R_r%di1p1'%(i), "tas" ) )
op4 = cwt.Operation.from_dict( { 'name': 'CDSpark.regrid', 'crs':'gaussian~128' } )
op4.add_input( op2 )
op5 = cwt.Operation.from_dict( { 'name': 'CDSpark.multiAverage' } )
op5.add_input( op3 )
op5.add_input( op4 )
wps = cwt.WPS( 'http://localhost:9001/wps', log=True, log_file=os.path.expanduser("~/esgf_api.log") )
wps.init()
process = cwt.Process( wps, op5 )
process.execute( None, d0, [], True, True, "GET" )
def test_execute(self):
mock_job = mock.MagicMock()
variables = {
'v0': cwt.Variable('file:///test.nc', 'tas', name='v0'),
}
domains = {'d0': cwt.Domain([cwt.Dimension('time', 0, 200)])}
operation = cwt.Process(identifier='Oph.max', name='max')
operation.domain = 'd0'
result = self.backend.execute('Oph.max',
variables,
domains, {'max': operation},
job=mock_job,
user=self.user)
self.assertIsNotNone(result)
def test_prepare_data_inputs_preserve_process(self):
v1 = cwt.Variable('file:///test1.nc', 'tas')
self.process.add_inputs(v1)
self.process.add_parameters(feature='test')
domain = cwt.Domain(time=(10, 20), lat=(-90, 0))
self.process.domain = domain
data_inputs = self.client.prepare_data_inputs(self.process, [
self.variable,
],
domain=None,
axes='lats')
# Check original process is untouched
self.assertEqual(self.process.inputs, [v1])
self.assertEqual(self.process.domain, domain)
self.assertEqual(self.process.parameters,
{'feature': cwt.NamedParameter('feature', 'test')})
# Verify the outputs
self.assertEqual(len(data_inputs), 3)
self.assertIn(json.dumps(self.variable.to_dict()), data_inputs[0])
self.assertIn(json.dumps(domain.to_dict()), data_inputs[1])
# Complete setup that prepare_data_inputs does on temp_process
self.process.domain = domain
self.process.inputs = [v1, self.variable]
self.process.parameters = {
'feature': cwt.NamedParameter('feature', 'test'),
'axes': cwt.NamedParameter('axes', 'lats'),
}
self.assertIn(json.dumps(self.process.to_dict()), data_inputs[2])
def test_prepare_data_inputs(self):
variable = cwt.Variable('file:///test.nc', 'tas', name='v0')
domain = cwt.Domain([
cwt.Dimension('time', 0, 365),
], name='d0')
process = cwt.Process('CDAT.subset', name='subset')
process.description = mock.MagicMock()
process.description.metadata.return_value = {}
client = cwt.WPSClient('http://idontexist/wps')
data_inputs = client.prepare_data_inputs_str(process, [variable],
domain)
self.assertIn('"id": "tas|v0"', data_inputs)
self.assertIn('"id": "d0"', data_inputs)
self.assertIn('"name": "CDAT.subset"', data_inputs)
def test_partitions_limit(self):
fm = file_manager.FileManager([])
def partitions():
for x in xrange(2):
yield 'part {}'.format(x)
dsc1 = mock.MagicMock()
dsc1.partitions.return_value = partitions()
dsc1.datasets.__getitem__.return_value.get_time.return_value.isTime.return_value = True
mock_axis = mock.MagicMock(id='lat')
dsc1.datasets.__getitem__.return_value.get_variable.return_value.getAxis.return_value = mock_axis
dsc2 = mock.MagicMock()
dsc2.partitions.return_value = partitions()
fm.collections.append(dsc1)
fm.collections.append(dsc2)
domain = cwt.Domain([
cwt.Dimension('time', 20, 200),
cwt.Dimension('lat', 0, 90),
])
result = [x for x in fm.partitions(domain, limit=1)]
dsc1.partitions.assert_called_with(domain, True, 'lat')
dsc1.datasets.__getitem__.return_value.get_time.assert_called()
dsc1.datasets.__getitem__.return_value.get_axis.assert_not_called()
dsc2.close.assert_called()
self.assertEqual(len(result), 2)
self.assertEqual(len(result[0]), 1)
def test_workflow(self):
self.backend.populate_processes()
process = models.Process.objects.get(identifier='CDAT.subset')
job = models.Job.objects.create(server=self.server, user=self.user, process=process)
var = cwt.Variable('file:///test.nc', 'tas', name='v0')
variables = {'v0': var}
domain = cwt.Domain([
cwt.Dimension('time', 0, 200)
], name='d0')
domains = {'d0': domain}
proc1 = cwt.Process(identifier='CDAT.aggregate', name='aggregate')
proc1.set_inputs('v0')
proc2 = cwt.Process(identifier='CDAT.subset', name='subset')
proc2.set_inputs('aggregate')
proc3 = cwt.Process(identifier='CDSpark.max', name='max')
proc3.set_inputs('subset')
proc4 = cwt.Process(identifier='Oph.avg', name='avg')
proc4.set_inputs('max')
operations = {'aggregate': proc1, 'subset': proc2, 'max': proc3, 'avg': proc4}
workflow = self.backend.workflow(proc4, variables, domains, operations, user=self.user, job=job)
def test_execute(self):
self.backend.populate_processes()
process = models.Process.objects.get(identifier='CDAT.subset')
job = models.Job.objects.create(server=self.server, user=self.user, process=process)
var = cwt.Variable('file:///test.nc', 'tas', name='v0')
variables = {'v0': var}
domain = cwt.Domain([
cwt.Dimension('time', 0, 200)
], name='d0')
domains = {'d0': domain}
proc = cwt.Process(identifier='CDAT.subset', name='subset')
operations = {'subset': proc}
proc = self.backend.execute('CDAT.subset', variables, domains, operations, user=self.user, job=job)
self.assertIsNotNone(proc)
def _get_domain(workflow_inputs):
# type: (JSON) -> Optional[cwt.Domain]
dimensions_names = [
InputNames.TIME,
InputNames.LAT,
InputNames.LON,
]
grouped_inputs = defaultdict(dict)
for dim_name in dimensions_names:
for param, v in workflow_inputs.items():
if param.startswith(dim_name + "_"):
param_split = param.split("_", 1)[1]
grouped_inputs[dim_name][param_split] = v
# grouped_inputs is of the form:
# {"lat": {"start": 1, "end": 3, "crs": "values"}}
# ensure data is casted properly
for dim_name, values in grouped_inputs.items():
for value_name, value in values.items():
if value_name in [InputArguments.START, InputArguments.END
] and value:
values[value_name] = float(value)
allowed_crs = {
c.name: c
for c in [cwt.VALUES, cwt.INDICES, cwt.TIMESTAMPS]
}
allowed_crs[None] = None
# fix unintuitive latitude that must be given 'reversed' (start is larger than end)
if InputNames.LAT in grouped_inputs:
values = grouped_inputs[InputNames.LAT][InputArguments.START], \
grouped_inputs[InputNames.LAT][InputArguments.END]
grouped_inputs[InputNames.LAT][InputArguments.START] = max(values)
grouped_inputs[InputNames.LAT][InputArguments.END] = min(values)
dimensions = []
for param_name, values in grouped_inputs.items():
for start_end in [InputArguments.START, InputArguments.END]:
if start_end not in values:
raise ValueError(
"Missing required parameter: {}_{}".format(
param_name, start_end))
crs = cwt.VALUES
if InputArguments.CRS in values:
if values[InputArguments.CRS] not in allowed_crs:
raise ValueError("CRS must be in {}".format(", ".join(
map(str, allowed_crs))))
crs = allowed_crs[values[InputArguments.CRS]]
dimension = cwt.Dimension(param_name,
values[InputArguments.START],
values[InputArguments.END],
crs=crs)
dimensions.append(dimension)
if dimensions:
domain = cwt.Domain(dimensions)
return domain
