def test_map_domain(self):
domain = cwt.Domain([
cwt.Dimension('time', 20, 200, cwt.INDICES),
cwt.Dimension('lat', 45, -45),
])
mock_var = mock.MagicMock()
mock_var.getAxisIndex.side_effect = [0, 1]
mock_axis1 = mock.MagicMock()
mock_axis1.isTime.return_value = True
mock_axis1.id = 'time'
mock_axis1.shape = (200,)
mock_axis2 = mock.MagicMock()
mock_axis2.isTime.return_value = False
mock_axis2.id = 'lat'
mock_axis2.shape = (180,)
mock_var.getAxis.side_effect = [
mock_axis1,
mock_axis2
]
self.ds.variable = mock_var
self.ds.map_domain(domain, 'days since 1990')
self.assertEqual(self.ds.temporal, slice(20, 200, 1))
self.assertIn('lat', self.ds.spatial)
self.assertEqual(self.ds.spatial['lat'], (45, -45))
def test_map_domain_missing_axis(self):
domain = cwt.Domain([
cwt.Dimension('time', 20, 200, cwt.INDICES),
cwt.Dimension('lat', 45, -45),
])
mock_var = mock.MagicMock()
mock_var.getAxisIndex.side_effect = [0, -1]
mock_axis1 = mock.MagicMock()
mock_axis1.isTime.return_value = True
mock_axis1.id = 'time'
mock_axis1.shape = (200,)
mock_axis2 = mock.MagicMock()
mock_axis2.isTime.return_value = False
mock_axis2.id = 'lat'
mock_axis2.shape = (180,)
mock_var.getAxis.side_effect = [
mock_axis1,
mock_axis2
]
self.ds.variable = mock_var
with self.assertRaises(tasks.WPSError):
self.ds.map_domain(domain, 'days since 1990')
def test_partitions(self):
domain = cwt.Domain([
cwt.Dimension('time', 20, 100),
cwt.Dimension('lat', 0, 90),
])
mock_cache = mock.MagicMock()
mock_cache_obj = mock.MagicMock()
collection = file_manager.DataSetCollection()
collection.check_cache = mock.MagicMock(return_value=(mock_cache, mock_cache_obj))
mock_dataset1 = mock.MagicMock()
mock_dataset1.variable_name = 'tas'
mock_dataset1.get_time.return_value.units = 'days since 2000'
mock_dataset1.partitions.return_value = [
mock.MagicMock(),
]
mock_dataset2 = mock.MagicMock()
mock_dataset2.variable_name = 'tas'
mock_dataset2.get_time.return_value.units = 'days since 1990'
mock_dataset2.partitions.return_value = [
mock.MagicMock(),
]
collection.datasets = [
mock_dataset1,
mock_dataset2,
]
result = [x for x in collection.partitions(domain, False)]
mock_dataset1.partitions.return_value[0].getTime.return_value.toRelativeTime.assert_called_with('days since 1990')
mock_dataset2.partitions.return_value[0].getTime.return_value.toRelativeTime.assert_called_with('days since 1990')
mock_cache_obj.write.assert_has_calls([
mock.call(mock_dataset2.partitions.return_value[0], id='tas'),
mock.call(mock_dataset1.partitions.return_value[0], id='tas'),
])
mock_cache_obj.sync.assert_called()
mock_cache.set_size.assert_called()
mock_dataset1.map_domain.assert_called_with(domain, 'days since 1990')
mock_dataset2.map_domain.assert_called_with(domain, 'days since 1990')
self.assertEqual(collection.datasets[0], mock_dataset2)
self.assertEqual(collection.datasets[1], mock_dataset1)
self.assertEqual(result[0], (mock_dataset2, mock_dataset2.partitions.return_value[0]))
self.assertEqual(result[1], (mock_dataset1, mock_dataset1.partitions.return_value[0]))
def test_prepare_data_inputs_parameters(self):
variable = cwt.Variable('file:///test.nc', 'tas', name='v0')
domain = cwt.Domain([
cwt.Dimension('time', 0, 365),
], name='d0')
process = cwt.Process('CDAT.subset', name='subset')
process.description = mock.MagicMock()
process.description.metadata.return_value = {}
client = cwt.WPSClient('http://idontexist/wps')
data_inputs = client.prepare_data_inputs_str(process, [variable],
domain,
axes=['lats', 'lons'],
weightoptions='generated',
test=cwt.NamedParameter(
'test', 'True'))
self.assertIn('"axes": "lats|lons"', data_inputs)
self.assertIn('"weightoptions": "generated"', data_inputs)
self.assertIn('"test": "True"', data_inputs)
def test_script_generator(self):
user = models.User.objects.first()
variables = {
'v0': cwt.Variable('file:///test.nc', 'tas', name='v0'),
'v1': cwt.Variable('file:///test.nc', 'tas', name='v1'),
}
domains = {
'd0': cwt.Domain([cwt.Dimension('time', 0, 200)], name='d0')
}
gridder = cwt.Gridder(grid='gaussian~32')
op = cwt.Process(identifier='CDAT.subset')
op.domain = domains.values()[0]
op.add_inputs(*variables.values())
op.parameters['gridder'] = gridder
op.parameters['axes'] = cwt.NamedParameter('axes', 'time')
operations = {'subset': op}
sg = wps_service.WPSScriptGenerator(variables, domains, operations,
user)
data = sg.generate()
self.assertIsNotNone(data)
def test_execute_block(self, mock_request):
mock_request.return_value.status_code = 200
mock_request.return_value.text = self.execute.toxml(bds=cwt.bds)
client = cwt.WPSClient('http://idontexist/wps')
process = cwt.Process.from_identifier('CDAT.subset')
with mock.patch('cwt.process.Process.output',
new_callable=mock.PropertyMock) as output:
output.return_value = 'test output'
process.wait = mock.MagicMock()
process.description = mock.MagicMock()
result = client.execute(process,
[cwt.Variable('file:///test.nc', 'tas')],
cwt.Domain([cwt.Dimension('time', 0,
365)]),
block=True)
process.wait.assert_called()
self.assertEqual(result, 'test output')
def test_execute(self):
process = models.Process.objects.create(identifier='CDSpark.max',
backend='EDAS')
job = models.Job.objects.create(server=self.server,
user=self.user,
process=process)
domain = cwt.Domain([cwt.Dimension('time', 0, 200)], name='d0')
domains = {'d0': domain}
var = cwt.Variable('file:///test.nc', 'tas', name='v0')
variables = {'v0': var}
proc = cwt.Process(identifier='CDSpark.max', name='max')
proc.domain = 'd0'
proc.set_inputs('v0')
operations = {'max': proc}
task = self.backend.execute('CDSpark.max',
variables,
domains,
operations,
user=self.user,
job=job)
self.assertIsNotNone(task)
def test_map_axis_unknown_crs(self, target):
axis = mock.MagicMock()
axis.isTime.return_value = False
dimension = cwt.Dimension('time', 0, 365, cwt.CRS('test'))
with self.assertRaises(WPSError):
preprocess.map_axis(axis, dimension, 'days since 2020')
def test_map_axis_indices(self, target):
axis = mock.MagicMock()
axis.isTime.return_value = False
dimension = cwt.Dimension('time', 0, 365, cwt.INDICES)
preprocess.map_axis(axis, dimension, 'days since 2020')
target.assert_called()
def setUp(self):
self.data = {
'uri': 'file:///tas.nc',
'id': 'tas|tas1',
'domain': 'd0',
'mime_type': 'application/netcdf',
}
self.d0 = cwt.Domain([cwt.Dimension('time', 1980, 2000)], name='d0')
def test_map_axis_time_axis_no_units(self):
axis = mock.MagicMock()
axis.isTime.return_value = True
dimension = cwt.Dimension('time', 0, 365, cwt.TIMESTAMPS)
preprocess.map_axis(axis, dimension, None)
axis.clone.assert_not_called()
def test_execute_failed(self, mock_request):
mock_request.return_value.status_code = 200
mock_request.return_value.text = self.execute_failed.toxml(bds=cwt.bds)
client = cwt.WPSClient('http://idontexist/wps')
process = cwt.Process.from_identifier('CDAT.subset')
with self.assertRaises(Exception):
client.execute(process, [cwt.Variable('file:///test.nc', 'tas')],
cwt.Domain([cwt.Dimension('time', 0, 365)]))
def test_parameterize(self):
process = cwt.Process.from_identifier('CDAT.subset')
process.set_domain(cwt.Domain([
cwt.Dimension('time', 0, 365),
]))
process.add_parameters(test=['value1'])
process.add_inputs(cwt.Variable('file:///test.nc', 'tas'))
data = process.parameterize()
def test_map_axis_time_axis(self):
axis = mock.MagicMock()
axis.isTime.return_value = True
dimension = cwt.Dimension('time', 0, 365, cwt.TIMESTAMPS)
preprocess.map_axis(axis, dimension, 'days since 2020')
axis.clone.assert_called()
axis.clone.return_value.toRelativeTime.assert_called_with(
'days since 2020')
def test_map_axis_values(self, map_axis_interval):
map_axis_interval.return_value = (0, 365)
axis = mock.MagicMock()
dimension = cwt.Dimension('time', 0, 365)
selector = preprocess.map_axis_values(axis, dimension)
self.assertEqual(select, slice(0, 365, 1))
map_axis_timestamps.assert_called_with(axis, 0, 365)
def test_map_axis_indices_dimension(self):
axis = mock.MagicMock()
axis.isTime.return_vale = False
axis.__len__.return_value = 365
dimension = cwt.Dimension('time', 20, 300, cwt.INDICES)
selector = preprocess.map_axis_indices(axis, dimension)
self.assertEqual(selector, slice(20, 300, 1))
self.assertEqual(dimension.start, 0)
self.assertEqual(dimension.end, 0)
def test_partitions_limit(self):
fm = file_manager.FileManager([])
def partitions():
for x in xrange(2):
yield 'part {}'.format(x)
dsc1 = mock.MagicMock()
dsc1.partitions.return_value = partitions()
dsc1.datasets.__getitem__.return_value.get_time.return_value.isTime.return_value = True
mock_axis = mock.MagicMock(id='lat')
dsc1.datasets.__getitem__.return_value.get_variable.return_value.getAxis.return_value = mock_axis
dsc2 = mock.MagicMock()
dsc2.partitions.return_value = partitions()
fm.collections.append(dsc1)
fm.collections.append(dsc2)
domain = cwt.Domain([
cwt.Dimension('time', 20, 200),
cwt.Dimension('lat', 0, 90),
])
result = [x for x in fm.partitions(domain, limit=1)]
dsc1.partitions.assert_called_with(domain, True, 'lat')
dsc1.datasets.__getitem__.return_value.get_time.assert_called()
dsc1.datasets.__getitem__.return_value.get_axis.assert_not_called()
dsc2.close.assert_called()
self.assertEqual(len(result), 2)
self.assertEqual(len(result[0]), 1)
def test_parameterize(self):
expected = {
'id': 'd0',
'lat': {
'start': 0,
'end': 90,
'step': 1,
'crs': 'values'
}
}
dom = cwt.Domain([cwt.Dimension('lat', 0, 90)], name='d0')
self.assertDictContainsSubset(expected, dom.parameterize())
def test_api_key(self, mock_request):
mock_request.return_value.status_code = 200
mock_request.return_value.text = self.execute.toxml(bds=cwt.bds)
client = cwt.WPSClient('http://idontexist/wps', api_key='api_key_7')
process = cwt.Process.from_identifier('CDAT.subset')
process.description = mock.MagicMock()
response = client.execute(process,
[cwt.Variable('file:///test.nc', 'tas')],
cwt.Domain([cwt.Dimension('time', 0, 365)]))
def test_dimension_to_selector_unknown_crs(self):
mock_axis_clone = mock.MagicMock()
mock_axis_clone.mapInterval.return_value = (2, 5)
mock_axis = mock.MagicMock()
mock_axis.isTime.return_value = True
mock_axis.clone.return_value = mock_axis_clone
dimension = cwt.Dimension('time', 20, 100, cwt.CRS('new'))
with self.assertRaises(tasks.WPSError):
selector = self.ds.dimension_to_selector(dimension, mock_axis, 'days since 1990')
def test_dimension_to_selector_type_error(self):
mock_axis_clone = mock.MagicMock()
mock_axis_clone.mapInterval.side_effect = TypeError()
mock_axis = mock.MagicMock()
mock_axis.isTime.return_value = True
mock_axis.clone.return_value = mock_axis_clone
dimension = cwt.Dimension('time', 20, 100)
with self.assertRaises(tasks.DomainMappingError):
selector = self.ds.dimension_to_selector(dimension, mock_axis, 'days since 1990')
def test_execute_missing_operation(self):
mock_job = mock.MagicMock()
variables = {
'v0': cwt.Variable('file:///test.nc', 'tas', name='v0'),
}
domains = {'d0': cwt.Domain([cwt.Dimension('time', 0, 200)])}
with self.assertRaises(WPSError) as e:
self.backend.execute('Oph.max',
variables,
domains, {},
job=mock_job,
user=self.user)
def test_execute_no_inputs(self, mock_request):
mock_request.return_value.status_code = 200
mock_request.return_value.text = self.execute.toxml(bds=cwt.bds)
client = cwt.WPSClient('http://idontexist/wps')
process = cwt.Process.from_identifier('CDAT.subset')
process.description = mock.MagicMock()
client.execute(process, None,
cwt.Domain([cwt.Dimension('time', 0, 365)]))
self.assertIsNotNone(process.response)
def test_dimension_to_selector_spatial(self):
mock_axis_clone = mock.MagicMock()
mock_axis_clone.mapInterval.return_value = (2, 5)
mock_axis = mock.MagicMock()
mock_axis.isTime.return_value = False
mock_axis.clone.return_value = mock_axis_clone
dimension = cwt.Dimension('time', 20, 100)
selector = self.ds.dimension_to_selector(dimension, mock_axis, 'days since 1990')
self.assertEqual(selector, (20, 100))
def test_execute_get(self, mock_request):
mock_request.return_value.status_code = 200
mock_request.return_value.text = self.execute.toxml(bds=cwt.bds)
client = cwt.WPSClient('http://idontexist/wps')
process = cwt.Process.from_identifier('CDAT.subset')
process.description = mock.MagicMock()
client.execute(process, [cwt.Variable('file:///test.nc', 'tas')],
cwt.Domain([cwt.Dimension('time', 0, 365)]),
method='GET')
self.assertIsNotNone(process.response)
def test_execute_no_duplicates(self, mock_request):
mock_request.return_value.status_code = 200
mock_request.return_value.text = self.execute.toxml(bds=cwt.bds)
client = cwt.WPSClient('http://idontexist/wps')
process = cwt.Process.from_identifier('CDAT.subset')
process.description = mock.MagicMock()
client.execute(process, [cwt.Variable('file:///test.nc', 'tas')],
cwt.Domain([cwt.Dimension('time', 0, 365)]),
axes=['time'])
self.assertEqual(len(process.parameters), 0)
self.assertIsNone(process.domain)
self.assertEqual(len(process.inputs), 0)
self.assertIsNotNone(process.response)
def test_dimension_to_selector(self):
mock_axis_clone = mock.MagicMock()
mock_axis_clone.mapInterval.return_value = (2, 5)
mock_axis = mock.MagicMock()
mock_axis.isTime.return_value = True
mock_axis.clone.return_value = mock_axis_clone
dimension = cwt.Dimension('time', 20, 100)
selector = self.ds.dimension_to_selector(dimension, mock_axis, 'days since 1990')
self.assertEqual(selector, slice(2, 5))
mock_axis_clone.toRelativeTime.assert_called_with('days since 1990')
mock_axis_clone.mapInterval.assert_called_with((20, 100))
def test_execute(self):
mock_job = mock.MagicMock()
variables = {
'v0': cwt.Variable('file:///test.nc', 'tas', name='v0'),
}
domains = {'d0': cwt.Domain([cwt.Dimension('time', 0, 200)])}
operation = cwt.Process(identifier='Oph.max', name='max')
operation.domain = 'd0'
result = self.backend.execute('Oph.max',
variables,
domains, {'max': operation},
job=mock_job,
user=self.user)
self.assertIsNotNone(result)
def test_prepare_data_inputs(self):
variable = cwt.Variable('file:///test.nc', 'tas', name='v0')
domain = cwt.Domain([
cwt.Dimension('time', 0, 365),
], name='d0')
process = cwt.Process('CDAT.subset', name='subset')
process.description = mock.MagicMock()
process.description.metadata.return_value = {}
client = cwt.WPSClient('http://idontexist/wps')
data_inputs = client.prepare_data_inputs_str(process, [variable],
domain)
self.assertIn('"id": "tas|v0"', data_inputs)
self.assertIn('"id": "d0"', data_inputs)
self.assertIn('"name": "CDAT.subset"', data_inputs)
def test_dimension_to_selector_indices(self):
mock_axis_clone = mock.MagicMock()
mock_axis_clone.mapInterval.return_value = (2, 5)
mock_axis = mock.MagicMock()
mock_axis.shape = (80,)
mock_axis.isTime.return_value = True
mock_axis.clone.return_value = mock_axis_clone
dimension = cwt.Dimension('time', 20, 100, cwt.INDICES)
selector = self.ds.dimension_to_selector(dimension, mock_axis, 'days since 1990')
self.assertEqual(selector, slice(20, 80, 1))
self.assertEqual(dimension.start, 0)
self.assertEqual(dimension.end, 20)
