def test_prepare_data_inputs_parameters(self): variable = cwt.Variable('file:///test.nc', 'tas', name='v0') domain = cwt.Domain([ cwt.Dimension('time', 0, 365), ], name='d0') process = cwt.Process('CDAT.subset', name='subset') process.description = mock.MagicMock() process.description.metadata.return_value = {} client = cwt.WPSClient('http://idontexist/wps') data_inputs = client.prepare_data_inputs_str(process, [variable], domain, axes=['lats', 'lons'], weightoptions='generated', test=cwt.NamedParameter( 'test', 'True')) self.assertIn('"axes": "lats|lons"', data_inputs) self.assertIn('"weightoptions": "generated"', data_inputs) self.assertIn('"test": "True"', data_inputs)
def test_execute(self): process = models.Process.objects.create(identifier='CDSpark.max', backend='EDAS') job = models.Job.objects.create(server=self.server, user=self.user, process=process) domain = cwt.Domain([cwt.Dimension('time', 0, 200)], name='d0') domains = {'d0': domain} var = cwt.Variable('file:///test.nc', 'tas', name='v0') variables = {'v0': var} proc = cwt.Process(identifier='CDSpark.max', name='max') proc.domain = 'd0' proc.set_inputs('v0') operations = {'max': proc} task = self.backend.execute('CDSpark.max', variables, domains, operations, user=self.user, job=job) self.assertIsNotNone(task)
def test_script_generator(self): user = models.User.objects.first() variables = { 'v0': cwt.Variable('file:///test.nc', 'tas', name='v0'), 'v1': cwt.Variable('file:///test.nc', 'tas', name='v1'), } domains = { 'd0': cwt.Domain([cwt.Dimension('time', 0, 200)], name='d0') } gridder = cwt.Gridder(grid='gaussian~32') op = cwt.Process(identifier='CDAT.subset') op.domain = domains.values()[0] op.add_inputs(*variables.values()) op.parameters['gridder'] = gridder op.parameters['axes'] = cwt.NamedParameter('axes', 'time') operations = {'subset': op} sg = wps_service.WPSScriptGenerator(variables, domains, operations, user) data = sg.generate() self.assertIsNotNone(data)
def setUp(self): self.client = cwt.WPSClient('https://0.0.0.0:10000/wps') # Mock owslib.WebProcessingService self.client.client = mock.MagicMock() subset = mock.MagicMock() type(subset).identifier = mock.PropertyMock(return_value='CDAT.subset') type(subset).title = mock.PropertyMock(return_value='CDAT.subset') type(subset).processVersion = mock.PropertyMock(return_value='1.0.0') metrics = mock.MagicMock() type(metrics).identifier = mock.PropertyMock( return_value='CDAT.metrics') type(metrics).title = mock.PropertyMock(return_value='CDAT.metrics') type(metrics).processVersion = mock.PropertyMock(return_value='1.0.0') type(self.client.client).processes = mock.PropertyMock(return_value=[ subset, metrics, ]) self.process = cwt.Process.from_dict({ 'name': 'CDAT.subset', 'input': [], 'domain': None, 'result': 'p0', }) self.domain = cwt.Domain(time=(0, 365), name='d0') self.variable = cwt.Variable('file:///test.nc', 'tas', name='v0')
def test_map_domain(self): domain = cwt.Domain([ cwt.Dimension('time', 20, 200, cwt.INDICES), cwt.Dimension('lat', 45, -45), ]) mock_var = mock.MagicMock() mock_var.getAxisIndex.side_effect = [0, 1] mock_axis1 = mock.MagicMock() mock_axis1.isTime.return_value = True mock_axis1.id = 'time' mock_axis1.shape = (200,) mock_axis2 = mock.MagicMock() mock_axis2.isTime.return_value = False mock_axis2.id = 'lat' mock_axis2.shape = (180,) mock_var.getAxis.side_effect = [ mock_axis1, mock_axis2 ] self.ds.variable = mock_var self.ds.map_domain(domain, 'days since 1990') self.assertEqual(self.ds.temporal, slice(20, 200, 1)) self.assertIn('lat', self.ds.spatial) self.assertEqual(self.ds.spatial['lat'], (45, -45))
def test_map_domain_missing_axis(self): domain = cwt.Domain([ cwt.Dimension('time', 20, 200, cwt.INDICES), cwt.Dimension('lat', 45, -45), ]) mock_var = mock.MagicMock() mock_var.getAxisIndex.side_effect = [0, -1] mock_axis1 = mock.MagicMock() mock_axis1.isTime.return_value = True mock_axis1.id = 'time' mock_axis1.shape = (200,) mock_axis2 = mock.MagicMock() mock_axis2.isTime.return_value = False mock_axis2.id = 'lat' mock_axis2.shape = (180,) mock_var.getAxis.side_effect = [ mock_axis1, mock_axis2 ] self.ds.variable = mock_var with self.assertRaises(tasks.WPSError): self.ds.map_domain(domain, 'days since 1990')
def test_execute_block(self, mock_request): mock_request.return_value.status_code = 200 mock_request.return_value.text = self.execute.toxml(bds=cwt.bds) client = cwt.WPSClient('http://idontexist/wps') process = cwt.Process.from_identifier('CDAT.subset') with mock.patch('cwt.process.Process.output', new_callable=mock.PropertyMock) as output: output.return_value = 'test output' process.wait = mock.MagicMock() process.description = mock.MagicMock() result = client.execute(process, [cwt.Variable('file:///test.nc', 'tas')], cwt.Domain([cwt.Dimension('time', 0, 365)]), block=True) process.wait.assert_called() self.assertEqual(result, 'test output')
def test_add_dimension_timestamps(self): dom = cwt.Domain() dom.add_dimension('lat', ('1980-01-01', '2000-01-01')) self.assertEqual(len(dom.dimensions), 1) self.assertEqual(dom.dimensions['lat'].crs, cwt.TIMESTAMPS)
def test_add_dimension_values(self): dom = cwt.Domain() dom.add_dimension('lat', (0, 20)) self.assertEqual(len(dom.dimensions), 1) self.assertEqual(dom.dimensions['lat'].crs, cwt.VALUES)
def test_add_dimension_indices(self): dom = cwt.Domain() dom.add_dimension('lat', slice(0, 20)) self.assertEqual(len(dom.dimensions), 1) self.assertEqual(dom.dimensions['lat'].crs, cwt.INDICES)
def setUp(self): self.data = { 'uri': 'file:///tas.nc', 'id': 'tas|tas1', 'domain': 'd0', 'mime_type': 'application/netcdf', } self.d0 = cwt.Domain(time=(1980, 2000), name='d0')
def test_prepare_data_inputs(self): proc = cwt.Process(type('Process', (object,), dict(identifier='CDAT.avg')), name='avg') tas = cwt.Variable('file:///data/tas_6h.nc', 'tas', name='tas1') d0 = cwt.Domain(name='d0') data_inputs = self.wps.prepare_data_inputs(proc, [tas], d0) self.assertEqual(self.data_inputs, data_inputs)
def test_partitions(self): domain = cwt.Domain([ cwt.Dimension('time', 20, 100), cwt.Dimension('lat', 0, 90), ]) mock_cache = mock.MagicMock() mock_cache_obj = mock.MagicMock() collection = file_manager.DataSetCollection() collection.check_cache = mock.MagicMock(return_value=(mock_cache, mock_cache_obj)) mock_dataset1 = mock.MagicMock() mock_dataset1.variable_name = 'tas' mock_dataset1.get_time.return_value.units = 'days since 2000' mock_dataset1.partitions.return_value = [ mock.MagicMock(), ] mock_dataset2 = mock.MagicMock() mock_dataset2.variable_name = 'tas' mock_dataset2.get_time.return_value.units = 'days since 1990' mock_dataset2.partitions.return_value = [ mock.MagicMock(), ] collection.datasets = [ mock_dataset1, mock_dataset2, ] result = [x for x in collection.partitions(domain, False)] mock_dataset1.partitions.return_value[0].getTime.return_value.toRelativeTime.assert_called_with('days since 1990') mock_dataset2.partitions.return_value[0].getTime.return_value.toRelativeTime.assert_called_with('days since 1990') mock_cache_obj.write.assert_has_calls([ mock.call(mock_dataset2.partitions.return_value[0], id='tas'), mock.call(mock_dataset1.partitions.return_value[0], id='tas'), ]) mock_cache_obj.sync.assert_called() mock_cache.set_size.assert_called() mock_dataset1.map_domain.assert_called_with(domain, 'days since 1990') mock_dataset2.map_domain.assert_called_with(domain, 'days since 1990') self.assertEqual(collection.datasets[0], mock_dataset2) self.assertEqual(collection.datasets[1], mock_dataset1) self.assertEqual(result[0], (mock_dataset2, mock_dataset2.partitions.return_value[0])) self.assertEqual(result[1], (mock_dataset1, mock_dataset1.partitions.return_value[0]))
def test_parameterize(self): process = cwt.Process.from_identifier('CDAT.subset') process.set_domain(cwt.Domain([ cwt.Dimension('time', 0, 365), ])) process.add_parameters(test=['value1']) process.add_inputs(cwt.Variable('file:///test.nc', 'tas')) data = process.parameterize()
def test_execute_failed(self, mock_request): mock_request.return_value.status_code = 200 mock_request.return_value.text = self.execute_failed.toxml(bds=cwt.bds) client = cwt.WPSClient('http://idontexist/wps') process = cwt.Process.from_identifier('CDAT.subset') with self.assertRaises(Exception): client.execute(process, [cwt.Variable('file:///test.nc', 'tas')], cwt.Domain([cwt.Dimension('time', 0, 365)]))
def execute(self, context, request, client, identifier, files, variable, domain, **kwargs): process = client.processes(identifier)[0] domain = None if domain is None else cwt.Domain(**domain) inputs = [cwt.Variable(x, variable) for x in files] client.execute(process, inputs, domain) context.set_data_inputs(request, inputs, domain, process) return process
def test_parameterize(self): expected = { 'id': 'd0', 'lat': { 'start': 0, 'end': 90, 'step': 1, 'crs': 'values' } } dom = cwt.Domain([cwt.Dimension('lat', 0, 90)], name='d0') self.assertDictContainsSubset(expected, dom.parameterize())
def test_api_key(self, mock_request): mock_request.return_value.status_code = 200 mock_request.return_value.text = self.execute.toxml(bds=cwt.bds) client = cwt.WPSClient('http://idontexist/wps', api_key='api_key_7') process = cwt.Process.from_identifier('CDAT.subset') process.description = mock.MagicMock() response = client.execute(process, [cwt.Variable('file:///test.nc', 'tas')], cwt.Domain([cwt.Dimension('time', 0, 365)]))
def test_execute_no_inputs(self, mock_request): mock_request.return_value.status_code = 200 mock_request.return_value.text = self.execute.toxml(bds=cwt.bds) client = cwt.WPSClient('http://idontexist/wps') process = cwt.Process.from_identifier('CDAT.subset') process.description = mock.MagicMock() client.execute(process, None, cwt.Domain([cwt.Dimension('time', 0, 365)])) self.assertIsNotNone(process.response)
def test_execute_missing_operation(self): mock_job = mock.MagicMock() variables = { 'v0': cwt.Variable('file:///test.nc', 'tas', name='v0'), } domains = {'d0': cwt.Domain([cwt.Dimension('time', 0, 200)])} with self.assertRaises(WPSError) as e: self.backend.execute('Oph.max', variables, domains, {}, job=mock_job, user=self.user)
def test_execute_get(self, mock_request): mock_request.return_value.status_code = 200 mock_request.return_value.text = self.execute.toxml(bds=cwt.bds) client = cwt.WPSClient('http://idontexist/wps') process = cwt.Process.from_identifier('CDAT.subset') process.description = mock.MagicMock() client.execute(process, [cwt.Variable('file:///test.nc', 'tas')], cwt.Domain([cwt.Dimension('time', 0, 365)]), method='GET') self.assertIsNotNone(process.response)
def test_execute_no_duplicates(self, mock_request): mock_request.return_value.status_code = 200 mock_request.return_value.text = self.execute.toxml(bds=cwt.bds) client = cwt.WPSClient('http://idontexist/wps') process = cwt.Process.from_identifier('CDAT.subset') process.description = mock.MagicMock() client.execute(process, [cwt.Variable('file:///test.nc', 'tas')], cwt.Domain([cwt.Dimension('time', 0, 365)]), axes=['time']) self.assertEqual(len(process.parameters), 0) self.assertIsNone(process.domain) self.assertEqual(len(process.inputs), 0) self.assertIsNotNone(process.response)
def run( self ): d0 = cwt.Domain([], name="d0") op1 = cwt.Operation.from_dict( { 'name': "CDSpark.multiAverage" } ) op1.add_input( cwt.Variable("file:///dass/nobackup/tpmaxwel/.edas/cache/collections/NCML/MERRA_TAS1hr.ncml", "tas" ) ) op3 = cwt.Operation.from_dict( { 'name': 'CDSpark.regrid', 'crs':'gaussian~128' } ) op3.add_input( op1 ) op2 = cwt.Operation.from_dict( { 'name': "CDSpark.multiAverage" } ) for i in range(1,3): op2.add_input( cwt.Variable('collection:/GISS-E2-R_r%di1p1'%(i), "tas" ) ) op4 = cwt.Operation.from_dict( { 'name': 'CDSpark.regrid', 'crs':'gaussian~128' } ) op4.add_input( op2 ) op5 = cwt.Operation.from_dict( { 'name': 'CDSpark.multiAverage' } ) op5.add_input( op3 ) op5.add_input( op4 ) wps = cwt.WPS( 'http://localhost:9001/wps', log=True, log_file=os.path.expanduser("~/esgf_api.log") ) wps.init() process = cwt.Process( wps, op5 ) process.execute( None, d0, [], True, True, "GET" )
def test_execute(self): mock_job = mock.MagicMock() variables = { 'v0': cwt.Variable('file:///test.nc', 'tas', name='v0'), } domains = {'d0': cwt.Domain([cwt.Dimension('time', 0, 200)])} operation = cwt.Process(identifier='Oph.max', name='max') operation.domain = 'd0' result = self.backend.execute('Oph.max', variables, domains, {'max': operation}, job=mock_job, user=self.user) self.assertIsNotNone(result)
def test_prepare_data_inputs_preserve_process(self): v1 = cwt.Variable('file:///test1.nc', 'tas') self.process.add_inputs(v1) self.process.add_parameters(feature='test') domain = cwt.Domain(time=(10, 20), lat=(-90, 0)) self.process.domain = domain data_inputs = self.client.prepare_data_inputs(self.process, [ self.variable, ], domain=None, axes='lats') # Check original process is untouched self.assertEqual(self.process.inputs, [v1]) self.assertEqual(self.process.domain, domain) self.assertEqual(self.process.parameters, {'feature': cwt.NamedParameter('feature', 'test')}) # Verify the outputs self.assertEqual(len(data_inputs), 3) self.assertIn(json.dumps(self.variable.to_dict()), data_inputs[0]) self.assertIn(json.dumps(domain.to_dict()), data_inputs[1]) # Complete setup that prepare_data_inputs does on temp_process self.process.domain = domain self.process.inputs = [v1, self.variable] self.process.parameters = { 'feature': cwt.NamedParameter('feature', 'test'), 'axes': cwt.NamedParameter('axes', 'lats'), } self.assertIn(json.dumps(self.process.to_dict()), data_inputs[2])
def test_prepare_data_inputs(self): variable = cwt.Variable('file:///test.nc', 'tas', name='v0') domain = cwt.Domain([ cwt.Dimension('time', 0, 365), ], name='d0') process = cwt.Process('CDAT.subset', name='subset') process.description = mock.MagicMock() process.description.metadata.return_value = {} client = cwt.WPSClient('http://idontexist/wps') data_inputs = client.prepare_data_inputs_str(process, [variable], domain) self.assertIn('"id": "tas|v0"', data_inputs) self.assertIn('"id": "d0"', data_inputs) self.assertIn('"name": "CDAT.subset"', data_inputs)
def test_partitions_limit(self): fm = file_manager.FileManager([]) def partitions(): for x in xrange(2): yield 'part {}'.format(x) dsc1 = mock.MagicMock() dsc1.partitions.return_value = partitions() dsc1.datasets.__getitem__.return_value.get_time.return_value.isTime.return_value = True mock_axis = mock.MagicMock(id='lat') dsc1.datasets.__getitem__.return_value.get_variable.return_value.getAxis.return_value = mock_axis dsc2 = mock.MagicMock() dsc2.partitions.return_value = partitions() fm.collections.append(dsc1) fm.collections.append(dsc2) domain = cwt.Domain([ cwt.Dimension('time', 20, 200), cwt.Dimension('lat', 0, 90), ]) result = [x for x in fm.partitions(domain, limit=1)] dsc1.partitions.assert_called_with(domain, True, 'lat') dsc1.datasets.__getitem__.return_value.get_time.assert_called() dsc1.datasets.__getitem__.return_value.get_axis.assert_not_called() dsc2.close.assert_called() self.assertEqual(len(result), 2) self.assertEqual(len(result[0]), 1)
def test_workflow(self): self.backend.populate_processes() process = models.Process.objects.get(identifier='CDAT.subset') job = models.Job.objects.create(server=self.server, user=self.user, process=process) var = cwt.Variable('file:///test.nc', 'tas', name='v0') variables = {'v0': var} domain = cwt.Domain([ cwt.Dimension('time', 0, 200) ], name='d0') domains = {'d0': domain} proc1 = cwt.Process(identifier='CDAT.aggregate', name='aggregate') proc1.set_inputs('v0') proc2 = cwt.Process(identifier='CDAT.subset', name='subset') proc2.set_inputs('aggregate') proc3 = cwt.Process(identifier='CDSpark.max', name='max') proc3.set_inputs('subset') proc4 = cwt.Process(identifier='Oph.avg', name='avg') proc4.set_inputs('max') operations = {'aggregate': proc1, 'subset': proc2, 'max': proc3, 'avg': proc4} workflow = self.backend.workflow(proc4, variables, domains, operations, user=self.user, job=job)
def test_execute(self): self.backend.populate_processes() process = models.Process.objects.get(identifier='CDAT.subset') job = models.Job.objects.create(server=self.server, user=self.user, process=process) var = cwt.Variable('file:///test.nc', 'tas', name='v0') variables = {'v0': var} domain = cwt.Domain([ cwt.Dimension('time', 0, 200) ], name='d0') domains = {'d0': domain} proc = cwt.Process(identifier='CDAT.subset', name='subset') operations = {'subset': proc} proc = self.backend.execute('CDAT.subset', variables, domains, operations, user=self.user, job=job) self.assertIsNotNone(proc)
def _get_domain(workflow_inputs): # type: (JSON) -> Optional[cwt.Domain] dimensions_names = [ InputNames.TIME, InputNames.LAT, InputNames.LON, ] grouped_inputs = defaultdict(dict) for dim_name in dimensions_names: for param, v in workflow_inputs.items(): if param.startswith(dim_name + "_"): param_split = param.split("_", 1)[1] grouped_inputs[dim_name][param_split] = v # grouped_inputs is of the form: # {"lat": {"start": 1, "end": 3, "crs": "values"}} # ensure data is casted properly for dim_name, values in grouped_inputs.items(): for value_name, value in values.items(): if value_name in [InputArguments.START, InputArguments.END ] and value: values[value_name] = float(value) allowed_crs = { c.name: c for c in [cwt.VALUES, cwt.INDICES, cwt.TIMESTAMPS] } allowed_crs[None] = None # fix unintuitive latitude that must be given 'reversed' (start is larger than end) if InputNames.LAT in grouped_inputs: values = grouped_inputs[InputNames.LAT][InputArguments.START], \ grouped_inputs[InputNames.LAT][InputArguments.END] grouped_inputs[InputNames.LAT][InputArguments.START] = max(values) grouped_inputs[InputNames.LAT][InputArguments.END] = min(values) dimensions = [] for param_name, values in grouped_inputs.items(): for start_end in [InputArguments.START, InputArguments.END]: if start_end not in values: raise ValueError( "Missing required parameter: {}_{}".format( param_name, start_end)) crs = cwt.VALUES if InputArguments.CRS in values: if values[InputArguments.CRS] not in allowed_crs: raise ValueError("CRS must be in {}".format(", ".join( map(str, allowed_crs)))) crs = allowed_crs[values[InputArguments.CRS]] dimension = cwt.Dimension(param_name, values[InputArguments.START], values[InputArguments.END], crs=crs) dimensions.append(dimension) if dimensions: domain = cwt.Domain(dimensions) return domain