def test_get_parameter(self): process = cwt.Process.from_identifier('CDAT.subset') process.parameters['test'] = cwt.NamedParameter('test', 'value1') param = process.get_parameter('test') self.assertEqual(param.name, 'test')
def test_prepare_data_inputs_preserve_process(self): v1 = cwt.Variable('file:///test1.nc', 'tas') self.process.add_inputs(v1) self.process.add_parameters(feature='test') domain = cwt.Domain(time=(10, 20), lat=(-90, 0)) self.process.domain = domain data_inputs = self.client.prepare_data_inputs(self.process, [ self.variable, ], domain=None, axes='lats') # Check original process is untouched self.assertEqual(self.process.inputs, [v1]) self.assertEqual(self.process.domain, domain) self.assertEqual(self.process.parameters, {'feature': cwt.NamedParameter('feature', 'test')}) # Verify the outputs self.assertEqual(len(data_inputs), 3) self.assertIn(json.dumps(self.variable.to_dict()), data_inputs[0]) self.assertIn(json.dumps(domain.to_dict()), data_inputs[1]) # Complete setup that prepare_data_inputs does on temp_process self.process.domain = domain self.process.inputs = [v1, self.variable] self.process.parameters = { 'feature': cwt.NamedParameter('feature', 'test'), 'axes': cwt.NamedParameter('axes', 'lats'), } self.assertIn(json.dumps(self.process.to_dict()), data_inputs[2])
def test_parameterize(self): expected = { 'input': ['tas', 'sum'], 'axes': 'x|y', 'name': 'CDAT.avg', 'result': 'avg' } self.sum.inputs = [self.clt, self.tas] self.avg.inputs = [self.tas, self.sum] self.avg.add_parameters(cwt.NamedParameter('axes', ('x', 'y'))) self.assertDictContainsSubset(expected, self.avg.parameterize())
def add_parameters(self, *args, **kwargs): """ Add a parameter. kwargs can contain two formats. k=v where v is a NamedParameter Args: args: A list of NamedParameter objects. kwargs: A dict of NamedParameter objects or k=v where k is the name and v is string. """ for a in args: if not isinstance(a, cwt.NamedParameter): raise ProcessError('Invalid parameter type "{}", should be a cwt.NamedParameter', type(a)) self.parameters[a.name] = a for k, v in kwargs.iteritems(): if not isinstance(v, (tuple, list)): v = [v] self.parameters[k] = cwt.NamedParameter(k, *v)
def __prepare_data_inputs(self, process, inputs, domain, **kwargs): """ Preparse a process inputs for the data_inputs string. Args: process: A object of type Process to be executed. inputs: A list of Variables/Operations. domain: A Domain object. kwargs: A dict of addiontal named_parameters or k=v pairs. Returns: A dictionary containing the operations, domains and variables associated with the process. """ domains = [] if domain is not None: process.domain = domain domains.append(domain.parameterize()) parameters = [cwt.NamedParameter(x, y) for x, y in kwargs.iteritems()] process.inputs.extend(inputs) process.add_parameters(*parameters) processes, variables = process.collect_input_processes() variables = [x.parameterize() for x in variables] operation = [process.parameterize() ] + [x.parameterize() for x in processes] return { 'variable': variables, 'domain': domains, 'operation': operation }
if gridder is not None: op.parameters['gridder'] = gridder with cdms2.open(TEST_OUTPUT, 'w') as out_file: value.value = proc.retrieve(op, len(op.inputs), out_file) stats = os.path.join(output_dir, '{}.txt'.format(stats)) var_name = execute(stats, test) with cdms2.open(TEST_OUTPUT) as in_file: if in_file[var_name].shape != result: raise Exception('File shape {} does not match {} test {}'.format(in_file[var_name].shape, result, stats)) benchmark_process = [ ('CDAT.max', dataset2, 1, None, cwt.NamedParameter('axes', 'time'), None, MV.max, (45, 72), 'max'), ('CDAT.max', dataset2, 1, dataset2_domain1, cwt.NamedParameter('axes', 'time'), None, MV.max, (45, 72), 'max_domain1'), ('CDAT.max', dataset2, 1, dataset2_domain2, cwt.NamedParameter('axes', 'time'), None, MV.max, (23, 19), 'max_domain2'), ] for identifier, inputs, num_inputs, domain, param, gridder, operation, result, stats in benchmark_process: clean() def test(value): proc = process.Process(0) proc.job = mock.MagicMock() op = cwt.Process(identifier=identifier) op.set_inputs(*inputs)
def prepare_data_inputs(self, process, inputs, domain, **kwargs): """ Preparse a process inputs for the data_inputs string. Args: process: A object of type Process to be executed. inputs: A list of Variables/Operations. domain: A Domain object. kwargs: A dict of addiontal named_parameters or k=v pairs. Returns: A dictionary containing the operations, domains and variables associated with the process. """ if process.description is None: self.describe_process(process) new_process = cwt.Process.from_identifier(process.identifier) new_process.description = process.description new_process.name = process.name new_process.domain = process.domain domains = {} if domain is not None: domains[domain.name] = domain.parameterize() new_process.domain = domain new_process.inputs = [x for x in process.inputs] new_process.inputs.extend(inputs) new_process.validate() parameters = [cwt.NamedParameter(x, y) for x, y in kwargs.iteritems()] new_process.add_parameters(*parameters) if len(process.parameters) > 0: new_process.add_parameters(*process.parameters.values()) processes, variables = new_process.collect_input_processes() variables = [x.parameterize() for x in variables] for x in [new_process] + processes: if x.domain is not None and x.domain.name not in domains: domains[x.domain.name] = x.domain.parameterize() domains = domains.values() operation = [new_process.parameterize() ] + [x.parameterize() for x in processes] return { 'variable': variables, 'domain': domains, 'operation': operation }
def test_get_parameter_missing(self): process = cwt.Process.from_identifier('CDAT.subset') process.parameters['test'] = cwt.NamedParameter('test', 'value1') self.assertIsNone(process.get_parameter('test2'))
def test_add_parameter(self): process = cwt.Process.from_identifier('CDAT.subset') process.add_parameters(cwt.NamedParameter('test', 'value1')) self.assertEqual(len(process.parameters), 1)
def test_parameters_property(self): axes = cwt.NamedParameter('axes', ('x', 'y')) self.avg.parameters = [axes] self.assertEqual(axes, self.avg.parameters[0])