def test_get_parameter(self):
process = cwt.Process.from_identifier('CDAT.subset')
process.parameters['test'] = cwt.NamedParameter('test', 'value1')
param = process.get_parameter('test')
self.assertEqual(param.name, 'test')
def test_prepare_data_inputs_preserve_process(self):
v1 = cwt.Variable('file:///test1.nc', 'tas')
self.process.add_inputs(v1)
self.process.add_parameters(feature='test')
domain = cwt.Domain(time=(10, 20), lat=(-90, 0))
self.process.domain = domain
data_inputs = self.client.prepare_data_inputs(self.process, [
self.variable,
],
domain=None,
axes='lats')
# Check original process is untouched
self.assertEqual(self.process.inputs, [v1])
self.assertEqual(self.process.domain, domain)
self.assertEqual(self.process.parameters,
{'feature': cwt.NamedParameter('feature', 'test')})
# Verify the outputs
self.assertEqual(len(data_inputs), 3)
self.assertIn(json.dumps(self.variable.to_dict()), data_inputs[0])
self.assertIn(json.dumps(domain.to_dict()), data_inputs[1])
# Complete setup that prepare_data_inputs does on temp_process
self.process.domain = domain
self.process.inputs = [v1, self.variable]
self.process.parameters = {
'feature': cwt.NamedParameter('feature', 'test'),
'axes': cwt.NamedParameter('axes', 'lats'),
}
self.assertIn(json.dumps(self.process.to_dict()), data_inputs[2])
def test_parameterize(self):
expected = {
'input': ['tas', 'sum'],
'axes': 'x|y',
'name': 'CDAT.avg',
'result': 'avg'
}
self.sum.inputs = [self.clt, self.tas]
self.avg.inputs = [self.tas, self.sum]
self.avg.add_parameters(cwt.NamedParameter('axes', ('x', 'y')))
self.assertDictContainsSubset(expected, self.avg.parameterize())
def add_parameters(self, *args, **kwargs):
""" Add a parameter.
kwargs can contain two formats.
k=v where v is a NamedParameter
Args:
args: A list of NamedParameter objects.
kwargs: A dict of NamedParameter objects or k=v where k is the name and v is string.
"""
for a in args:
if not isinstance(a, cwt.NamedParameter):
raise ProcessError('Invalid parameter type "{}", should be a cwt.NamedParameter', type(a))
self.parameters[a.name] = a
for k, v in kwargs.iteritems():
if not isinstance(v, (tuple, list)):
v = [v]
self.parameters[k] = cwt.NamedParameter(k, *v)
def __prepare_data_inputs(self, process, inputs, domain, **kwargs):
""" Preparse a process inputs for the data_inputs string.
Args:
process: A object of type Process to be executed.
inputs: A list of Variables/Operations.
domain: A Domain object.
kwargs: A dict of addiontal named_parameters or k=v pairs.
Returns:
A dictionary containing the operations, domains and variables
associated with the process.
"""
domains = []
if domain is not None:
process.domain = domain
domains.append(domain.parameterize())
parameters = [cwt.NamedParameter(x, y) for x, y in kwargs.iteritems()]
process.inputs.extend(inputs)
process.add_parameters(*parameters)
processes, variables = process.collect_input_processes()
variables = [x.parameterize() for x in variables]
operation = [process.parameterize()
] + [x.parameterize() for x in processes]
return {
'variable': variables,
'domain': domains,
'operation': operation
}
if gridder is not None:
op.parameters['gridder'] = gridder
with cdms2.open(TEST_OUTPUT, 'w') as out_file:
value.value = proc.retrieve(op, len(op.inputs), out_file)
stats = os.path.join(output_dir, '{}.txt'.format(stats))
var_name = execute(stats, test)
with cdms2.open(TEST_OUTPUT) as in_file:
if in_file[var_name].shape != result:
raise Exception('File shape {} does not match {} test {}'.format(in_file[var_name].shape, result, stats))
benchmark_process = [
('CDAT.max', dataset2, 1, None, cwt.NamedParameter('axes', 'time'), None, MV.max, (45, 72), 'max'),
('CDAT.max', dataset2, 1, dataset2_domain1, cwt.NamedParameter('axes', 'time'), None, MV.max, (45, 72), 'max_domain1'),
('CDAT.max', dataset2, 1, dataset2_domain2, cwt.NamedParameter('axes', 'time'), None, MV.max, (23, 19), 'max_domain2'),
]
for identifier, inputs, num_inputs, domain, param, gridder, operation, result, stats in benchmark_process:
clean()
def test(value):
proc = process.Process(0)
proc.job = mock.MagicMock()
op = cwt.Process(identifier=identifier)
op.set_inputs(*inputs)
def prepare_data_inputs(self, process, inputs, domain, **kwargs):
""" Preparse a process inputs for the data_inputs string.
Args:
process: A object of type Process to be executed.
inputs: A list of Variables/Operations.
domain: A Domain object.
kwargs: A dict of addiontal named_parameters or k=v pairs.
Returns:
A dictionary containing the operations, domains and variables
associated with the process.
"""
if process.description is None:
self.describe_process(process)
new_process = cwt.Process.from_identifier(process.identifier)
new_process.description = process.description
new_process.name = process.name
new_process.domain = process.domain
domains = {}
if domain is not None:
domains[domain.name] = domain.parameterize()
new_process.domain = domain
new_process.inputs = [x for x in process.inputs]
new_process.inputs.extend(inputs)
new_process.validate()
parameters = [cwt.NamedParameter(x, y) for x, y in kwargs.iteritems()]
new_process.add_parameters(*parameters)
if len(process.parameters) > 0:
new_process.add_parameters(*process.parameters.values())
processes, variables = new_process.collect_input_processes()
variables = [x.parameterize() for x in variables]
for x in [new_process] + processes:
if x.domain is not None and x.domain.name not in domains:
domains[x.domain.name] = x.domain.parameterize()
domains = domains.values()
operation = [new_process.parameterize()
] + [x.parameterize() for x in processes]
return {
'variable': variables,
'domain': domains,
'operation': operation
}
def test_get_parameter_missing(self):
process = cwt.Process.from_identifier('CDAT.subset')
process.parameters['test'] = cwt.NamedParameter('test', 'value1')
self.assertIsNone(process.get_parameter('test2'))
def test_add_parameter(self):
process = cwt.Process.from_identifier('CDAT.subset')
process.add_parameters(cwt.NamedParameter('test', 'value1'))
self.assertEqual(len(process.parameters), 1)
def test_parameters_property(self):
axes = cwt.NamedParameter('axes', ('x', 'y'))
self.avg.parameters = [axes]
self.assertEqual(axes, self.avg.parameters[0])
