def test_script_generator(self):
user = models.User.objects.first()
variables = {
'v0': cwt.Variable('file:///test.nc', 'tas', name='v0'),
'v1': cwt.Variable('file:///test.nc', 'tas', name='v1'),
}
domains = {
'd0': cwt.Domain([cwt.Dimension('time', 0, 200)], name='d0')
}
gridder = cwt.Gridder(grid='gaussian~32')
op = cwt.Process(identifier='CDAT.subset')
op.domain = domains.values()[0]
op.add_inputs(*variables.values())
op.parameters['gridder'] = gridder
op.parameters['axes'] = cwt.NamedParameter('axes', 'time')
operations = {'subset': op}
sg = wps_service.WPSScriptGenerator(variables, domains, operations,
user)
data = sg.generate()
self.assertIsNotNone(data)
def test_execute(self):
process = models.Process.objects.create(identifier='CDSpark.max',
backend='EDAS')
job = models.Job.objects.create(server=self.server,
user=self.user,
process=process)
domain = cwt.Domain([cwt.Dimension('time', 0, 200)], name='d0')
domains = {'d0': domain}
var = cwt.Variable('file:///test.nc', 'tas', name='v0')
variables = {'v0': var}
proc = cwt.Process(identifier='CDSpark.max', name='max')
proc.domain = 'd0'
proc.set_inputs('v0')
operations = {'max': proc}
task = self.backend.execute('CDSpark.max',
variables,
domains,
operations,
user=self.user,
job=job)
self.assertIsNotNone(task)
def test_execute(self):
with mock.patch.object(self.wps, '_WPS__request') as m:
m.return_value = self.execute_data
op = cwt.Process(type('Process', (object,), dict(identifier='CDAT.avg')))
response = self.wps.execute(op)
self.assertIsInstance(op.response, operations.ExecuteResponse)
def test_prepare_data_inputs(self):
proc = cwt.Process(type('Process', (object,), dict(identifier='CDAT.avg')), name='avg')
tas = cwt.Variable('file:///data/tas_6h.nc', 'tas', name='tas1')
d0 = cwt.Domain(name='d0')
data_inputs = self.wps.prepare_data_inputs(proc, [tas], d0)
self.assertEqual(self.data_inputs, data_inputs)
def test_collect_ip_simple(self):
v1 = cwt.Variable('file:///file1', 'tas')
v2 = cwt.Variable('file:///file2', 'tas')
p1 = cwt.Process(identifier='CDAT.aggregate')
p1.add_inputs(v1, v2)
p2 = cwt.Process(identifier='CDAT.subset')
p2.add_inputs(p1)
processes, inputs = p2.collect_input_processes()
self.assertEqual(len(inputs), 2)
self.assertIn(v1.name, inputs)
self.assertIn(v2.name, inputs)
self.assertEqual(len(processes), 2)
self.assertIn(p1.name, processes)
self.assertIn(p2.name, processes)
def test_combine_inputs(self):
inputs = [cwt.Variable('file:///tas.nc', 'tas{}'.format(x)) for x in range(2)]
avg = cwt.Process(type('Process', (object,), dict(identifier='CDAT.avg')))
avg.set_inputs(inputs[0])
with mock.patch.object(self.wps, '_WPS__request') as m:
m.return_value = self.execute_data
self.wps.execute(avg, inputs=[inputs[1]])
self.assertIn('tas0|', m.call_args_list[0][1]['data'])
self.assertIn('tas1|', m.call_args_list[0][1]['data'])
def processes(self, regex=None, refresh=False, method='GET'):
""" Returns a list of WPS processes.
Args:
regex: A string regex used to filter the processes by identifier.
refresh: A boolean to force a GetCapabilites refresh.
method: A string HTTP method.
Returns:
A GetCapabilities object.
"""
if self.__capabilities is None or refresh:
self.__capabilities = self.__get_capabilities(method)
return [cwt.Process(x) for x in self.__capabilities.process_offerings]
def test_workflow(self):
self.backend.populate_processes()
process = models.Process.objects.get(identifier='CDAT.subset')
job = models.Job.objects.create(server=self.server, user=self.user, process=process)
var = cwt.Variable('file:///test.nc', 'tas', name='v0')
variables = {'v0': var}
domain = cwt.Domain([
cwt.Dimension('time', 0, 200)
], name='d0')
domains = {'d0': domain}
proc1 = cwt.Process(identifier='CDAT.aggregate', name='aggregate')
proc1.set_inputs('v0')
proc2 = cwt.Process(identifier='CDAT.subset', name='subset')
proc2.set_inputs('aggregate')
proc3 = cwt.Process(identifier='CDSpark.max', name='max')
proc3.set_inputs('subset')
proc4 = cwt.Process(identifier='Oph.avg', name='avg')
proc4.set_inputs('max')
operations = {'aggregate': proc1, 'subset': proc2, 'max': proc3, 'avg': proc4}
workflow = self.backend.workflow(proc4, variables, domains, operations, user=self.user, job=job)
def edas_submit(self, context):
params = context.operation.parameters
parts = context.operation.identifier.split('.')
operation = cwt.Process(parts[1].replace('-', '.'))
operation.parameters = params
variables = [x.variable for x in context.inputs]
data_inputs = prepare_data_inputs(variables, context.domain, operation)
zmq_context = zmq.Context.instance()
with connect_socket(zmq_context, zmq.PULL, settings.WPS_EDAS_HOST,
settings.WPS_EDAS_RES_PORT) as pull_sock:
self.status('Connected to EDASK pull socket')
with connect_socket(zmq_context, zmq.REQ, settings.WPS_EDAS_HOST,
settings.WPS_EDAS_REQ_PORT) as req_sock:
self.status('Connected to EDASK request socket')
extras = json.dumps({
'storeExecuteResponse': 'true',
'status': 'true',
'responseform': 'xml',
'sendData': 'false',
})
message = '{}!execute!{}!{}!{}'.format(context.job.id,
operation.identifier,
data_inputs, extras)
edas_send(req_sock, req_sock, message)
self.status('Sent {!r} byte request to EDASK', len(message))
output = edas_result(pull_sock)
self.status('Completed with output {!r}', output)
context.output_data = json.dumps(output.parameterize())
return context
def run( self ):
d0 = cwt.Domain([], name="d0")
op1 = cwt.Operation.from_dict( { 'name': "CDSpark.multiAverage" } )
op1.add_input( cwt.Variable("file:///dass/nobackup/tpmaxwel/.edas/cache/collections/NCML/MERRA_TAS1hr.ncml", "tas" ) )
op3 = cwt.Operation.from_dict( { 'name': 'CDSpark.regrid', 'crs':'gaussian~128' } )
op3.add_input( op1 )
op2 = cwt.Operation.from_dict( { 'name': "CDSpark.multiAverage" } )
for i in range(1,3): op2.add_input( cwt.Variable('collection:/GISS-E2-R_r%di1p1'%(i), "tas" ) )
op4 = cwt.Operation.from_dict( { 'name': 'CDSpark.regrid', 'crs':'gaussian~128' } )
op4.add_input( op2 )
op5 = cwt.Operation.from_dict( { 'name': 'CDSpark.multiAverage' } )
op5.add_input( op3 )
op5.add_input( op4 )
wps = cwt.WPS( 'http://localhost:9001/wps', log=True, log_file=os.path.expanduser("~/esgf_api.log") )
wps.init()
process = cwt.Process( wps, op5 )
process.execute( None, d0, [], True, True, "GET" )
def test(value):
proc = process.Process(0)
proc.job = mock.MagicMock()
op = cwt.Process(identifier=identifier)
op.set_inputs(*inputs)
if domain is not None:
op.domain = domain
if param is not None:
op.parameters[param.name] = param
if gridder is not None:
op.parameters['gridder'] = gridder
with cdms2.open(TEST_OUTPUT, 'w') as out_file:
value.value = proc.process(op, num_inputs, out_file, operation)
def test_execute(self):
mock_job = mock.MagicMock()
variables = {
'v0': cwt.Variable('file:///test.nc', 'tas', name='v0'),
}
domains = {'d0': cwt.Domain([cwt.Dimension('time', 0, 200)])}
operation = cwt.Process(identifier='Oph.max', name='max')
operation.domain = 'd0'
result = self.backend.execute('Oph.max',
variables,
domains, {'max': operation},
job=mock_job,
user=self.user)
self.assertIsNotNone(result)
def test_prepare_data_inputs(self):
variable = cwt.Variable('file:///test.nc', 'tas', name='v0')
domain = cwt.Domain([
cwt.Dimension('time', 0, 365),
], name='d0')
process = cwt.Process('CDAT.subset', name='subset')
process.description = mock.MagicMock()
process.description.metadata.return_value = {}
client = cwt.WPSClient('http://idontexist/wps')
data_inputs = client.prepare_data_inputs_str(process, [variable],
domain)
self.assertIn('"id": "tas|v0"', data_inputs)
self.assertIn('"id": "d0"', data_inputs)
self.assertIn('"name": "CDAT.subset"', data_inputs)
def get_process(self, identifier, method='GET'):
""" Return a specified process.
Args:
identifier: A string process identifier.
Returns:
The specified process instance.
Raises:
Exception: A process with identifier was not found.
"""
if self.__capabilities is None:
self.__capabilities = self.__get_capabilities(method)
try:
return [
cwt.Process(x) for x in self.__capabilities.process_offerings
if x.identifier == identifier
][0]
except IndexError:
raise Exception(
'Failed to find process with identifier "{}"'.format(
identifier))
def test_execute(self):
self.backend.populate_processes()
process = models.Process.objects.get(identifier='CDAT.subset')
job = models.Job.objects.create(server=self.server, user=self.user, process=process)
var = cwt.Variable('file:///test.nc', 'tas', name='v0')
variables = {'v0': var}
domain = cwt.Domain([
cwt.Dimension('time', 0, 200)
], name='d0')
domains = {'d0': domain}
proc = cwt.Process(identifier='CDAT.subset', name='subset')
operations = {'subset': proc}
proc = self.backend.execute('CDAT.subset', variables, domains, operations, user=self.user, job=job)
self.assertIsNotNone(proc)