def run(self):
domain_data = {
'id': 'd0',
'lat': {
'start': 70,
'end': 90,
'crs': 'values'
},
'lon': {
'start': 5,
'end': 45,
'crs': 'values'
}
}
d0 = cwt.Domain.from_dict(domain_data)
inputs = cwt.Variable(
"file:///dass/nobackup/tpmaxwel/.edas/cache/collections/NCML/MERRA_TAS1hr.ncml",
"tas",
domain="d0",
axes="xy")
op = cwt.Process.from_dict({'name': "CDSpark.average"})
op.set_inputs(inputs)
wps = cwt.WPS('http://localhost:9000/wps',
log=True,
log_file=os.path.expanduser("~/esgf_api.log"))
wps.execute(op, domain=d0, method="GET")
def setUp(self):
self.wps = cwt.WPS('http://nohost/wps')
test_path = lambda x: os.path.join(os.path.dirname(__file__), x)
with open(test_path('data/get_capabilities_response.txt'), 'r') as infile:
self.capabilities_data = infile.read()
with open(test_path('data/describe_process_cdat_avg_response.txt'), 'r') as infile:
self.describe_data = infile.read()
with open(test_path('data/execute_cdat_avg_response.txt'), 'r') as infile:
self.execute_data = infile.read()
self.data_inputs = '[variable=[{"uri": "file:///data/tas_6h.nc", "id": "tas|tas1"}];domain=[{"id": "d0"}];operation=[{"input": ["tas1"], "domain": "d0", "name": "CDAT.avg", "result": "avg"}]]'
def run( self ):
d0 = cwt.Domain([], name="d0")
op1 = cwt.Operation.from_dict( { 'name': "CDSpark.multiAverage" } )
op1.add_input( cwt.Variable("file:///dass/nobackup/tpmaxwel/.edas/cache/collections/NCML/MERRA_TAS1hr.ncml", "tas" ) )
op3 = cwt.Operation.from_dict( { 'name': 'CDSpark.regrid', 'crs':'gaussian~128' } )
op3.add_input( op1 )
op2 = cwt.Operation.from_dict( { 'name': "CDSpark.multiAverage" } )
for i in range(1,3): op2.add_input( cwt.Variable('collection:/GISS-E2-R_r%di1p1'%(i), "tas" ) )
op4 = cwt.Operation.from_dict( { 'name': 'CDSpark.regrid', 'crs':'gaussian~128' } )
op4.add_input( op2 )
op5 = cwt.Operation.from_dict( { 'name': 'CDSpark.multiAverage' } )
op5.add_input( op3 )
op5.add_input( op4 )
wps = cwt.WPS( 'http://localhost:9001/wps', log=True, log_file=os.path.expanduser("~/esgf_api.log") )
wps.init()
process = cwt.Process( wps, op5 )
process.execute( None, d0, [], True, True, "GET" )
def netcdf_processing(params_file):
# Extract parameters for processing
with open(params_file) as data_file:
plist = json.load(data_file)
# Extract processing function parameters
var_name = plist["function"]["var_name"]
api_key = plist["function"]["api_key"]
out_var_name = plist["function"]["out_var_name"]
slice_mode = plist["function"]["slice_mode"]
if slice_mode == "None": slice_mode = None
in_file = plist["function"]["in_file"]
out_file = plist["function"]["out_file"]
esgf_cwt_wps_url = plist["function"]["esgf_cwt_wps_url"]
try:
geo_domain_latmin = plist["function"]["geo_domain"]["latitude"]["min"]
except KeyError:
geo_domain_latmin = -9999
try:
geo_domain_latmax = plist["function"]["geo_domain"]["latitude"]["max"]
except KeyError:
geo_domain_latmax = -9999
try:
geo_domain_lonmin = plist["function"]["geo_domain"]["longitude"]["min"]
except KeyError:
geo_domain_lonmin = -9999
try:
geo_domain_lonmax = plist["function"]["geo_domain"]["longitude"]["max"]
except KeyError:
geo_domain_lonmax = -9999
if plist["function"]["calc_operation"] == "time_avg":
my_indice_params = {'indice_name': out_var_name,
'calc_operation': 'mean'
}
else:
raise ValueError('Operation specified in calc_operation is not implemented: '+plist["function"]["calc_operation"])
dd_b, mm_b, yyyy_b = map(int, plist["function"]["time_range_b"].split('-'))
dd_e, mm_e, yyyy_e = map(int, plist["function"]["time_range_e"].split('-'))
period = [datetime.datetime(yyyy_b,mm_b,dd_b), datetime.datetime(yyyy_e,mm_e,dd_e)]
if geo_domain_latmin != -9999 and geo_domain_latmax != -9999 and geo_domain_lonmin != -9999 and geo_domain_lonmax != -9999:
# Setup ESGF CWT WPS
wps = cwt.WPS(esgf_cwt_wps_url, api_key=api_key)
# Get variable from in_file
in_file_s = in_file.encode("ascii")
var_name_s = var_name.encode("ascii")
varproc = cwt.Variable(in_file_s, var_name_s)
# Setup domain
d0 = cwt.Domain([
cwt.Dimension('time', 0, 9999999, cwt.INDICES),
cwt.Dimension('lat', geo_domain_latmin, geo_domain_latmax),
cwt.Dimension('lon', geo_domain_lonmin, geo_domain_lonmax),
])
# Select gridder
cwt.gridder.Gridder('esmf', 'linear', varproc)
# Select processor
proc = wps.get_process('CDAT.subset')
# Execute processing
wps.execute(proc, inputs=[varproc], domain=d0)
# Check status and wait for completion
while proc.processing:
print proc.status
time.sleep(1)
# Output some feedback to user
print proc.status
print proc.output.uri
print proc.output.var_name
# Set output information
in_file = proc.output.uri
var_name = proc.output.var_name
# Launch processing using icclim
icclim.indice(user_indice=my_indice_params, in_files=in_file, var_name=var_name, slice_mode=slice_mode, base_period_time_range=period, out_unit='days', out_file=out_file, callback=callback.defaultCallback2)
def edas_submit(self, parent_variables, variables, domains, operation, user_id,
job_id):
self.PUBLISH = base.ALL
req_sock = None
sub_sock = None
edas_output_path = None
proc = process.Process(self.request.id)
proc.initialize(user_id, job_id)
proc.job.started()
v, d, o = self.load(parent_variables, variables, domains, operation)
domain = d.get(o.domain, None)
data_inputs = cwt.WPS('').prepare_data_inputs(o, {}, domain)
logger.info('Generated datainputs: {}'.format(data_inputs))
context = zmq.Context.instance()
poller = None
try:
req_sock = initialize_socket(context, zmq.REQ, settings.EDAS_HOST,
settings.EDAS_REQ_PORT)
sub_sock = initialize_socket(context, zmq.SUB, settings.EDAS_HOST,
settings.EDAS_RES_PORT)
sub_sock.setsockopt(zmq.SUBSCRIBE, b'{}'.format(proc.job.id))
poller = zmq.Poller()
poller.register(sub_sock)
proc.log('Connected to EDAS backend')
extra = '{"response":"file"}'
req_sock.send(
str('{}!execute!{}!{}!{}'.format(proc.job.id, o.identifier,
data_inputs, extra)))
if (req_sock.poll(settings.EDAS_TIMEOUT * 1000) == 0):
raise WPSError('EDAS timed out waiting for accept response')
data = req_sock.recv()
proc.log('Sent request to EDAS backend')
check_exceptions(data)
edas_output_path = listen_edas_output(self, poller, proc)
if edas_output_path is None:
raise WPSError('Failed to receive output from EDAS')
except:
raise
finally:
if req_sock is not None:
req_sock.close()
if sub_sock is not None:
if poller is not None:
poller.unregister(sub_sock)
sub_sock.close()
proc.log('Received result from EDAS backend')
output_name = '{}.nc'.format(str(uuid.uuid4()))
output_path = os.path.join(settings.LOCAL_OUTPUT_PATH, output_name)
shutil.move(edas_output_path, output_path)
proc.log('Localizing output to THREDDS server')
if settings.DAP:
output_url = settings.DAP_URL.format(filename=output_name)
else:
output_url = settings.OUTPUT_URL.format(filename=output_name)
var_name = None
try:
with cdms2.open(output_path) as infile:
var_name = infile.variables.keys()[0]
except:
raise WPSError('Failed to determine variable name of the EDAS output')
proc.log('Variable name from EDAS result "{}"', var_name)
output_var = cwt.Variable(output_url, var_name, name=o.name)
return {o.name: output_var.parameterize()}