def _handler(self, request, response): init_process_logger('log.txt') response.outputs['output_log'].file = 'log.txt' LOGGER.info('Start process') response.update_status('execution started at : %s ' % dt.now(), 10) ################################ # reading in the input arguments ################################ try: response.update_status( 'execution started at : {}'.format(dt.now()), 20) ################################ # reading in the input arguments ################################ LOGGER.info('read in the arguments') resource = archiveextract( resource=[res.file for res in request.inputs['resource']]) # If files are from different datasets. # i.e. files: ...output1/slp.1999.nc and ...output2/slp.1997.nc will not be sorted with just .sort() # So: if type(resource) == list: resource = sorted( resource, key=lambda i: path.splitext(path.basename(i))[0]) else: resource = [resource] season = request.inputs['season'][0].data LOGGER.info('season %s', season) bboxDef = '-80,50,20,70' # in general format bbox = [] bboxStr = request.inputs['BBox'][0].data LOGGER.debug('BBOX selected by user: %s ' % (bboxStr)) bboxStr = bboxStr.split(',') # Checking for wrong cordinates and apply default if nesessary if (abs(float(bboxStr[0])) > 180 or abs(float(bboxStr[1]) > 180) or abs(float(bboxStr[2]) > 90) or abs(float(bboxStr[3])) > 90): bboxStr = bboxDef # request.inputs['BBox'].default # .default doesn't work anymore!!! LOGGER.debug( 'BBOX is out of the range, using default instead: %s ' % (bboxStr)) bboxStr = bboxStr.split(',') bbox.append(float(bboxStr[0])) bbox.append(float(bboxStr[2])) bbox.append(float(bboxStr[1])) bbox.append(float(bboxStr[3])) LOGGER.debug('BBOX for ocgis: %s ' % (bbox)) LOGGER.debug('BBOX original: %s ' % (bboxStr)) period = request.inputs['period'][0].data LOGGER.info('period %s', period) anualcycle = request.inputs['anualcycle'][0].data kappa = request.inputs['kappa'][0].data LOGGER.info('kappa %s', kappa) method = request.inputs['method'][0].data LOGGER.info('Calc annual cycle with %s', method) sseas = request.inputs['sseas'][0].data LOGGER.info('Annual cycle calc with %s', sseas) start = dt.strptime(period.split('-')[0], '%Y%m%d') end = dt.strptime(period.split('-')[1], '%Y%m%d') # OCGIS for models workaround - to catch 31 of Dec start = dt.combine(start, dt_time(12, 0)) end = dt.combine(end, dt_time(12, 0)) cycst = anualcycle.split('-')[0] cycen = anualcycle.split('-')[1] reference = [ dt.strptime(cycst, '%Y%m%d'), dt.strptime(cycen, '%Y%m%d') ] LOGGER.debug('Reference start: %s , end: %s ' % (reference[0], reference[1])) reference[0] = dt.combine(reference[0], dt_time(12, 0)) reference[1] = dt.combine(reference[1], dt_time(12, 0)) LOGGER.debug('New Reference start: %s , end: %s ' % (reference[0], reference[1])) # Check if 360_day calendar (all months are exactly 30 days): try: modcal, calunits = get_calendar(resource[0]) if '360_day' in modcal: if start.day == 31: start = start.replace(day=30) LOGGER.debug('Date has been changed for: %s' % (start)) if end.day == 31: end = end.replace(day=30) LOGGER.debug('Date has been changed for: %s' % (end)) if reference[0].day == 31: reference[0] = reference[0].replace(day=30) LOGGER.debug('Date has been changed for: %s' % (reference[0])) if reference[1].day == 31: reference[1] = reference[1].replace(day=30) LOGGER.debug('Date has been changed for: %s' % (reference[1])) except: LOGGER.debug('Could not detect calendar') LOGGER.debug('start: %s , end: %s ', start, end) LOGGER.info('bbox %s', bbox) LOGGER.info('period %s', period) LOGGER.info('season %s', season) except Exception as e: msg = 'failed to read in the arguments' LOGGER.exception(msg) raise Exception(msg) ############################################################ # get the required bbox and time region from resource data ############################################################ response.update_status('start subsetting', 30) from blackswan.ocgis_module import call from blackswan.utils import get_variable, get_timerange time_range = [start, end] tmp_resource = [] for re in resource: s, e = get_timerange(re) tmpSt = dt.strptime(s, '%Y%m%d') tmpEn = dt.strptime(e, '%Y%m%d') if ((tmpSt <= end) and (tmpEn >= start)): tmp_resource.append(re) LOGGER.debug('Selected file: %s ' % (re)) resource = tmp_resource # Here start trick with z... levels and regriding... # Otherwise call will give memory error for hires models like MIROC4h # TODO: Add level and domain selection as in wps_analogs_model for 4D var. variable = get_variable(resource) from blackswan.datafetch import reanalyses import uuid ref_var = 'slp' refR = 'NCEP' ref_rea = reanalyses(start=2014, end=2014, variable=ref_var, dataset=refR) regr_res = [] for z in resource: tmp_n = 'tmp_%s' % (uuid.uuid1()) # XXXXXXX s, e = get_timerange(z) tmpSt = dt.strptime(s, '%Y%m%d') tmpEn = dt.strptime(e, '%Y%m%d') tmpSt = dt.combine(tmpSt, dt_time(0, 0)) tmpEn = dt.combine(tmpEn, dt_time(23, 0)) if ((tmpSt <= start) and (tmpEn >= end)): LOGGER.debug('Resource contains full record, selecting : %s ' % (time_range)) full_res = call(z, variable=variable, time_range=time_range) else: full_res = z LOGGER.debug( 'The subset from the big model file, or initial file: %s ' % (full_res)) # XXXXXXX # TODO: regrid needs here (???) # Check how to manage one big file with geopotential # TODO: # Adapt to work with levels for geopotential (check how its done for reanalysis process) # b0=call(resource=z, variable=variable, b0 = call(resource=full_res, variable=variable, spatial_wrapping='wrap', cdover='system', regrid_destination=ref_rea[0], regrid_options='bil', prefix=tmp_n) # TODO: Use cdo regrid outside call - before call... # Some issues with produced ocgis file inside ocgis_module cdo regrid: # cdo remapbil (Abort): Unsupported projection coordinates (Variable: psl)! # select domain b01 = call(resource=b0, geom=bbox, spatial_wrapping='wrap', prefix='levregr_' + path.basename(z)[0:-3]) tbr = 'rm -f %s' % (b0) system(tbr) tbr = 'rm -f %s.nc' % (tmp_n) system(tbr) # get full resource regr_res.append(b01) model_subset = call(regr_res, time_range=time_range) # TODO: CHANGE to cross-platform for i in regr_res: tbr = 'rm -f %s' % (i) system(tbr) # Get domain # from cdo import Cdo # from os import environ # cdo = Cdo(env=environ) # regr_res = [] # for res_fn in resource: # tmp_f = 'dom_' + path.basename(res_fn) # comcdo = '%s,%s,%s,%s' % (bbox[0],bbox[2],bbox[1],bbox[3]) # cdo.sellonlatbox(comcdo, input=res_fn, output=tmp_f) # # tmp_f = call(resource=res_fn, geom=bbox, spatial_wrapping='wrap', prefix=tmp_f) # regr_res.append(tmp_f) # LOGGER.debug('File with selected domain: %s ' % (tmp_f)) # model_subset = call( # # resource=resource, variable=variable, # resource=regr_res, variable=variable, # geom=bbox, spatial_wrapping='wrap', time_range=time_range, # conform_units_to=conform_units_to # ) LOGGER.info('Dataset subset done: %s ' % model_subset) response.update_status('dataset subsetted', 40) ##################### # computing anomalies ##################### response.update_status('computing anomalies ', 50) model_anomal = wr.get_anomalies(model_subset, reference=reference, method=method, sseas=sseas) ################### # extracting season #################### model_season = wr.get_season(model_anomal, season=season) response.update_status('values normalized', 60) #################### # call the R scripts #################### response.update_status('Start weather regime clustering ', 70) import shlex import subprocess from blackswan import config from os.path import curdir, exists, join try: # rworkspace = curdir Rsrc = config.Rsrc_dir() Rfile = 'weatherregimes_model.R' infile = model_season # model_subset #model_ponderate # modelname = 'MODEL' # yr1 = start.year # yr2 = end.year ip, output_graphics = mkstemp(dir=curdir, suffix='.pdf') ip, file_pca = mkstemp(dir=curdir, suffix='.txt') ip, file_class = mkstemp(dir=curdir, suffix='.Rdat') args = [ 'Rscript', join(Rsrc, Rfile), '%s/' % curdir, '%s/' % Rsrc, '%s' % infile, '%s' % variable, '%s' % output_graphics, '%s' % file_pca, '%s' % file_class, '%s' % season, '%s' % start.year, '%s' % end.year, '%s' % 'MODEL', '%s' % kappa ] LOGGER.info('Rcall builded') LOGGER.debug('ARGS: %s' % (args)) except Exception as e: msg = 'failed to build the R command %s' % e LOGGER.error(msg) raise Exception(msg) try: output, error = subprocess.Popen( args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() # ,shell=True LOGGER.info('R outlog info:\n %s ' % output) LOGGER.debug('R outlog errors:\n %s ' % error) if len(output) > 0: response.update_status('**** weatherregime in R suceeded', 80) else: LOGGER.error('NO! output returned from R call') except Exception as e: msg = 'weatherregime in R %s ' % e LOGGER.error(msg) raise Exception(msg) response.update_status('Weather regime clustering done ', 90) ############################################ # set the outputs ############################################ # response.update_status('Set the process outputs ', 95) response.outputs['Routput_graphic'].file = output_graphics response.outputs['output_pca'].file = file_pca response.outputs['output_classification'].file = file_class response.outputs['output_netcdf'].file = model_season response.update_status('done', 100) return response
def set_dynamic_md(resource): """ Dynamic meta data like time frequency, spatial extent, start/end time, etc. :param resource: netCDF file where basic meta data should be set """ from blackswan.utils import get_timerange, get_time frequency = get_frequency(resource) time_coverage_start, time_coverage_end = get_timerange(resource) time_number_steps = len(get_time(resource)) # max_lat, min_lat, max_lon, min_lat = get_extent(resource) ds = Dataset(resource, mode='a') try: driving_experiment = ds.driving_experiment ds.delncattr('driving_experiment') except Exception as e: LOGGER.error(e) driving_experiment = '' try: driving_experiment_name = ds.driving_experiment_name ds.delncattr('driving_experiment_name') except Exception as e: LOGGER.error(e) driving_experiment_name = '' try: driving_model_ensemble_member = ds.driving_model_ensemble_member ds.delncattr('driving_model_ensemble_member') except Exception as e: LOGGER.error(e) driving_model_ensemble_member = '' try: experiment = ds.experiment ds.delncattr('experiment') except Exception as e: LOGGER.error(e) experiment = '' try: tracking_id = ds.tracking_id ds.delncattr('tracking_id') except Exception as e: LOGGER.error(e) tracking_id = '' try: experiment_id = ds.experiment_id ds.delncattr('experiment_id') except Exception as e: LOGGER.error(e) experiment_id = '' try: project_id = ds.project_id ds.delncattr('project_id') except Exception as e: LOGGER.error(e) project_id = '' try: institution_id = ds.institution_id ds.delncattr('institution_id') except Exception as e: LOGGER.error(e) institution_id = '' try: model_version_id = ds.model_version_id ds.delncattr('model_version_id') except Exception as e: LOGGER.error(e) model_version_id = '' try: driving_model_id = ds.driving_model_id ds.delncattr('driving_model_id') except Exception as e: LOGGER.error(e) driving_model_id = '' try: driving_ensemble_member = ds.driving_ensemble_member ds.delncattr('driving_ensemble_member') except Exception as e: LOGGER.error(e) driving_ensemble_member = '' try: driving_model_id = ds.driving_model_id ds.delncattr('driving_model_id') except Exception as e: LOGGER.error(e) driving_model_id = '' try: model_id = ds.model_id ds.delncattr('model_id') except Exception as e: LOGGER.error(e) driving_model_id = '' try: contact = ds.contact ds.delncattr('contact') except Exception as e: LOGGER.error(e) contact = '' try: driving_experiment_id = ds.driving_experiment_id ds.delncattr('driving_experiment_id') except Exception as e: LOGGER.error(e) driving_experiment_id = '' try: domain = ds.CORDEX_domain except Exception as e: LOGGER.error(e) domain = '' ds.close() min_lat, max_lat, min_lon, max_lon = get_extent(resource) geospatial_increment = get_geospatial_increment(resource) try: md_dynamic = { 'in_var_driving_experiment': driving_experiment, 'in_var_driving_experiment_name': driving_experiment_name, 'in_var_driving_model_ensemble_member': driving_model_ensemble_member, 'in_var_experiment': experiment, 'in_var_experiment_id': experiment_id, 'in_var_project_id': project_id, 'in_var_contact': contact, 'in_var_institution_id': institution_id, 'in_var_model_version_id': model_version_id, 'in_var_driving_model_id': driving_model_id, 'in_var_model_id': model_id, 'in_var_driving_ensemble_member': driving_ensemble_member, 'in_var_driving_experiment_id': driving_experiment_id, 'in_var_domain': domain, 'in_var_tracking_id': tracking_id, 'frequency': frequency, 'time_coverage_start': time_coverage_start, 'time_coverage_end': time_coverage_end, 'time_number_steps': time_number_steps, #'time_number_gaps': '', #'cdm_datatype':'' , 'domain': '%s_subset' % domain, 'geospatial_increment': geospatial_increment, 'geospatial_lat_min': min_lat, 'geospatial_lat_max': max_lat, 'geospatial_lon_min': min_lon, 'geospatial_lon_max': max_lon, } #:product = "output" ; #:rcm_version_id = "v1" ; #:references = "http://www.smhi.se/en/Research/Research-departments/climate-research-rossby-centre" ; except Exception as e: LOGGER.error('failed to populate dynamic metadata dictionary') try: ds = Dataset(resource, mode='a') ds.setncatts(md_dynamic) ds.close() except Exception as e: LOGGER.error(e) return (resource)
def _handler(self, request, response): init_process_logger('log.txt') response.outputs['output_log'].file = 'log.txt' LOGGER.info('Start process') response.update_status('execution started at : {}'.format(dt.now()), 5) process_start_time = time.time() # measure process execution time ... start_time = time.time() # measure init ... ################################ # reading in the input arguments ################################ # response.update_status('execution started at : %s ' % dt.now(), 5) # start_time = time.time() # measure init ... ################################ # reading in the input arguments ################################ try: response.update_status('read input parameter : %s ' % dt.now(), 10) resource = archiveextract(resource=rename_complexinputs(request.inputs['resource'])) refSt = request.inputs['refSt'][0].data refEn = request.inputs['refEn'][0].data dateSt = request.inputs['dateSt'][0].data dateEn = request.inputs['dateEn'][0].data seasonwin = request.inputs['seasonwin'][0].data nanalog = request.inputs['nanalog'][0].data bboxDef = '-20,40,30,70' # in general format # level = 500 level = request.inputs['level'][0].data if (level == 500): dummylevel = 1000 # dummy workaround for cdo sellevel else: dummylevel = 500 LOGGER.debug('LEVEL selected: %s hPa' % (level)) bbox = [] bboxStr = request.inputs['BBox'][0].data LOGGER.debug('BBOX selected by user: %s ' % (bboxStr)) bboxStr = bboxStr.split(',') # Checking for wrong cordinates and apply default if nesessary if (abs(float(bboxStr[0])) > 180 or abs(float(bboxStr[1]) > 180) or abs(float(bboxStr[2]) > 90) or abs(float(bboxStr[3])) > 90): bboxStr = bboxDef # request.inputs['BBox'].default # .default doesn't work anymore!!! LOGGER.debug('BBOX is out of the range, using default instead: %s ' % (bboxStr)) bboxStr = bboxStr.split(',') # for i in bboxStr: bbox.append(int(i)) bbox.append(float(bboxStr[0])) bbox.append(float(bboxStr[2])) bbox.append(float(bboxStr[1])) bbox.append(float(bboxStr[3])) LOGGER.debug('BBOX for ocgis: %s ' % (bbox)) LOGGER.debug('BBOX original: %s ' % (bboxStr)) normalize = request.inputs['normalize'][0].data plot = request.inputs['plot'][0].data distance = request.inputs['dist'][0].data outformat = request.inputs['outformat'][0].data timewin = request.inputs['timewin'][0].data detrend = request.inputs['detrend'][0].data LOGGER.info('input parameters set') response.update_status('Read in and convert the arguments', 20) except Exception as e: msg = 'failed to read input prameter %s ' % e LOGGER.error(msg) raise Exception(msg) ###################################### # convert types and set environment ###################################### try: # not nesessary if fix ocgis_module.py refSt = dt.combine(refSt, dt_time(12, 0)) refEn = dt.combine(refEn, dt_time(12, 0)) dateSt = dt.combine(dateSt, dt_time(12, 0)) dateEn = dt.combine(dateEn, dt_time(12, 0)) # Check if 360_day calendar: try: if type(resource) is not list: resource = [resource] modcal, calunits = get_calendar(resource[0]) if '360_day' in modcal: if refSt.day == 31: refSt = refSt.replace(day=30) LOGGER.debug('Date has been changed for: %s' % (refSt)) if refEn.day == 31: refEn = refEn.replace(day=30) LOGGER.debug('Date has been changed for: %s' % (refEn)) if dateSt.day == 31: dateSt = dateSt.replace(day=30) LOGGER.debug('Date has been changed for: %s' % (dateSt)) if dateEn.day == 31: dateEn = dateEn.replace(day=30) LOGGER.debug('Date has been changed for: %s' % (dateEn)) except: LOGGER.debug('Could not detect calendar') if normalize == 'None': seacyc = False else: seacyc = True if outformat == 'ascii': outformat = '.txt' elif outformat == 'netCDF': outformat = '.nc' else: LOGGER.error('output format not valid') start = min(refSt, dateSt) end = max(refEn, dateEn) LOGGER.info('environment set') except Exception as e: msg = 'failed to set environment %s ' % e LOGGER.error(msg) raise Exception(msg) LOGGER.debug("init took %s seconds.", time.time() - start_time) response.update_status('Read in and convert the arguments', 30) ######################## # input data preperation ######################## # TODO: Check if files containing more than one dataset response.update_status('Start preparing input data', 40) start_time = time.time() # mesure data preperation ... try: # TODO: Add selection of the level. maybe bellow in call(..., level_range=[...,...]) if type(resource) == list: # resource.sort() resource = sorted(resource, key=lambda i: path.splitext(path.basename(i))[0]) else: resource = [resource] # =============================================================== # REMOVE resources which are out of interest from the list # (years > and < than requested for calculation) tmp_resource = [] for re in resource: s,e = get_timerange(re) tmpSt = dt.strptime(s, '%Y%m%d') tmpEn = dt.strptime(e, '%Y%m%d') if ((tmpSt <= end) and (tmpEn >= start)): tmp_resource.append(re) LOGGER.debug('Selected file: %s ' % (re)) resource = tmp_resource # Try to fix memory issue... (ocgis call for files like 20-30 gb... ) # IF 4D - select pressure level before domain cut # # resource properties ds = Dataset(resource[0]) variable = get_variable(resource[0]) var = ds.variables[variable] dims = list(var.dimensions) dimlen = len(dims) try: model_id = ds.getncattr('model_id') except AttributeError: model_id = 'Unknown model' LOGGER.debug('MODEL: %s ' % (model_id)) lev_units = 'hPa' if (dimlen > 3): lev = ds.variables[dims[1]] # actually index [1] need to be detected... assuming zg(time, plev, lat, lon) lev_units = lev.units if (lev_units == 'Pa'): level = level * 100 dummylevel = dummylevel * 100 # TODO: OR check the NAME and units of vertical level and find 200 , 300, or 500 mbar in it # Not just level = level * 100. # Get Levels from cdo import Cdo cdo = Cdo(env=environ) lev_res = [] if(dimlen > 3): for res_fn in resource: tmp_f = 'lev_' + path.basename(res_fn) try: tmp_f = call(resource=res_fn, variable=variable, spatial_wrapping='wrap', level_range=[int(level), int(level)], prefix=tmp_f[0:-3]) except: comcdo = '%s,%s' % (level, dummylevel) cdo.sellevel(comcdo, input=res_fn, output=tmp_f) lev_res.append(tmp_f) else: lev_res = resource # =============================================================== # TODO: Before domain, Regrid to selected grid! (???) if no rean. # ================================================================ # Get domain regr_res = [] for res_fn in lev_res: tmp_f = 'dom_' + path.basename(res_fn) comcdo = '%s,%s,%s,%s' % (bbox[0], bbox[2], bbox[1], bbox[3]) try: tmp_f = call(resource=res_fn, geom=bbox, spatial_wrapping='wrap', prefix=tmp_f[0:-3]) except: cdo.sellonlatbox(comcdo, input=res_fn, output=tmp_f) regr_res.append(tmp_f) # ============================ # Block to Detrend data # TODO 1 Keep trend as separate file # TODO 2 Think how to add options to plot abomalies AND original data... # May be do archive and simulation = call.. over NOT detrended data and keep it as well if (dimlen > 3): res_tmp = get_level(regr_res, level=level) variable = 'z%s' % level else: res_tmp = call(resource=regr_res, spatial_wrapping='wrap') if detrend == 'None': orig_model_subset = res_tmp else: orig_model_subset = remove_mean_trend(res_tmp, varname=variable) # ============================ # archive_tmp = call(resource=regr_res, time_range=[refSt, refEn], spatial_wrapping='wrap') # simulation_tmp = call(resource=regr_res, time_range=[dateSt, dateEn], spatial_wrapping='wrap') ################################ # Prepare names for config.txt # ################################ # refDatesString = dt.strftime(refSt, '%Y-%m-%d') + "_" + dt.strftime(refEn, '%Y-%m-%d') # simDatesString = dt.strftime(dateSt, '%Y-%m-%d') + "_" + dt.strftime(dateEn, '%Y-%m-%d') # Fix < 1900 issue... refDatesString = refSt.isoformat().strip().split("T")[0] + "_" + refEn.isoformat().strip().split("T")[0] simDatesString = dateSt.isoformat().strip().split("T")[0] + "_" + dateEn.isoformat().strip().split("T")[0] archiveNameString = "base_" + variable + "_" + refDatesString + '_%.1f_%.1f_%.1f_%.1f' \ % (bbox[0], bbox[2], bbox[1], bbox[3]) simNameString = "sim_" + variable + "_" + simDatesString + '_%.1f_%.1f_%.1f_%.1f' \ % (bbox[0], bbox[2], bbox[1], bbox[3]) archive = call(resource=res_tmp, time_range=[refSt, refEn], spatial_wrapping='wrap', prefix=archiveNameString) simulation = call(resource=res_tmp, time_range=[dateSt, dateEn], spatial_wrapping='wrap', prefix=simNameString) ####################################################################################### if seacyc is True: seasoncyc_base, seasoncyc_sim = analogs.seacyc(archive, simulation, method=normalize) else: seasoncyc_base = None seasoncyc_sim = None except Exception as e: msg = 'failed to prepare archive and simulation files %s ' % e LOGGER.debug(msg) raise Exception(msg) ip, output = mkstemp(dir='.', suffix='.txt') output_file = path.abspath(output) files = [path.abspath(archive), path.abspath(simulation), output_file] LOGGER.debug("data preperation took %s seconds.", time.time() - start_time) ############################ # generating the config file ############################ # TODO: add MODEL name as argument response.update_status('writing config file', 50) start_time = time.time() # measure write config ... try: config_file = analogs.get_configfile( files=files, seasoncyc_base=seasoncyc_base, seasoncyc_sim=seasoncyc_sim, base_id=model_id, sim_id=model_id, timewin=timewin, varname=variable, seacyc=seacyc, cycsmooth=91, nanalog=nanalog, seasonwin=seasonwin, distfun=distance, outformat=outformat, calccor=True, silent=False, # period=[dt.strftime(refSt, '%Y-%m-%d'), dt.strftime(refEn, '%Y-%m-%d')], period=[refSt.isoformat().strip().split("T")[0], refEn.isoformat().strip().split("T")[0]], bbox="%s,%s,%s,%s" % (bbox[0], bbox[2], bbox[1], bbox[3])) except Exception as e: msg = 'failed to generate config file %s ' % e LOGGER.debug(msg) raise Exception(msg) LOGGER.debug("write_config took %s seconds.", time.time() - start_time) ############## # CASTf90 call ############## import subprocess import shlex start_time = time.time() # measure call castf90 response.update_status('Start CASTf90 call', 60) # ----------------------- try: import ctypes # TODO: This lib is for linux mkl_rt = ctypes.CDLL('libmkl_rt.so') nth = mkl_rt.mkl_get_max_threads() LOGGER.debug('Current number of threads: %s' % (nth)) mkl_rt.mkl_set_num_threads(ctypes.byref(ctypes.c_int(64))) nth = mkl_rt.mkl_get_max_threads() LOGGER.debug('NEW number of threads: %s' % (nth)) # TODO: Does it \/\/\/ work with default shell=False in subprocess... (?) environ['MKL_NUM_THREADS'] = str(nth) environ['OMP_NUM_THREADS'] = str(nth) except Exception as e: msg = 'Failed to set THREADS %s ' % e LOGGER.debug(msg) # ----------------------- # ##### TEMPORAL WORKAROUND! With instaled hdf5-1.8.18 in anaconda ############### # ##### MUST be removed after castf90 recompiled with the latest hdf version # ##### NOT safe environ['HDF5_DISABLE_VERSION_CHECK'] = '1' # hdflib = os.path.expanduser("~") + '/anaconda/lib' # hdflib = os.getenv("HOME") + '/anaconda/lib' import pwd hdflib = pwd.getpwuid(getuid()).pw_dir + '/anaconda/lib' environ['LD_LIBRARY_PATH'] = hdflib # ################################################################################ try: # response.update_status('execution of CASTf90', 50) cmd = 'analogue.out %s' % path.relpath(config_file) # system(cmd) args = shlex.split(cmd) output, error = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() LOGGER.info('analogue.out info:\n %s ' % output) LOGGER.debug('analogue.out errors:\n %s ' % error) response.update_status('**** CASTf90 suceeded', 70) except Exception as e: msg = 'CASTf90 failed %s ' % e LOGGER.error(msg) raise Exception(msg) LOGGER.debug("castf90 took %s seconds.", time.time() - start_time) # TODO: Add try - except for pdfs if plot == 'Yes': analogs_pdf = analogs.plot_analogs(configfile=config_file) else: analogs_pdf = 'dummy_plot.pdf' with open(analogs_pdf, 'a'): utime(analogs_pdf, None) response.update_status('preparing output', 80) response.outputs['analog_pdf'].file = analogs_pdf response.outputs['config'].file = config_file response.outputs['analogs'].file = output_file response.outputs['output_netcdf'].file = simulation response.outputs['target_netcdf'].file = archive if seacyc is True: response.outputs['base_netcdf'].file = seasoncyc_base response.outputs['sim_netcdf'].file = seasoncyc_sim else: # TODO: Still unclear how to overpass unknown number of outputs dummy_base = 'dummy_base.nc' dummy_sim = 'dummy_sim.nc' with open(dummy_base, 'a'): utime(dummy_base, None) with open(dummy_sim, 'a'): utime(dummy_sim, None) response.outputs['base_netcdf'].file = dummy_base response.outputs['sim_netcdf'].file = dummy_sim ######################## # generate analog viewer ######################## formated_analogs_file = analogs.reformat_analogs(output_file) # response.outputs['formated_analogs'].storage = FileStorage() response.outputs['formated_analogs'].file = formated_analogs_file LOGGER.info('analogs reformated') response.update_status('reformatted analog file', 90) viewer_html = analogs.render_viewer( # configfile=response.outputs['config'].get_url(), configfile=config_file, # datafile=response.outputs['formated_analogs'].get_url()) datafile=formated_analogs_file) response.outputs['output'].file = viewer_html response.update_status('Successfully generated analogs viewer', 95) LOGGER.info('rendered pages: %s ', viewer_html) response.update_status('execution ended', 100) LOGGER.debug("total execution took %s seconds.", time.time() - process_start_time) return response
def _handler(self, request, response): chdir(self.workdir) init_process_logger('log.txt') process_start_time = time.time() # measure process execution time ... response.update_status('execution started at : %s ' % dt.now(), 5) start_time = time.time() # measure init ... resource = archiveextract( resource=rename_complexinputs(request.inputs['resource'])) # Filter resource: if type(resource) == list: resource = sorted(resource, key=lambda i: path.splitext(path.basename(i))[0]) else: resource = [resource] refSt = request.inputs['refSt'][0].data refEn = request.inputs['refEn'][0].data dateSt = request.inputs['dateSt'][0].data dateEn = request.inputs['dateEn'][0].data regrset = request.inputs['regrset'][0].data direction = request.inputs['direction'][0].data # Check if model has 360_day calendar: try: modcal, calunits = get_calendar(resource[0]) LOGGER.debug('CALENDAR: %s' % (modcal)) if '360_day' in modcal: if direction == 're2mo': if refSt.day == 31: refSt = refSt.replace(day=30) LOGGER.debug('Date has been changed for: %s' % (refSt)) if refEn.day == 31: refEn = refEn.replace(day=30) LOGGER.debug('Date has been changed for: %s' % (refEn)) else: # mo2re if dateSt.day == 31: dateSt = dateSt.replace(day=30) LOGGER.debug('Date has been changed for: %s' % (dateSt)) if dateEn.day == 31: dateEn = dateEn.replace(day=30) LOGGER.debug('Date has been changed for: %s' % (dateEn)) except: LOGGER.debug('Could not detect calendar') seasonwin = request.inputs['seasonwin'][0].data nanalog = request.inputs['nanalog'][0].data bboxDef = '-20,40,30,70' # in general format bbox = [] bboxStr = request.inputs['BBox'][0].data LOGGER.debug('BBOX selected by user: %s ' % (bboxStr)) bboxStr = bboxStr.split(',') # Checking for wrong cordinates and apply default if nesessary if (abs(float(bboxStr[0])) > 180 or abs(float(bboxStr[1]) > 180) or abs(float(bboxStr[2]) > 90) or abs(float(bboxStr[3])) > 90): bboxStr = bboxDef # request.inputs['BBox'].default # .default doesn't work anymore!!! LOGGER.debug( 'BBOX is out of the range, using default instead: %s ' % (bboxStr)) bboxStr = bboxStr.split(',') bbox.append(float(bboxStr[0])) bbox.append(float(bboxStr[2])) bbox.append(float(bboxStr[1])) bbox.append(float(bboxStr[3])) normalize = request.inputs['normalize'][0].data plot = request.inputs['plot'][0].data distance = request.inputs['dist'][0].data outformat = request.inputs['outformat'][0].data timewin = request.inputs['timewin'][0].data model_var = request.inputs['reanalyses'][0].data model, var = model_var.split('_') try: if direction == 're2mo': anaSt = dt.combine(dateSt, dt_time( 0, 0)) # dt.strptime(dateSt[0], '%Y-%m-%d') anaEn = dt.combine(dateEn, dt_time( 0, 0)) # dt.strptime(dateEn[0], '%Y-%m-%d') refSt = dt.combine(refSt, dt_time( 12, 0)) # dt.strptime(refSt[0], '%Y-%m-%d') refEn = dt.combine(refEn, dt_time( 12, 0)) # dt.strptime(refEn[0], '%Y-%m-%d') r_time_range = [anaSt, anaEn] m_time_range = [refSt, refEn] elif direction == 'mo2re': anaSt = dt.combine(dateSt, dt_time( 12, 0)) # dt.strptime(refSt[0], '%Y-%m-%d') anaEn = dt.combine(dateEn, dt_time( 12, 0)) # dt.strptime(refEn[0], '%Y-%m-%d') refSt = dt.combine(refSt, dt_time( 0, 0)) # dt.strptime(dateSt[0], '%Y-%m-%d') refEn = dt.combine(refEn, dt_time( 0, 0)) # dt.strptime(dateEn[0], '%Y-%m-%d') r_time_range = [refSt, refEn] m_time_range = [anaSt, anaEn] else: LOGGER.exception( 'failed to find time periods for comparison direction') except: msg = 'failed to put simulation and reference time in order' LOGGER.exception(msg) raise Exception(msg) if normalize == 'None': seacyc = False else: seacyc = True if outformat == 'ascii': outformat = '.txt' elif outformat == 'netCDF': outformat = '.nc' else: LOGGER.exception('output format not valid') try: if model == 'NCEP': # getlevel = True getlevel = False if 'z' in var: level = var.strip('z') variable = 'hgt' # conform_units_to='hPa' else: variable = 'slp' level = None # conform_units_to='hPa' elif '20CRV2' in model: getlevel = False if 'z' in var: variable = 'hgt' level = var.strip('z') # conform_units_to=None else: variable = 'prmsl' level = None # conform_units_to='hPa' else: LOGGER.exception('Reanalyses model not known') LOGGER.info('environment set') except: msg = 'failed to set environment' LOGGER.exception(msg) raise Exception(msg) # LOGGER.exception("init took %s seconds.", time.time() - start_time) response.update_status('Read in the arguments', 10) ################# # get input data ################# # TODO: do not forget to select years start_time = time.time() # measure get_input_data ... response.update_status('fetching input data', 20) try: if direction == 're2mo': nc_reanalyses = reanalyses(start=anaSt.year, end=anaEn.year, variable=var, dataset=model, getlevel=getlevel) else: nc_reanalyses = reanalyses(start=refSt.year, end=refEn.year, variable=var, dataset=model, getlevel=getlevel) if type(nc_reanalyses) == list: nc_reanalyses = sorted( nc_reanalyses, key=lambda i: path.splitext(path.basename(i))[0]) else: nc_reanalyses = [nc_reanalyses] # For 20CRV2 geopotential height, daily dataset for 100 years is about 50 Gb # So it makes sense, to operate it step-by-step # TODO: need to create dictionary for such datasets (for models as well) # TODO: benchmark the method bellow for NCEP z500 for 60 years, may be use the same (!) # TODO Now everything regrid to the reanalysis # if ('20CRV2' in model) and ('z' in var): if ('z' in var): tmp_total = [] origvar = get_variable(nc_reanalyses[0]) for z in nc_reanalyses: # tmp_n = 'tmp_%s' % (uuid.uuid1()) b0 = call(resource=z, variable=origvar, level_range=[int(level), int(level)], geom=bbox, spatial_wrapping='wrap', prefix='levdom_' + path.basename(z)[0:-3]) tmp_total.append(b0) tmp_total = sorted( tmp_total, key=lambda i: path.splitext(path.basename(i))[0]) inter_subset_tmp = call(resource=tmp_total, variable=origvar, time_range=r_time_range) # Create new variable ds = Dataset(inter_subset_tmp, mode='a') z_var = ds.variables.pop(origvar) dims = z_var.dimensions new_var = ds.createVariable('z%s' % level, z_var.dtype, dimensions=(dims[0], dims[2], dims[3])) new_var[:, :, :] = squeeze(z_var[:, 0, :, :]) # new_var.setncatts({k: z_var.getncattr(k) for k in z_var.ncattrs()}) ds.close() nc_subset = call(inter_subset_tmp, variable='z%s' % level) # Clean for i in tmp_total: tbr = 'rm -f %s' % (i) system(tbr) # for i in inter_subset_tmp tbr = 'rm -f %s' % (inter_subset_tmp) system(tbr) else: # TODO: ADD HERE serial as well as in analogs reanalysis process!! nc_subset = call( resource=nc_reanalyses, variable=var, geom=bbox, spatial_wrapping='wrap', time_range=r_time_range, ) response.update_status('**** Input reanalyses data fetched', 30) except: msg = 'failed to fetch or subset input files' LOGGER.exception(msg) raise Exception(msg) ######################## # input data preperation ######################## response.update_status('Start preparing input data', 40) m_start = m_time_range[0] m_end = m_time_range[1] # =============================================================== # REMOVE resources from the list which are out of interest from the list # (years > and < than requested for calculation) tmp_resource = [] for re in resource: s, e = get_timerange(re) tmpSt = dt.strptime(s, '%Y%m%d') tmpEn = dt.strptime(e, '%Y%m%d') if ((tmpSt <= m_end) and (tmpEn >= m_start)): tmp_resource.append(re) LOGGER.debug('Selected file: %s ' % (re)) resource = tmp_resource start_time = time.time() # mesure data preperation ... # TODO: Check the callendars ! for model vs reanalyses. # TODO: Check the units! model vs reanalyses. try: m_total = [] modvar = get_variable(resource) # resource properties ds = Dataset(resource[0]) m_var = ds.variables[modvar] dims = list(m_var.dimensions) dimlen = len(dims) try: model_id = ds.getncattr('model_id') except AttributeError: model_id = 'Unknown model' LOGGER.debug('MODEL: %s ' % (model_id)) lev_units = 'hPa' if (dimlen > 3): lev = ds.variables[dims[1]] # TODO: actually index [1] need to be detected... assuming zg(time, plev, lat, lon) lev_units = lev.units if (lev_units == 'Pa'): m_level = str(int(level) * 100) else: m_level = level else: m_level = None if level == None: level_range = None else: level_range = [int(m_level), int(m_level)] ds.close() for z in resource: tmp_n = 'tmp_%s' % (uuid.uuid1()) # TODO: Important! if only 1 file - select time period from that first! # select level and regrid # \/\/ working version 19Feb2019 # b0 = call(resource=z, variable=modvar, level_range=level_range, # spatial_wrapping='wrap', cdover='system', # regrid_destination=nc_reanalyses[0], regrid_options='bil', prefix=tmp_n) try: b0 = call(resource=z, variable=modvar, level_range=level_range, spatial_wrapping='wrap', cdover='system', regrid_destination=nc_subset, regrid_options='bil', prefix=tmp_n) except: b0 = call(resource=z, variable=modvar, level_range=level_range, spatial_wrapping='wrap', cdover='system', regrid_destination=nc_reanalyses[0], regrid_options='bil', prefix=tmp_n) # select domain (already selected in fact, if regrided to 'nc_subset') b01 = call(resource=b0, geom=bbox, spatial_wrapping='wrap', prefix='levregr_' + path.basename(z)[0:-3]) # TODO: REPLACE rm -f by os.remove() ! tbr = 'rm -f %s' % (b0) system(tbr) tbr = 'rm -f %s.nc' % (tmp_n) system(tbr) # get full resource m_total.append(b01) model_subset = call(m_total, time_range=m_time_range) for i in m_total: tbr = 'rm -f %s' % (i) system(tbr) if m_level is not None: # Create new variable in model set ds = Dataset(model_subset, mode='a') mod_var = ds.variables.pop(modvar) dims = mod_var.dimensions new_modvar = ds.createVariable('z%s' % level, mod_var.dtype, dimensions=(dims[0], dims[2], dims[3])) new_modvar[:, :, :] = squeeze(mod_var[:, 0, :, :]) # new_var.setncatts({k: z_var.getncattr(k) for k in z_var.ncattrs()}) ds.close() mod_subset = call(model_subset, variable='z%s' % level) else: mod_subset = model_subset except: msg = 'failed to subset simulation or reference data' LOGGER.exception(msg) raise Exception(msg) # -------------------------------------------- try: if direction == 'mo2re': simulation = mod_subset archive = nc_subset base_id = model sim_id = model_id elif direction == 're2mo': simulation = nc_subset archive = mod_subset base_id = model_id sim_id = model else: LOGGER.exception('direction not valid: %s ' % direction) except: msg = 'failed to find comparison direction' LOGGER.exception(msg) raise Exception(msg) try: if level is not None: out_var = 'z%s' % level else: var_archive = get_variable(archive) var_simulation = get_variable(simulation) if var_archive != var_simulation: rename_variable(archive, oldname=var_archive, newname=var_simulation) out_var = var_simulation LOGGER.info('varname %s in netCDF renamed to %s' % (var_archive, var_simulation)) except: msg = 'failed to rename variable in target files' LOGGER.exception(msg) raise Exception(msg) try: if seacyc is True: seasoncyc_base, seasoncyc_sim = analogs.seacyc( archive, simulation, method=normalize) else: seasoncyc_base = None seasoncyc_sim = None except: msg = 'failed to prepare seasonal cycle reference files' LOGGER.exception(msg) raise Exception(msg) # ip, output = mkstemp(dir='.', suffix='.txt') # output_file = path.abspath(output) output_file = 'output.txt' ################################ # Prepare names for config.txt # ################################ # refDatesString = dt.strftime(refSt, '%Y-%m-%d') + "_" + dt.strftime(refEn, '%Y-%m-%d') # simDatesString = dt.strftime(dateSt, '%Y-%m-%d') + "_" + dt.strftime(dateEn, '%Y-%m-%d') # Fix < 1900 issue... refDatesString = refSt.isoformat().strip().split( "T")[0] + "_" + refEn.isoformat().strip().split("T")[0] simDatesString = dateSt.isoformat().strip().split( "T")[0] + "_" + dateEn.isoformat().strip().split("T")[0] archiveNameString = "base_" + out_var + "_" + refDatesString + '_%.1f_%.1f_%.1f_%.1f' \ % (bbox[0], bbox[2], bbox[1], bbox[3]) + '.nc' simNameString = "sim_" + out_var + "_" + simDatesString + '_%.1f_%.1f_%.1f_%.1f' \ % (bbox[0], bbox[2], bbox[1], bbox[3]) + '.nc' move(archive, archiveNameString) move(simulation, simNameString) archive = archiveNameString simulation = simNameString files = [path.abspath(archive), path.abspath(simulation), output_file] ############################ # generating the config file ############################ response.update_status('writing config file', 50) start_time = time.time() # measure write config ... try: config_file = analogs.get_configfile( files=files, seasoncyc_base=seasoncyc_base, seasoncyc_sim=seasoncyc_sim, base_id=base_id, sim_id=sim_id, timewin=timewin, # varname=var, varname=out_var, seacyc=seacyc, cycsmooth=91, nanalog=nanalog, seasonwin=seasonwin, distfun=distance, outformat=outformat, calccor=True, silent=False, # period=[dt.strftime(refSt, '%Y-%m-%d'), dt.strftime(refEn, '%Y-%m-%d')], period=[ refSt.isoformat().strip().split("T")[0], refEn.isoformat().strip().split("T")[0] ], bbox="%s,%s,%s,%s" % (bbox[0], bbox[2], bbox[1], bbox[3])) except: msg = 'failed to generate config file' LOGGER.exception(msg) raise Exception(msg) ####################### # CASTf90 call ####################### import subprocess import shlex start_time = time.time() # measure call castf90 response.update_status('Start CASTf90 call', 60) # ----------------------- try: import ctypes # TODO: This lib is for linux mkl_rt = ctypes.CDLL('libmkl_rt.so') nth = mkl_rt.mkl_get_max_threads() LOGGER.debug('Current number of threads: %s' % (nth)) mkl_rt.mkl_set_num_threads(ctypes.byref(ctypes.c_int(64))) nth = mkl_rt.mkl_get_max_threads() LOGGER.debug('NEW number of threads: %s' % (nth)) # TODO: Does it \/\/\/ work with default shell=False in subprocess... (?) environ['MKL_NUM_THREADS'] = str(nth) environ['OMP_NUM_THREADS'] = str(nth) except Exception as e: msg = 'Failed to set THREADS %s ' % e LOGGER.debug(msg) # ----------------------- # ##### TEMPORAL WORKAROUND! With instaled hdf5-1.8.18 in anaconda ############### # ##### MUST be removed after castf90 recompiled with the latest hdf version # ##### NOT safe environ['HDF5_DISABLE_VERSION_CHECK'] = '1' # hdflib = os.path.expanduser("~") + '/anaconda/lib' # hdflib = os.getenv("HOME") + '/anaconda/lib' import pwd hdflib = pwd.getpwuid(getuid()).pw_dir + '/anaconda/lib' environ['LD_LIBRARY_PATH'] = hdflib # ################################################################################ try: response.update_status('execution of CASTf90', 70) cmd = 'analogue.out %s' % path.relpath(config_file) # system(cmd) args = shlex.split(cmd) output, error = subprocess.Popen( args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() LOGGER.info('analogue.out info:\n %s ' % output) LOGGER.exception('analogue.out errors:\n %s ' % error) response.update_status('**** CASTf90 suceeded', 80) except: msg = 'CASTf90 failed' LOGGER.exception(msg) raise Exception(msg) LOGGER.debug("castf90 took %s seconds.", time.time() - start_time) # TODO: Add try - except for pdfs if plot == 'Yes': analogs_pdf = analogs.plot_analogs(configfile=config_file) else: analogs_pdf = 'dummy_plot.pdf' with open(analogs_pdf, 'a'): utime(analogs_pdf, None) response.update_status('preparting output', 90) # Stopper to keep twitcher results, for debug # dummy=dummy response.outputs['analog_pdf'].file = analogs_pdf response.outputs['config'].file = config_file response.outputs['analogs'].file = output_file response.outputs['output_netcdf'].file = simulation response.outputs['target_netcdf'].file = archive ######################## # generate analog viewer ######################## formated_analogs_file = analogs.reformat_analogs(output_file) response.outputs['formated_analogs'].file = formated_analogs_file LOGGER.info('analogs reformated') # response.update_status('reformatted analog file', 95) viewer_html = analogs.render_viewer( # configfile=response.outputs['config'].get_url(), configfile=config_file, # datafile=response.outputs['formated_analogs'].get_url()) datafile=formated_analogs_file) response.outputs['output'].file = viewer_html response.update_status('Successfully generated analogs viewer', 95) LOGGER.info('rendered pages: %s ', viewer_html) response.update_status('execution ended', 100) LOGGER.debug("total execution took %s seconds.", time.time() - process_start_time) response.outputs['output_log'].file = 'log.txt' return response
def _handler(self, request, response): init_process_logger('log.txt') response.outputs['output_log'].file = 'log.txt' LOGGER.info('Start process') response.update_status('execution started at : {}'.format(dt.now()), 5) process_start_time = time.time() # measure process execution time ... start_time = time.time() # measure init ... ################################ # reading in the input arguments ################################ # response.update_status('execution started at : %s ' % dt.now(), 5) # start_time = time.time() # measure init ... ################################ # reading in the input arguments ################################ try: response.update_status('read input parameter : %s ' % dt.now(), 10) resource = archiveextract( resource=rename_complexinputs(request.inputs['resource'])) dateSt = request.inputs['dateSt'][0].data dateEn = request.inputs['dateEn'][0].data bboxDef = '-20,40,30,70' # in general format # level = 500 season = request.inputs['season'][0].data level = request.inputs['level'][0].data if (level == 500): dummylevel = 1000 # dummy workaround for cdo sellevel else: dummylevel = 500 LOGGER.debug('LEVEL selected: %s hPa' % (level)) bbox = [] bboxStr = request.inputs['BBox'][0].data LOGGER.debug('BBOX selected by user: %s ' % (bboxStr)) bboxStr = bboxStr.split(',') # Checking for wrong cordinates and apply default if nesessary if (abs(float(bboxStr[0])) > 180 or abs(float(bboxStr[1]) > 180) or abs(float(bboxStr[2]) > 90) or abs(float(bboxStr[3])) > 90): bboxStr = bboxDef # request.inputs['BBox'].default # .default doesn't work anymore!!! LOGGER.debug( 'BBOX is out of the range, using default instead: %s ' % (bboxStr)) bboxStr = bboxStr.split(',') # for i in bboxStr: bbox.append(int(i)) bbox.append(float(bboxStr[0])) bbox.append(float(bboxStr[2])) bbox.append(float(bboxStr[1])) bbox.append(float(bboxStr[3])) LOGGER.debug('BBOX for ocgis: %s ' % (bbox)) LOGGER.debug('BBOX original: %s ' % (bboxStr)) distance = request.inputs['dist'][0].data method = request.inputs['method'][0].data LOGGER.info('input parameters set') response.update_status('Read in and convert the arguments', 20) except Exception as e: msg = 'failed to read input prameter %s ' % e LOGGER.error(msg) raise Exception(msg) ###################################### # convert types and set environment ###################################### try: # not nesessary if fix ocgis_module.py dateSt = dt.combine(dateSt, dt_time(12, 0)) dateEn = dt.combine(dateEn, dt_time(12, 0)) # Check if 360_day calendar: try: if type(resource) is not list: resource = [resource] modcal, calunits = get_calendar(resource[0]) if '360_day' in modcal: if dateSt.day == 31: dateSt = dateSt.replace(day=30) LOGGER.debug('Date has been changed for: %s' % (dateSt)) if dateEn.day == 31: dateEn = dateEn.replace(day=30) LOGGER.debug('Date has been changed for: %s' % (dateEn)) except: LOGGER.debug('Could not detect calendar') start = dateSt end = dateEn time_range = [start, end] LOGGER.info('environment set') except Exception as e: msg = 'failed to set environment %s ' % e LOGGER.error(msg) raise Exception(msg) LOGGER.debug("init took %s seconds.", time.time() - start_time) response.update_status('Read in and convert the arguments', 30) ######################## # input data preperation ######################## # TODO: Check if files containing more than one dataset response.update_status('Start preparing input data', 40) start_time = time.time() # mesure data preperation ... try: # TODO: Add selection of the level. maybe bellow in call(..., level_range=[...,...]) if type(resource) == list: # resource.sort() resource = sorted( resource, key=lambda i: path.splitext(path.basename(i))[0]) else: resource = [resource] # =============================================================== # REMOVE resources which are out of interest from the list # (years > and < than requested for calculation) tmp_resource = [] for re in resource: s, e = get_timerange(re) tmpSt = dt.strptime(s, '%Y%m%d') tmpEn = dt.strptime(e, '%Y%m%d') if ((tmpSt <= end) and (tmpEn >= start)): tmp_resource.append(re) LOGGER.debug('Selected file: %s ' % (re)) resource = tmp_resource # Try to fix memory issue... (ocgis call for files like 20-30 gb... ) # IF 4D - select pressure level before domain cut # # resource properties ds = Dataset(resource[0]) variable = get_variable(resource[0]) var = ds.variables[variable] dims = list(var.dimensions) dimlen = len(dims) try: model_id = ds.getncattr('model_id') except AttributeError: model_id = 'Unknown_model' LOGGER.debug('MODEL: %s ' % (model_id)) lev_units = 'hPa' if (dimlen > 3): lev = ds.variables[dims[1]] # actually index [1] need to be detected... assuming zg(time, plev, lat, lon) lev_units = lev.units if (lev_units == 'Pa'): level = level * 100 dummylevel = dummylevel * 100 # TODO: OR check the NAME and units of vertical level and find 200 , 300, or 500 mbar in it # Not just level = level * 100. # Get Levels from cdo import Cdo cdo = Cdo(env=environ) lev_res = [] if (dimlen > 3): for res_fn in resource: tmp_f = 'lev_' + path.basename(res_fn) try: tmp_f = call(resource=res_fn, variable=variable, spatial_wrapping='wrap', level_range=[int(level), int(level)], prefix=tmp_f[0:-3]) except: comcdo = '%s,%s' % (level, dummylevel) cdo.sellevel(comcdo, input=res_fn, output=tmp_f) lev_res.append(tmp_f) else: lev_res = resource # =============================================================== # TODO: Before domain, Regrid to selected grid! (???) if no rean. # ================================================================ # Get domain regr_res = [] for res_fn in lev_res: tmp_f = 'dom_' + path.basename(res_fn) comcdo = '%s,%s,%s,%s' % (bbox[0], bbox[2], bbox[1], bbox[3]) try: tmp_f = call(resource=res_fn, geom=bbox, spatial_wrapping='wrap', prefix=tmp_f[0:-3]) except: cdo.sellonlatbox(comcdo, input=res_fn, output=tmp_f) regr_res.append(tmp_f) # ============================ # Block to collect final data if (dimlen > 3): res_tmp_tmp = get_level(regr_res, level=level) variable = 'z%s' % level res_tmp = call(resource=res_tmp_tmp, variable=variable, time_range=time_range) else: res_tmp = call(resource=regr_res, time_range=time_range, spatial_wrapping='wrap') ####################################################################################### except Exception as e: msg = 'failed to prepare archive and simulation files %s ' % e LOGGER.debug(msg) raise Exception(msg) LOGGER.debug("data preperation took %s seconds.", time.time() - start_time) # ----------------------- # try: # import ctypes # # TODO: This lib is for linux # mkl_rt = ctypes.CDLL('libmkl_rt.so') # nth = mkl_rt.mkl_get_max_threads() # LOGGER.debug('Current number of threads: %s' % (nth)) # mkl_rt.mkl_set_num_threads(ctypes.byref(ctypes.c_int(64))) # nth = mkl_rt.mkl_get_max_threads() # LOGGER.debug('NEW number of threads: %s' % (nth)) # # TODO: Does it \/\/\/ work with default shell=False in subprocess... (?) # environ['MKL_NUM_THREADS'] = str(nth) # environ['OMP_NUM_THREADS'] = str(nth) # except Exception as e: # msg = 'Failed to set THREADS %s ' % e # LOGGER.debug(msg) # ----------------------- response.update_status('Start DIM calc', 50) # Calculation of Local Dimentsions ================== LOGGER.debug('Calculation of the distances using: %s metric' % (distance)) LOGGER.debug('Calculation of the dims with: %s' % (method)) dim_filename = '%s.txt' % model_id tmp_dim_fn = '%s.txt' % uuid.uuid1() Rsrc = config.Rsrc_dir() if (method == 'Python'): try: l_dist, l_theta = localdims(resource=res_tmp, variable=variable, distance=str(distance)) response.update_status('**** Dims with Python suceeded', 60) except: LOGGER.exception('NO! output returned from Python call') if (method == 'Python_wrap'): try: l_dist, l_theta = localdims_par(resource=res_tmp, variable=variable, distance=str(distance)) response.update_status('**** Dims with Python suceeded', 60) except: LOGGER.exception('NO! output returned from Python call') if (method == 'R'): # from os.path import join Rfile = 'localdimension_persistence_fullD.R' args = [ 'Rscript', path.join(Rsrc, Rfile), '%s' % res_tmp, '%s' % variable, '%s' % tmp_dim_fn ] LOGGER.info('Rcall builded') LOGGER.debug('ARGS: %s' % (args)) try: output, error = subprocess.Popen( args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() LOGGER.info('R outlog info:\n %s ' % output) LOGGER.exception('R outlog errors:\n %s ' % error) if len(output) > 0: response.update_status('**** Dims with R suceeded', 60) else: LOGGER.exception('NO! output returned from R call') # HERE READ DATA FROM TEXT FILES R_resdim = loadtxt(fname=tmp_dim_fn, delimiter=',') l_theta = R_resdim[:, 0] l_dist = R_resdim[:, 1] except: msg = 'Dim with R' LOGGER.exception(msg) raise Exception(msg) if (method == 'R_wrap'): # from os.path import join Rfile = 'localdimension_persistence_serrD.R' args = [ 'Rscript', path.join(Rsrc, Rfile), '%s' % res_tmp, '%s' % variable, '%s' % tmp_dim_fn ] LOGGER.info('Rcall builded') LOGGER.debug('ARGS: %s' % (args)) try: output, error = subprocess.Popen( args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() LOGGER.info('R outlog info:\n %s ' % output) LOGGER.exception('R outlog errors:\n %s ' % error) if len(output) > 0: response.update_status('**** Dims with R_wrap suceeded', 60) else: LOGGER.exception('NO! output returned from R call') # HERE READ DATA FROM TEXT FILES R_resdim = loadtxt(fname=tmp_dim_fn, delimiter=',') l_theta = R_resdim[:, 0] l_dist = R_resdim[:, 1] except: msg = 'Dim with R_wrap' LOGGER.exception(msg) raise Exception(msg) try: res_times = get_time(res_tmp) except: LOGGER.debug('Not standard calendar') res_times = analogs.get_time_nc(res_tmp) # plot 1 ld_pdf = analogs.pdf_from_ld(x=l_dist, y=l_theta) # res_times = [ res_times[i].isoformat().strip().split("T")[0].replace('-', '') for i in range(len(res_times)) ] # concatenation of values concat_vals = column_stack([res_times, l_theta, l_dist]) savetxt(dim_filename, concat_vals, fmt='%s', delimiter=',') # output season try: seas = _TIMEREGIONS_[season]['month'] # [12, 1, 2] LOGGER.info('Season to grep from TIMEREGIONS: %s ' % season) LOGGER.info('Season N to grep from TIMEREGIONS: %s ' % seas) except: LOGGER.info('No months in TIMEREGIONS, moving to months') try: seas = _MONTHS_[season]['month'] # [1] or [2] or ... LOGGER.info('Season to grep from MONTHS: %s ' % season) LOGGER.info('Season N to grep from MONTHS: %s ' % seas) except: seas = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] ind = [] # TODO: change concat_vals[i][0][4:6] to dt_obj.month !!! for i in range(len(res_times)): if (int(concat_vals[i][0][4:6]) in seas[:]): ind.append(i) sf = column_stack([concat_vals[i] for i in ind]).T seas_dim_filename = season + '_' + dim_filename savetxt(seas_dim_filename, sf, fmt='%s', delimiter=',') # -------------------------- plot with R --------------- R_plot_file = 'plot_csv.R' ld2_pdf = 'local_dims.pdf' ld2_seas_pdf = season + '_local_dims.pdf' args = [ 'Rscript', path.join(Rsrc, R_plot_file), '%s' % dim_filename, '%s' % ld2_pdf ] try: output, error = subprocess.Popen( args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() LOGGER.info('R outlog info:\n %s ' % output) LOGGER.exception('R outlog errors:\n %s ' % error) except: msg = 'Could not produce plot' LOGGER.exception(msg) # TODO: Here need produce empty pdf to pass to output args = [ 'Rscript', path.join(Rsrc, R_plot_file), '%s' % seas_dim_filename, '%s' % ld2_seas_pdf ] try: output, error = subprocess.Popen( args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() LOGGER.info('R outlog info:\n %s ' % output) LOGGER.exception('R outlog errors:\n %s ' % error) except: msg = 'Could not produce plot' LOGGER.exception(msg) # TODO: Here need produce empty pdf(s) to pass to output # # ==================================================== response.update_status('preparing output', 80) response.outputs['ldist'].file = dim_filename response.outputs['ldist_seas'].file = seas_dim_filename response.outputs['ld_pdf'].file = ld_pdf response.outputs['ld2_pdf'].file = ld2_pdf response.outputs['ld2_seas_pdf'].file = ld2_seas_pdf response.update_status('execution ended', 100) LOGGER.debug("total execution took %s seconds.", time.time() - process_start_time) return response