def _handler(self, request, response): # LOGGER.debug('CURDIR XXXX : %s ' % (abspath(curdir))) # LOGGER.debug('WORKDIR XXXX : %s ' % (self.workdir)) os.chdir(self.workdir) # LOGGER.debug('CURDIR XXXX : %s ' % (abspath(curdir))) init_process_logger('log.txt') # init_process_logger(os.path.join(self.workdir, 'log.txt')) # response.outputs['output_log'].file = 'log.txt' LOGGER.info('Start process') response.update_status('execution started at : {}'.format(dt.now()), 5) process_start_time = time.time() # measure process execution time ... start_time = time.time() # measure init ... ################################ # reading in the input arguments ################################ try: response.update_status('read input parameter : %s ' % dt.now(), 6) refSt = request.inputs['refSt'][0].data refEn = request.inputs['refEn'][0].data dateSt = request.inputs['dateSt'][0].data dateEn = request.inputs['dateEn'][0].data seasonwin = request.inputs['seasonwin'][0].data nanalog = request.inputs['nanalog'][0].data timres = request.inputs['timeres'][0].data bboxDef = '-20,40,30,70' # in general format bbox = [] bboxStr = request.inputs['BBox'][0].data LOGGER.debug('BBOX selected by user: %s ' % (bboxStr)) bboxStr = bboxStr.split(',') # Checking for wrong cordinates and apply default if nesessary if (abs(float(bboxStr[0])) > 180 or abs(float(bboxStr[1]) > 180) or abs(float(bboxStr[2]) > 90) or abs(float(bboxStr[3])) > 90): bboxStr = bboxDef # request.inputs['BBox'].default # .default doesn't work anymore!!! LOGGER.debug('BBOX is out of the range, using default instead: %s ' % (bboxStr)) bboxStr = bboxStr.split(',') bbox.append(float(bboxStr[0])) bbox.append(float(bboxStr[2])) bbox.append(float(bboxStr[1])) bbox.append(float(bboxStr[3])) LOGGER.debug('BBOX for ocgis: %s ' % (bbox)) LOGGER.debug('BBOX original: %s ' % (bboxStr)) normalize = request.inputs['normalize'][0].data detrend = request.inputs['detrend'][0].data plot = request.inputs['plot'][0].data distance = request.inputs['dist'][0].data outformat = request.inputs['outformat'][0].data timewin = request.inputs['timewin'][0].data model_var = request.inputs['reanalyses'][0].data model, var = model_var.split('_') LOGGER.info('input parameters set') response.update_status('Read in and convert the arguments', 7) except Exception as e: msg = 'failed to read input prameter %s ' % e LOGGER.exception(msg) raise Exception(msg) ###################################### # convert types and set environment ###################################### try: response.update_status('Preparing enviroment converting arguments', 8) LOGGER.debug('date: %s %s %s %s ' % (type(refSt), refEn, dateSt, dateSt)) start = min(refSt, dateSt) end = max(refEn, dateEn) if normalize == 'None': seacyc = False else: seacyc = True if outformat == 'ascii': outformat = '.txt' elif outformat == 'netCDF': outformat = '.nc' else: LOGGER.exception('output format not valid') except Exception as e: msg = 'failed to set environment %s ' % e LOGGER.exception(msg) raise Exception(msg) ########################### # set the environment ########################### response.update_status('fetching data from archive', 9) try: if model == 'NCEP': getlevel = False if 'z' in var: level = var.strip('z') # conform_units_to = None else: level = None if var == 'precip': var = 'pr_wtr' # conform_units_to = 'hPa' elif '20CRV2' in model: getlevel = False if 'z' in var: level = var.strip('z') # conform_units_to = None else: level = None # conform_units_to = 'hPa' else: LOGGER.exception('Reanalyses dataset not known') LOGGER.info('environment set for model: %s' % model) except Exception: msg = 'failed to set environment' LOGGER.exception(msg) raise Exception(msg) ########################################## # fetch Data from original data archive ########################################## # NOTE: If ref is say 1950 - 1990, and sim is just 1 week in 2017: # - ALL the data will be downloaded, 1950 - 2017 try: model_nc = rl(start=start.year, end=end.year, dataset=model, variable=var, timres=timres, getlevel=getlevel) LOGGER.info('reanalyses data fetched') except Exception: msg = 'failed to get reanalyses data' LOGGER.exception(msg) raise Exception(msg) response.update_status('subsetting region of interest', 10) LOGGER.debug("start and end time: %s - %s" % (start, end)) time_range = [start, end] # Checking memory and dataset size model_size = get_files_size(model_nc) memory_avail = psutil.virtual_memory().available thrs = 0.3 # 30% if (model_size >= thrs * memory_avail): ser_r = True else: ser_r = False # ################################ # For 20CRV2 geopotential height, daily dataset for 100 years is about 50 Gb # So it makes sense, to operate it step-by-step # TODO: need to create dictionary for such datasets (for models as well) # TODO: benchmark the method bellow for NCEP z500 for 60 years # if ('20CRV2' in model) and ('z' in var): if ('z' in var): tmp_total = [] origvar = get_variable(model_nc) for z in model_nc: # tmp_n = 'tmp_%s' % (uuid.uuid1()) b0 = call(resource=z, variable=origvar, level_range=[int(level), int(level)], geom=bbox, spatial_wrapping='wrap', prefix='levdom_' + os.path.basename(z)[0:-3]) tmp_total.append(b0) tmp_total = sorted(tmp_total, key=lambda i: os.path.splitext(os.path.basename(i))[0]) inter_subset_tmp = call(resource=tmp_total, variable=origvar, time_range=time_range) # Clean for i in tmp_total: tbr = 'rm -f %s' % (i) os.system(tbr) # Create new variable ds = Dataset(inter_subset_tmp, mode='a') z_var = ds.variables.pop(origvar) dims = z_var.dimensions new_var = ds.createVariable('z%s' % level, z_var.dtype, dimensions=(dims[0], dims[2], dims[3])) new_var[:, :, :] = squeeze(z_var[:, 0, :, :]) # new_var.setncatts({k: z_var.getncattr(k) for k in z_var.ncattrs()}) ds.close() model_subset_tmp = call(inter_subset_tmp, variable='z%s' % level) else: if ser_r: LOGGER.debug('Process reanalysis step-by-step') tmp_total = [] for z in model_nc: # tmp_n = 'tmp_%s' % (uuid.uuid1()) b0 = call(resource=z, variable=var, geom=bbox, spatial_wrapping='wrap', prefix='Rdom_' + os.path.basename(z)[0:-3]) tmp_total.append(b0) tmp_total = sorted(tmp_total, key=lambda i: os.path.splitext(os.path.basename(i))[0]) model_subset_tmp = call(resource=tmp_total, variable=var, time_range=time_range) else: LOGGER.debug('Using whole dataset at once') model_subset_tmp = call(resource=model_nc, variable=var, geom=bbox, spatial_wrapping='wrap', time_range=time_range, ) # If dataset is 20CRV2 the 6 hourly file should be converted to daily. # Option to use previously 6h data from cache (if any) and not download daily files. # Disabled for now # if '20CRV2' in model: # if timres == '6h': # from cdo import Cdo # cdo = Cdo(env=os.environ) # model_subset = '%s.nc' % uuid.uuid1() # tmp_f = '%s.nc' % uuid.uuid1() # cdo_op = getattr(cdo, 'daymean') # cdo_op(input=model_subset_tmp, output=tmp_f) # sti = '00:00:00' # cdo_op = getattr(cdo, 'settime') # cdo_op(sti, input=tmp_f, output=model_subset) # LOGGER.debug('File Converted from: %s to daily' % (timres)) # else: # model_subset = model_subset_tmp # else: # model_subset = model_subset_tmp # Remove \/\/\/ if work with 6h data... model_subset = model_subset_tmp LOGGER.info('Dataset subset done: %s ', model_subset) response.update_status('dataset subsetted', 15) # BLOCK OF DETRENDING of model_subset ! # Original model subset kept to further visualisaion if needed # Now is issue with SLP: # TODO 1 Keep trend as separate file # TODO 2 Think how to add options to plot abomalies AND original data... # May be do archive and simulation = call.. over NOT detrended data and keep it as well # TODO 3 Check with faster smoother add removing trend of each grid if detrend == 'None': orig_model_subset = model_subset else: orig_model_subset = remove_mean_trend(model_subset, varname=var) # ====================================== LOGGER.debug("get_input_subset_dataset took %s seconds.", time.time() - start_time) response.update_status('**** Input data fetched', 20) ######################## # input data preperation ######################## response.update_status('Start preparing input data', 30) start_time = time.time() # measure data preperation ... try: # Construct descriptive filenames for the three files # listed in config file # refDatesString = dt.strftime(refSt, '%Y-%m-%d') + "_" + dt.strftime(refEn, '%Y-%m-%d') # simDatesString = dt.strftime(dateSt, '%Y-%m-%d') + "_" + dt.strftime(dateEn, '%Y-%m-%d') # Fix < 1900 issue... refDatesString = refSt.isoformat().strip().split("T")[0] + "_" + refEn.isoformat().strip().split("T")[0] simDatesString = dateSt.isoformat().strip().split("T")[0] + "_" + dateEn.isoformat().strip().split("T")[0] archiveNameString = "base_" + var + "_" + refDatesString + '_%.1f_%.1f_%.1f_%.1f' \ % (bbox[0], bbox[2], bbox[1], bbox[3]) simNameString = "sim_" + var + "_" + simDatesString + '_%.1f_%.1f_%.1f_%.1f' \ % (bbox[0], bbox[2], bbox[1], bbox[3]) archive = call(resource=model_subset, time_range=[refSt, refEn], prefix=archiveNameString) simulation = call(resource=model_subset, time_range=[dateSt, dateEn], prefix=simNameString) LOGGER.info('archive and simulation files generated: %s, %s' % (archive, simulation)) except Exception as e: msg = 'failed to prepare archive and simulation files %s ' % e LOGGER.exception(msg) raise Exception(msg) try: if seacyc is True: LOGGER.info('normalization function with method: %s ' % normalize) seasoncyc_base, seasoncyc_sim = analogs.seacyc( archive, simulation, method=normalize) else: seasoncyc_base = seasoncyc_sim = None except Exception as e: msg = 'failed to generate normalization files %s ' % e LOGGER.exception(msg) raise Exception(msg) output_file = 'output.txt' files = [os.path.abspath(archive), os.path.abspath(simulation), output_file] LOGGER.debug("Data preperation took %s seconds.", time.time() - start_time) ############################ # generate the config file ############################ config_file = analogs.get_configfile( files=files, seasoncyc_base=seasoncyc_base, seasoncyc_sim=seasoncyc_sim, base_id=model, sim_id=model, timewin=timewin, varname=var, seacyc=seacyc, cycsmooth=91, nanalog=nanalog, seasonwin=seasonwin, distfun=distance, outformat=outformat, calccor=True, silent=False, # period=[dt.strftime(refSt, '%Y-%m-%d'), dt.strftime(refEn, '%Y-%m-%d')], period=[refSt.isoformat().strip().split("T")[0], refEn.isoformat().strip().split("T")[0]], bbox="{0[0]},{0[2]},{0[1]},{0[3]}".format(bbox)) response.update_status('generated config file', 40) ####################### # CASTf90 call ####################### start_time = time.time() # measure call castf90 # ----------------------- try: import ctypes # TODO: This lib is for linux mkl_rt = ctypes.CDLL('libmkl_rt.so') nth = mkl_rt.mkl_get_max_threads() LOGGER.debug('Current number of threads: %s' % (nth)) mkl_rt.mkl_set_num_threads(ctypes.byref(ctypes.c_int(64))) nth = mkl_rt.mkl_get_max_threads() LOGGER.debug('NEW number of threads: %s' % (nth)) # TODO: Does it \/\/\/ work with default shell=False in subprocess... (?) os.environ['MKL_NUM_THREADS'] = str(nth) os.environ['OMP_NUM_THREADS'] = str(nth) except Exception as e: msg = 'Failed to set THREADS %s ' % e LOGGER.debug(msg) # ----------------------- # ##### TEMPORAL WORKAROUND! With instaled hdf5-1.8.18 in anaconda ############### # ##### MUST be removed after castf90 recompiled with the latest hdf version # ##### NOT safe os.environ['HDF5_DISABLE_VERSION_CHECK'] = '1' # hdflib = os.path.expanduser("~") + '/anaconda/lib' # hdflib = os.getenv("HOME") + '/anaconda/lib' import pwd hdflib = pwd.getpwuid(os.getuid()).pw_dir + '/anaconda/lib' os.environ['LD_LIBRARY_PATH'] = hdflib # ################################################################################ response.update_status('Start CASTf90 call', 50) try: # response.update_status('execution of CASTf90', 50) cmd = ['analogue.out', config_file] LOGGER.debug("castf90 command: %s", cmd) output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) LOGGER.info('analogue output:\n %s', output) response.update_status('**** CASTf90 suceeded', 60) except CalledProcessError as e: msg = 'CASTf90 failed:\n{0}'.format(e.output) LOGGER.exception(msg) raise Exception(msg) LOGGER.debug("castf90 took %s seconds.", time.time() - start_time) # TODO: Add try - except for pdfs if plot == 'Yes': analogs_pdf = analogs.plot_analogs(configfile=config_file) else: analogs_pdf = 'dummy_plot.pdf' with open(analogs_pdf, 'a'): os.utime(analogs_pdf, None) response.update_status('preparing output', 70) response.outputs['analog_pdf'].file = analogs_pdf response.outputs['config'].file = config_file response.outputs['analogs'].file = output_file response.outputs['output_netcdf'].file = simulation response.outputs['target_netcdf'].file = archive if seacyc is True: response.outputs['base_netcdf'].file = seasoncyc_base response.outputs['sim_netcdf'].file = seasoncyc_sim else: # TODO: Still unclear how to overpass unknown number of outputs dummy_base = 'dummy_base.nc' dummy_sim = 'dummy_sim.nc' with open(dummy_base, 'a'): os.utime(dummy_base, None) with open(dummy_sim, 'a'): os.utime(dummy_sim, None) response.outputs['base_netcdf'].file = dummy_base response.outputs['sim_netcdf'].file = dummy_sim ######################## # generate analog viewer ######################## formated_analogs_file = analogs.reformat_analogs(output_file) # response.outputs['formated_analogs'].storage = FileStorage() response.outputs['formated_analogs'].file = formated_analogs_file LOGGER.info('analogs reformated') response.update_status('reformatted analog file', 80) viewer_html = analogs.render_viewer( # configfile=response.outputs['config'].get_url(), configfile=config_file, # datafile=response.outputs['formated_analogs'].get_url()) datafile=formated_analogs_file) response.outputs['output'].file = viewer_html response.update_status('Successfully generated analogs viewer', 90) LOGGER.info('rendered pages: %s ', viewer_html) response.update_status('execution ended', 100) LOGGER.debug("total execution took %s seconds.", time.time() - process_start_time) response.outputs['output_log'].file = 'log.txt' return response
def _handler(self, request, response): init_process_logger('log.txt') response.outputs['output_log'].file = 'log.txt' LOGGER.info('Start process') response.update_status('execution started at : {}'.format(dt.now()), 5) process_start_time = time.time() # measure process execution time ... start_time = time.time() # measure init ... ################################ # reading in the input arguments ################################ try: response.update_status('read input parameter : %s ' % dt.now(), 6) dateSt = request.inputs['dateSt'][0].data dateEn = request.inputs['dateEn'][0].data # timres = request.inputs['timeres'][0].data timres = 'day' season = request.inputs['season'][0].data bboxDef = '-20,40,30,70' # in general format bbox = [] bboxStr = request.inputs['BBox'][0].data LOGGER.debug('BBOX selected by user: %s ' % (bboxStr)) bboxStr = bboxStr.split(',') # Checking for wrong cordinates and apply default if nesessary if (abs(float(bboxStr[0])) > 180 or abs(float(bboxStr[1]) > 180) or abs(float(bboxStr[2]) > 90) or abs(float(bboxStr[3])) > 90): bboxStr = bboxDef # request.inputs['BBox'].default # .default doesn't work anymore!!! LOGGER.debug('BBOX is out of the range, using default instead: %s ' % (bboxStr)) bboxStr = bboxStr.split(',') bbox.append(float(bboxStr[0])) bbox.append(float(bboxStr[2])) bbox.append(float(bboxStr[1])) bbox.append(float(bboxStr[3])) LOGGER.debug('BBOX for ocgis: %s ' % (bbox)) LOGGER.debug('BBOX original: %s ' % (bboxStr)) distance = request.inputs['dist'][0].data method = request.inputs['method'][0].data model_var = request.inputs['reanalyses'][0].data model, var = model_var.split('_') LOGGER.info('input parameters set') response.update_status('Read in and convert the arguments', 7) except Exception as e: msg = 'failed to read input prameter %s ' % e LOGGER.exception(msg) raise Exception(msg) ###################################### # convert types and set environment ###################################### start = dateSt end = dateEn ########################### # set the environment ########################### response.update_status('fetching data from archive', 9) try: if model == 'NCEP': getlevel = False if 'z' in var: level = var.strip('z') # conform_units_to = None else: level = None if var == 'precip': var = 'pr_wtr' # conform_units_to = 'hPa' elif '20CRV2' in model: getlevel = False if 'z' in var: level = var.strip('z') # conform_units_to = None else: level = None # conform_units_to = 'hPa' else: LOGGER.exception('Reanalyses dataset not known') LOGGER.info('environment set for model: %s' % model) except Exception: msg = 'failed to set environment' LOGGER.exception(msg) raise Exception(msg) ########################################## # fetch Data from original data archive ########################################## # NOTE: If ref is say 1950 - 1990, and sim is just 1 week in 2017: # - ALL the data will be downloaded, 1950 - 2017 try: model_nc = rl(start=start.year, end=end.year, dataset=model, variable=var, timres=timres, getlevel=getlevel) LOGGER.info('reanalyses data fetched') except Exception: msg = 'failed to get reanalyses data' LOGGER.exception(msg) raise Exception(msg) response.update_status('subsetting region of interest', 10) # from flyingpigeon.weatherregimes import get_level LOGGER.debug("start and end time: %s - %s" % (start, end)) time_range = [start, end] # Checking memory and dataset size model_size = get_files_size(model_nc) memory_avail = psutil.virtual_memory().available thrs = 0.5 # 50% if (model_size >= thrs * memory_avail): ser_r = True else: ser_r = False # ################################ # For 20CRV2 geopotential height, daily dataset for 100 years is about 50 Gb # So it makes sense, to operate it step-by-step # TODO: need to create dictionary for such datasets (for models as well) # TODO: benchmark the method bellow for NCEP z500 for 60 years # if ('20CRV2' in model) and ('z' in var): if ('z' in var): tmp_total = [] origvar = get_variable(model_nc) for z in model_nc: # tmp_n = 'tmp_%s' % (uuid.uuid1()) b0 = call(resource=z, variable=origvar, level_range=[int(level), int(level)], geom=bbox, spatial_wrapping='wrap', prefix='levdom_' + os.path.basename(z)[0:-3]) tmp_total.append(b0) tmp_total = sorted(tmp_total, key=lambda i: os.path.splitext(os.path.basename(i))[0]) inter_subset_tmp = call(resource=tmp_total, variable=origvar, time_range=time_range) # Clean for i in tmp_total: tbr = 'rm -f %s' % (i) os.system(tbr) # Create new variable ds = Dataset(inter_subset_tmp, mode='a') z_var = ds.variables.pop(origvar) dims = z_var.dimensions new_var = ds.createVariable('z%s' % level, z_var.dtype, dimensions=(dims[0], dims[2], dims[3])) new_var[:, :, :] = squeeze(z_var[:, 0, :, :]) # new_var.setncatts({k: z_var.getncattr(k) for k in z_var.ncattrs()}) ds.close() model_subset_tmp = call(inter_subset_tmp, variable='z%s' % level) else: if ser_r: LOGGER.debug('Process reanalysis step-by-step') tmp_total = [] for z in model_nc: # tmp_n = 'tmp_%s' % (uuid.uuid1()) b0 = call(resource=z, variable=var, geom=bbox, spatial_wrapping='wrap', prefix='Rdom_' + os.path.basename(z)[0:-3]) tmp_total.append(b0) tmp_total = sorted(tmp_total, key=lambda i: os.path.splitext(os.path.basename(i))[0]) model_subset_tmp = call(resource=tmp_total, variable=var, time_range=time_range) else: LOGGER.debug('Using whole dataset at once') model_subset_tmp = call(resource=model_nc, variable=var, geom=bbox, spatial_wrapping='wrap', time_range=time_range, ) # If dataset is 20CRV2 the 6 hourly file should be converted to daily. # Option to use previously 6h data from cache (if any) and not download daily files. if '20CRV2' in model: if timres == '6h': from cdo import Cdo cdo = Cdo(env=os.environ) model_subset = '%s.nc' % uuid.uuid1() tmp_f = '%s.nc' % uuid.uuid1() cdo_op = getattr(cdo, 'daymean') cdo_op(input=model_subset_tmp, output=tmp_f) sti = '00:00:00' cdo_op = getattr(cdo, 'settime') cdo_op(sti, input=tmp_f, output=model_subset) LOGGER.debug('File Converted from: %s to daily' % (timres)) else: model_subset = model_subset_tmp else: model_subset = model_subset_tmp LOGGER.info('Dataset subset done: %s ', model_subset) response.update_status('dataset subsetted', 15) # ====================================== LOGGER.debug("get_input_subset_dataset took %s seconds.", time.time() - start_time) response.update_status('**** Input data fetched', 20) ######################## # input data preperation ######################## response.update_status('Start preparing input data', 30) start_time = time.time() # measure data preperation ... # ----------------------- # try: # import ctypes # # TODO: This lib is for linux # mkl_rt = ctypes.CDLL('libmkl_rt.so') # nth = mkl_rt.mkl_get_max_threads() # LOGGER.debug('Current number of threads: %s' % (nth)) # mkl_rt.mkl_set_num_threads(ctypes.byref(ctypes.c_int(64))) # nth = mkl_rt.mkl_get_max_threads() # LOGGER.debug('NEW number of threads: %s' % (nth)) # # TODO: Does it \/\/\/ work with default shell=False in subprocess... (?) # os.environ['MKL_NUM_THREADS'] = str(nth) # os.environ['OMP_NUM_THREADS'] = str(nth) # except Exception as e: # msg = 'Failed to set THREADS %s ' % e # LOGGER.debug(msg) # ----------------------- response.update_status('Start DIM calc', 50) # Calculation of Local Dimentsions ================== LOGGER.debug('Calculation of the distances using: %s metric' % (distance)) LOGGER.debug('Calculation of the dims with: %s' % (method)) dim_filename = '%s.txt' % model tmp_dim_fn = '%s.txt' % uuid.uuid1() Rsrc = config.Rsrc_dir() if (method == 'Python'): try: l_dist, l_theta = localdims(resource=model_subset, variable=var, distance=str(distance)) response.update_status('**** Dims with Python suceeded', 60) except: LOGGER.exception('NO! output returned from Python call') if (method == 'Python_wrap'): try: l_dist, l_theta = localdims_par(resource=model_subset, variable=var, distance=str(distance)) response.update_status('**** Dims with Python suceeded', 60) except: LOGGER.exception('NO! output returned from Python call') if (method == 'R'): # from os.path import join Rfile = 'localdimension_persistence_fullD.R' args = ['Rscript', os.path.join(Rsrc, Rfile), '%s' % model_subset, '%s' % var, '%s' % tmp_dim_fn] LOGGER.info('Rcall builded') LOGGER.debug('ARGS: %s' % (args)) try: output, error = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() LOGGER.info('R outlog info:\n %s ' % output) LOGGER.exception('R outlog errors:\n %s ' % error) if len(output) > 0: response.update_status('**** Dims with R suceeded', 60) else: LOGGER.exception('NO! output returned from R call') # HERE READ DATA FROM TEXT FILES R_resdim = loadtxt(fname=tmp_dim_fn, delimiter=',') l_theta = R_resdim[:, 0] l_dist = R_resdim[:, 1] except: msg = 'Dim with R' LOGGER.exception(msg) raise Exception(msg) if (method == 'R_wrap'): # from os.path import join Rfile = 'localdimension_persistence_serrD.R' args = ['Rscript', os.path.join(Rsrc, Rfile), '%s' % model_subset, '%s' % var, '%s' % tmp_dim_fn] LOGGER.info('Rcall builded') LOGGER.debug('ARGS: %s' % (args)) try: output, error = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() LOGGER.info('R outlog info:\n %s ' % output) LOGGER.exception('R outlog errors:\n %s ' % error) if len(output) > 0: response.update_status('**** Dims with R_wrap suceeded', 60) else: LOGGER.exception('NO! output returned from R call') # HERE READ DATA FROM TEXT FILES R_resdim = loadtxt(fname=tmp_dim_fn, delimiter=',') l_theta = R_resdim[:, 0] l_dist = R_resdim[:, 1] except: msg = 'Dim with R_wrap' LOGGER.exception(msg) raise Exception(msg) try: res_times = get_time(model_subset) except: LOGGER.debug('Not standard calendar') res_times = analogs.get_time_nc(model_subset) # plot 1 ld_pdf = analogs.pdf_from_ld(x=l_dist, y=l_theta) # res_times=[res_times[i].isoformat().strip().split("T")[0].replace('-','') for i in range(len(res_times))] # concatenation of values concat_vals = column_stack([res_times, l_theta, l_dist]) savetxt(dim_filename, concat_vals, fmt='%s', delimiter=',') # output season try: seas = _TIMEREGIONS_[season]['month'] # [12, 1, 2] LOGGER.info('Season to grep from TIMEREGIONS: %s ' % season) LOGGER.info('Season N to grep from TIMEREGIONS: %s ' % seas) except: LOGGER.info('No months in TIMEREGIONS, moving to months') try: seas = _MONTHS_[season]['month'] # [1] or [2] or ... LOGGER.info('Season to grep from MONTHS: %s ' % season) LOGGER.info('Season N to grep from MONTHS: %s ' % seas) except: seas = [1,2,3,4,5,6,7,8,9,10,11,12] ind = [] # TODO: change concat_vals[i][0][4:6] to dt_obj.month !!! for i in range(len(res_times)): if (int(concat_vals[i][0][4:6]) in seas[:]): ind.append(i) sf = column_stack([concat_vals[i] for i in ind]).T seas_dim_filename = season + '_' + dim_filename savetxt(seas_dim_filename, sf, fmt='%s', delimiter=',') # -------------------------- plot with R --------------- R_plot_file = 'plot_csv.R' ld2_pdf = 'local_dims.pdf' ld2_seas_pdf = season + '_local_dims.pdf' args = ['Rscript', os.path.join(Rsrc, R_plot_file), '%s' % dim_filename, '%s' % ld2_pdf] try: output, error = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() LOGGER.info('R outlog info:\n %s ' % output) LOGGER.exception('R outlog errors:\n %s ' % error) except: msg = 'Could not produce plot' LOGGER.exception(msg) # TODO: Here need produce empty pdf(s) to pass to output args = ['Rscript', os.path.join(Rsrc, R_plot_file), '%s' % seas_dim_filename, '%s' % ld2_seas_pdf] try: output, error = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() LOGGER.info('R outlog info:\n %s ' % output) LOGGER.exception('R outlog errors:\n %s ' % error) except: msg = 'Could not produce plot' LOGGER.exception(msg) # TODO: Here need produce empty pdf(s) to pass to output # # ==================================================== response.update_status('preparing output', 80) response.outputs['ldist'].file = dim_filename response.outputs['ldist_seas'].file = seas_dim_filename response.outputs['ld_pdf'].file = ld_pdf response.outputs['ld2_pdf'].file = ld2_pdf response.outputs['ld2_seas_pdf'].file = ld2_seas_pdf response.update_status('execution ended', 100) LOGGER.debug("total execution took %s seconds.", time.time() - process_start_time) return response
def _handler(self, request, response): init_process_logger('log.txt') response.outputs['output_log'].file = 'log.txt' LOGGER.info('Start process') response.update_status('execution started at : {}'.format(dt.now()), 5) process_start_time = time.time() # measure process execution time ... start_time = time.time() # measure init ... ################################ # reading in the input arguments ################################ try: response.update_status('read input parameter : %s ' % dt.now(), 7) refSt = request.inputs['refSt'][0].data #refEn = request.inputs['refEn'][0].data refEn = dt.strptime( '%s%s%s' % (dt.now().year, dt.now().month, dt.now().day), '%Y%m%d') refEn = refEn - timedelta(days=3) dateSt = request.inputs['dateSt'][0].data #dateEn = request.inputs['dateEn'][0].data dateEn = dt.strptime( '%s%s%s' % (dt.now().year, dt.now().month, dt.now().day), '%Y%m%d') dateEn = dateEn - timedelta(days=3) seasonwin = request.inputs['seasonwin'][0].data nanalog = request.inputs['nanalog'][0].data bboxDef = '-80,50,20,70' # in general format bbox = [] bboxStr = request.inputs['BBox'][0].data LOGGER.debug('BBOX selected by user: %s ' % (bboxStr)) bboxStr = bboxStr.split(',') # Checking for wrong cordinates and apply default if nesessary if (abs(float(bboxStr[0])) > 180 or abs(float(bboxStr[1]) > 180) or abs(float(bboxStr[2]) > 90) or abs(float(bboxStr[3])) > 90): bboxStr = bboxDef # request.inputs['BBox'].default # .default doesn't work anymore!!! LOGGER.debug( 'BBOX is out of the range, using default instead: %s ' % (bboxStr)) bboxStr = bboxStr.split(',') bbox.append(float(bboxStr[0])) bbox.append(float(bboxStr[2])) bbox.append(float(bboxStr[1])) bbox.append(float(bboxStr[3])) LOGGER.debug('BBOX for ocgis: %s ' % (bbox)) LOGGER.debug('BBOX original: %s ' % (bboxStr)) normalize = request.inputs['normalize'][0].data detrend = request.inputs['detrend'][0].data distance = request.inputs['dist'][0].data outformat = request.inputs['outformat'][0].data timewin = request.inputs['timewin'][0].data model_var = request.inputs['reanalyses'][0].data model, var = model_var.split('_') LOGGER.info('input parameters set') response.update_status('Read in and convert the arguments', 8) except Exception as e: msg = 'failed to read input prameter %s ' % e LOGGER.exception(msg) raise Exception(msg) ###################################### # convert types and set environment ###################################### try: response.update_status('Preparing enviroment converting arguments', 9) LOGGER.debug('date: %s %s %s %s ' % (type(refSt), refEn, dateSt, dateSt)) start = min(refSt, dateSt) end = max(refEn, dateEn) if normalize == 'None': seacyc = False else: seacyc = True if outformat == 'ascii': outformat = '.txt' elif outformat == 'netCDF': outformat = '.nc' else: LOGGER.exception('output format not valid') except Exception as e: msg = 'failed to set environment %s ' % e LOGGER.exception(msg) raise Exception(msg) ########################### # set the environment ########################### response.update_status('fetching data from archive', 10) # We work only with NCEP getlevel = False if 'z' in var: level = var.strip('z') else: level = None ########################################## # fetch Data from original data archive ########################################## try: model_nc = rl(start=start.year, end=end.year, dataset=model, variable=var, getlevel=getlevel) LOGGER.info('reanalyses data fetched') except Exception: msg = 'failed to get reanalyses data' LOGGER.exception(msg) raise Exception(msg) response.update_status('subsetting region of interest', 17) LOGGER.debug("start and end time: %s - %s" % (start, end)) time_range = [start, end] # Checking memory and dataset size m_size = get_files_size(model_nc) memory_avail = psutil.virtual_memory().available thrs = 0.2 # 20% if (m_size >= thrs * memory_avail): ser_r = True else: ser_r = False LOGGER.debug('Available Memory: %s ' % (memory_avail)) LOGGER.debug('Dataset size: %s ' % (m_size)) LOGGER.debug('Threshold: %s ' % (thrs * memory_avail)) LOGGER.debug('Serial or at once: %s ' % (ser_r)) # if ('20CRV2' in model) and ('z' in var): if ('z' in var): tmp_total = [] origvar = get_variable(model_nc) for z in model_nc: tmp_n = 'tmp_%s' % (uuid.uuid1()) b0 = call(resource=z, variable=origvar, level_range=[int(level), int(level)], geom=bbox, spatial_wrapping='wrap', prefix='levdom_' + os.path.basename(z)[0:-3]) tmp_total.append(b0) tmp_total = sorted( tmp_total, key=lambda i: os.path.splitext(os.path.basename(i))[0]) inter_subset_tmp = call(resource=tmp_total, variable=origvar, time_range=time_range) # Clean for i in tmp_total: tbr = 'rm -f %s' % (i) os.system(tbr) # Create new variable ds = Dataset(inter_subset_tmp, mode='a') z_var = ds.variables.pop(origvar) dims = z_var.dimensions new_var = ds.createVariable('z%s' % level, z_var.dtype, dimensions=(dims[0], dims[2], dims[3])) new_var[:, :, :] = squeeze(z_var[:, 0, :, :]) # new_var.setncatts({k: z_var.getncattr(k) for k in z_var.ncattrs()}) ds.close() model_subset_tmp = call(inter_subset_tmp, variable='z%s' % level) else: if ser_r: LOGGER.debug('Process reanalysis step-by-step') tmp_total = [] for z in model_nc: tmp_n = 'tmp_%s' % (uuid.uuid1()) b0 = call(resource=z, variable=var, geom=bbox, spatial_wrapping='wrap', prefix='Rdom_' + os.path.basename(z)[0:-3]) tmp_total.append(b0) tmp_total = sorted( tmp_total, key=lambda i: os.path.splitext(os.path.basename(i))[0]) model_subset_tmp = call(resource=tmp_total, variable=var, time_range=time_range) # Clean for i in tmp_total: tbr = 'rm -f %s' % (i) os.system(tbr) else: LOGGER.debug('Using whole dataset at once') model_subset_tmp = call( resource=model_nc, variable=var, geom=bbox, spatial_wrapping='wrap', time_range=time_range, ) # Rest from 20CRV... model_subset = model_subset_tmp LOGGER.info('Dataset subset done: %s ', model_subset) response.update_status('dataset subsetted', 19) # BLOCK OF DETRENDING of model_subset ! # Original model subset kept to further visualisaion if needed # Now is issue with SLP: # TODO 1 Keep trend as separate file # TODO 2 Think how to add options to plot abomalies AND original data... # May be do archive and simulation = call.. over NOT detrended data and keep it as well # TODO 3 Check with faster smoother add removing trend of each grid if detrend == 'None': orig_model_subset = model_subset else: orig_model_subset = remove_mean_trend(model_subset, varname=var) # ====================================== LOGGER.debug("get_input_subset_dataset took %s seconds.", time.time() - start_time) response.update_status('**** Input data fetched', 20) ######################## # input data preperation ######################## response.update_status('Start preparing input data', 22) start_time = time.time() # measure data preperation ... try: # Construct descriptive filenames for the three files # listed in config file # TODO check strftime for years <1900 (!) refDatesString = dt.strftime( refSt, '%Y-%m-%d') + "_" + dt.strftime(refEn, '%Y-%m-%d') simDatesString = dt.strftime( dateSt, '%Y-%m-%d') + "_" + dt.strftime(dateEn, '%Y-%m-%d') archiveNameString = "base_" + var + "_" + refDatesString + '_%.1f_%.1f_%.1f_%.1f' \ % (bbox[0], bbox[2], bbox[1], bbox[3]) simNameString = "sim_" + var + "_" + simDatesString + '_%.1f_%.1f_%.1f_%.1f' \ % (bbox[0], bbox[2], bbox[1], bbox[3]) archive = call(resource=model_subset, time_range=[refSt, refEn], prefix=archiveNameString) simulation = call(resource=model_subset, time_range=[dateSt, dateEn], prefix=simNameString) LOGGER.info('archive and simulation files generated: %s, %s' % (archive, simulation)) except Exception as e: msg = 'failed to prepare archive and simulation files %s ' % e LOGGER.exception(msg) raise Exception(msg) try: if seacyc is True: LOGGER.info('normalization function with method: %s ' % normalize) seasoncyc_base, seasoncyc_sim = analogs.seacyc( archive, simulation, method=normalize) else: seasoncyc_base = seasoncyc_sim = None except Exception as e: msg = 'failed to generate normalization files %s ' % e LOGGER.exception(msg) raise Exception(msg) output_file = 'output.txt' files = [ os.path.abspath(archive), os.path.abspath(simulation), output_file ] LOGGER.debug("Data preperation took %s seconds.", time.time() - start_time) ############################ # generate the config file ############################ config_file = analogs.get_configfile( files=files, seasoncyc_base=seasoncyc_base, seasoncyc_sim=seasoncyc_sim, base_id=model, sim_id=model, timewin=timewin, varname=var, seacyc=seacyc, cycsmooth=91, nanalog=nanalog, seasonwin=seasonwin, distfun=distance, outformat=outformat, calccor=True, silent=False, period=[ dt.strftime(refSt, '%Y-%m-%d'), dt.strftime(refEn, '%Y-%m-%d') ], bbox="{0[0]},{0[2]},{0[1]},{0[3]}".format(bbox)) response.update_status('generated config file', 25) ####################### # CASTf90 call ####################### start_time = time.time() # measure call castf90 #----------------------- try: import ctypes # TODO: This lib is for linux mkl_rt = ctypes.CDLL('libmkl_rt.so') nth = mkl_rt.mkl_get_max_threads() LOGGER.debug('Current number of threads: %s' % (nth)) mkl_rt.mkl_set_num_threads(ctypes.byref(ctypes.c_int(64))) nth = mkl_rt.mkl_get_max_threads() LOGGER.debug('NEW number of threads: %s' % (nth)) # TODO: Does it \/\/\/ work with default shell=False in subprocess... (?) os.environ['MKL_NUM_THREADS'] = str(nth) os.environ['OMP_NUM_THREADS'] = str(nth) except Exception as e: msg = 'Failed to set THREADS %s ' % e LOGGER.debug(msg) #----------------------- # ##### TEMPORAL WORKAROUND! With instaled hdf5-1.8.18 in anaconda ############### # ##### MUST be removed after castf90 recompiled with the latest hdf version # ##### NOT safe os.environ['HDF5_DISABLE_VERSION_CHECK'] = '1' #hdflib = os.path.expanduser("~") + '/anaconda/lib' #hdflib = os.getenv("HOME") + '/anaconda/lib' import pwd hdflib = pwd.getpwuid(os.getuid()).pw_dir + '/anaconda/lib' os.environ['LD_LIBRARY_PATH'] = hdflib # ################################################################################ response.update_status('Start CASTf90 call', 30) try: # response.update_status('execution of CASTf90', 50) cmd = ['analogue.out', config_file] LOGGER.debug("castf90 command: %s", cmd) output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) LOGGER.info('analogue output:\n %s', output) response.update_status('**** CASTf90 suceeded', 40) except CalledProcessError as e: msg = 'CASTf90 failed:\n{0}'.format(e.output) LOGGER.exception(msg) raise Exception(msg) LOGGER.debug("castf90 took %s seconds.", time.time() - start_time) # --------------- R cont analogs calcs ----------------------------------- ####################### # call the R scripts ####################### response.update_status( 'Start calculation of the stats and Return Periods ', 50) import shlex # import subprocess from blackswan import config from blackswan.visualisation import pdfmerge from os.path import curdir, exists, join try: #rworkspace = curdir Rsrc = config.Rsrc_dir() Rfile = 'analogs_diags-prox.R' Rdatfile = 'analogs_RT.Rdat' probs_c = 0.7 probs_n = 0.3 args = [ 'Rscript', join(Rsrc, Rfile), '%s' % output_file, '%s' % probs_c, '%s' % probs_n, '%s' % Rdatfile ] LOGGER.info('Rcall builded') LOGGER.debug('ARGS: %s' % (args)) except: msg = 'failed to build the R command' LOGGER.exception(msg) raise Exception(msg) try: output, error = subprocess.Popen( args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() LOGGER.info('R outlog info:\n %s ' % output) LOGGER.exception('R outlog errors:\n %s ' % error) if len(output) > 0: response.update_status('**** Return Periods with R suceeded', 60) else: LOGGER.exception('NO! output returned from R call') analogs_pdf = pdfmerge( ['analogs_score-diags_new.pdf', 'analogs_RP-diags_new.pdf']) except: msg = 'ReturnPeriods in R' LOGGER.exception(msg) raise Exception(msg) response.update_status('Calculation of Return Periods done ', 70) # --------------- END of R cont analogs calcs ---------------------------- response.update_status('preparing output', 75) response.outputs['analog_pdf'].file = analogs_pdf response.outputs['config'].file = config_file response.outputs['analogs'].file = output_file response.outputs['output_netcdf'].file = simulation response.outputs['target_netcdf'].file = archive if seacyc is True: response.outputs['base_netcdf'].file = seasoncyc_base response.outputs['sim_netcdf'].file = seasoncyc_sim else: # TODO: Still unclear how to overpass unknown number of outputs dummy_base = 'dummy_base.nc' dummy_sim = 'dummy_sim.nc' with open(dummy_base, 'a'): os.utime(dummy_base, None) with open(dummy_sim, 'a'): os.utime(dummy_sim, None) response.outputs['base_netcdf'].file = dummy_base response.outputs['sim_netcdf'].file = dummy_sim ######################## # generate analog viewer ######################## formated_analogs_file = analogs.reformat_analogs(output_file) # response.outputs['formated_analogs'].storage = FileStorage() response.outputs['formated_analogs'].file = formated_analogs_file LOGGER.info('analogs reformated') response.update_status('reformatted analog file', 80) viewer_html = analogs.render_viewer( # configfile=response.outputs['config'].get_url(), configfile=config_file, # datafile=response.outputs['formated_analogs'].get_url()) datafile=formated_analogs_file) response.outputs['output'].file = viewer_html response.update_status('Successfully generated analogs viewer', 90) LOGGER.info('rendered pages: %s ', viewer_html) response.update_status('execution ended', 100) LOGGER.debug("total execution took %s seconds.", time.time() - process_start_time) return response
def _handler(self, request, response): init_process_logger('log.txt') response.outputs['output_log'].file = 'log.txt' LOGGER.info('Start process') from datetime import datetime as dt from blackswan import weatherregimes as wr from tempfile import mkstemp response.update_status('execution started at : {}'.format(dt.now()), 5) ################################ # reading in the input arguments ################################ LOGGER.info('read in the arguments') # resources = self.getInputValues(identifier='resources') season = request.inputs['season'][0].data LOGGER.info('season %s', season) bboxDef = '-80,50,20,70' # in general format bbox = [] bboxStr = request.inputs['BBox'][0].data LOGGER.debug('BBOX selected by user: %s ' % (bboxStr)) bboxStr = bboxStr.split(',') # Checking for wrong cordinates and apply default if nesessary if (abs(float(bboxStr[0])) > 180 or abs(float(bboxStr[1]) > 180) or abs(float(bboxStr[2]) > 90) or abs(float(bboxStr[3])) > 90): bboxStr = bboxDef # request.inputs['BBox'].default # .default doesn't work anymore!!! LOGGER.debug( 'BBOX is out of the range, using default instead: %s ' % (bboxStr)) bboxStr = bboxStr.split(',') bbox.append(float(bboxStr[0])) bbox.append(float(bboxStr[2])) bbox.append(float(bboxStr[1])) bbox.append(float(bboxStr[3])) LOGGER.debug('BBOX for ocgis: %s ' % (bbox)) LOGGER.debug('BBOX original: %s ' % (bboxStr)) model_var = request.inputs['reanalyses'][0].data model, variable = model_var.split('_') period = request.inputs['period'][0].data LOGGER.info('period %s', period) anualcycle = request.inputs['anualcycle'][0].data kappa = request.inputs['kappa'][0].data LOGGER.info('kappa %s', kappa) method = request.inputs['method'][0].data LOGGER.info('Calc annual cycle with %s', method) sseas = request.inputs['sseas'][0].data LOGGER.info('Annual cycle calc with %s', sseas) start = dt.strptime(period.split('-')[0], '%Y%m%d') end = dt.strptime(period.split('-')[1], '%Y%m%d') LOGGER.debug('start: %s , end: %s ' % (start, end)) ########################### # set the environment ########################### response.update_status('fetching data from archive', 10) try: if model == 'NCEP': getlevel = False if 'z' in variable: level = variable.strip('z') # conform_units_to = None else: level = None # conform_units_to = 'hPa' elif '20CRV2' in model: getlevel = False if 'z' in variable: level = variable.strip('z') # conform_units_to = None else: level = None # conform_units_to = 'hPa' else: LOGGER.exception('Reanalyses dataset not known') LOGGER.info('environment set for model: %s' % model) except: msg = 'failed to set environment' LOGGER.exception(msg) raise Exception(msg) ########################################## # fetch Data from original data archive ########################################## from blackswan.datafetch import reanalyses as rl from blackswan.utils import get_variable # from os.path import basename, splitext from os import system from netCDF4 import Dataset from numpy import squeeze try: model_nc = rl(start=start.year, end=end.year, dataset=model, variable=variable, getlevel=getlevel) LOGGER.info('reanalyses data fetched') except: msg = 'failed to get reanalyses data' LOGGER.exception(msg) raise Exception(msg) response.update_status('fetching data done', 20) ############################################################ # get the required bbox and time region from resource data ############################################################ response.update_status('subsetting region of interest', 30) time_range = [start, end] ############################################################ # Block of level and domain selection for geop huge dataset ############################################################ LevMulti = False # =========================================================================================== # Temporary add step-by-step also for pressure... for slow VM machine... if ('z' in variable) or ('p' in variable): tmp_total = [] origvar = get_variable(model_nc) if ('z' in variable): level_range = [int(level), int(level)] else: level_range = None if (LevMulti == False): for z in model_nc: b0 = call(resource=z, variable=origvar, level_range=level_range, geom=bbox, spatial_wrapping='wrap', prefix='levdom_' + basename(z)[0:-3]) tmp_total.append(b0) else: # multiproc - no inprovements yet, need to check in hi perf machine... # ----------------------- try: import ctypes import os # TODO: This lib is for linux mkl_rt = ctypes.CDLL('libmkl_rt.so') nth = mkl_rt.mkl_get_max_threads() LOGGER.debug('Current number of threads: %s' % (nth)) mkl_rt.mkl_set_num_threads(ctypes.byref(ctypes.c_int(64))) nth = mkl_rt.mkl_get_max_threads() LOGGER.debug('NEW number of threads: %s' % (nth)) # TODO: Does it \/\/\/ work with default shell=False in subprocess... (?) os.environ['MKL_NUM_THREADS'] = str(nth) os.environ['OMP_NUM_THREADS'] = str(nth) except Exception as e: msg = 'Failed to set THREADS %s ' % e LOGGER.debug(msg) # ----------------------- from multiprocessing import Pool pool = Pool() # from multiprocessing.dummy import Pool as ThreadPool # pool = ThreadPool() tup_var = [origvar] * len(model_nc) tup_lev = [level] * len(model_nc) tup_bbox = [bbox] * len(model_nc) tup_args = zip(model_nc, tup_var, tup_lev, tup_bbox) tmp_total = pool.map(ocgis_call_wrap, tup_args) pool.close() pool.join() LOGGER.debug('Temporal subset files: %s' % (tmp_total)) tmp_total = sorted(tmp_total, key=lambda i: splitext(basename(i))[0]) inter_subset_tmp = call(resource=tmp_total, variable=origvar, time_range=time_range) # Clean for i in tmp_total: tbr = 'rm -f %s' % (i) system(tbr) if ('z' in variable): # Create new variable for Z geop ds = Dataset(inter_subset_tmp, mode='a') z_var = ds.variables.pop(origvar) dims = z_var.dimensions new_var = ds.createVariable('z%s' % level, z_var.dtype, dimensions=(dims[0], dims[2], dims[3])) new_var[:, :, :] = squeeze(z_var[:, 0, :, :]) # new_var.setncatts({k: z_var.getncattr(k) for k in z_var.ncattrs()}) ds.close() model_subset = call(inter_subset_tmp, variable='z%s' % level) else: model_subset = inter_subset_tmp else: model_subset = call( resource=model_nc, variable=variable, geom=bbox, spatial_wrapping='wrap', time_range=time_range, # conform_units_to=conform_units_to ) # ============================================================================================= LOGGER.info('Dataset subset done: %s ', model_subset) response.update_status('dataset subsetted', 40) ############################################## # computing anomalies ############################################## response.update_status('computing anomalies ', 50) cycst = anualcycle.split('-')[0] cycen = anualcycle.split('-')[1] reference = [ dt.strptime(cycst, '%Y%m%d'), dt.strptime(cycen, '%Y%m%d') ] LOGGER.info('reference time: %s', reference) model_anomal = wr.get_anomalies(model_subset, reference=reference, method=method, sseas=sseas) # , variable=variable) ##################### # extracting season ##################### response.update_status('normalizing data', 60) model_season = wr.get_season(model_anomal, season=season) response.update_status('anomalies computed and normalized', 70) ####################### # call the R scripts ####################### response.update_status('Start weather regime clustering ', 80) import shlex import subprocess from blackswan import config from os.path import curdir, exists, join try: # rworkspace = curdir Rsrc = config.Rsrc_dir() Rfile = 'weatherregimes_model.R' infile = model_season # model_subset #model_ponderate # modelname = model # yr1 = start.year # yr2 = end.year ip, output_graphics = mkstemp(dir=curdir, suffix='.pdf') ip, file_pca = mkstemp(dir=curdir, suffix='.txt') ip, file_class = mkstemp(dir=curdir, suffix='.Rdat') args = [ 'Rscript', join(Rsrc, Rfile), '%s/' % curdir, '%s/' % Rsrc, '%s' % infile, '%s' % variable, '%s' % output_graphics, '%s' % file_pca, '%s' % file_class, '%s' % season, '%s' % start.year, '%s' % end.year, '%s' % model_var, '%s' % kappa ] LOGGER.info('Rcall builded') LOGGER.debug('ARGS: %s' % (args)) except: msg = 'failed to build the R command' LOGGER.exception(msg) raise Exception(msg) try: output, error = subprocess.Popen( args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() LOGGER.info('R outlog info:\n %s ' % output) LOGGER.exception('R outlog errors:\n %s ' % error) if len(output) > 0: response.update_status('**** weatherregime in R suceeded', 90) else: LOGGER.exception('NO! output returned from R call') except: msg = 'weatherregime in R' LOGGER.exception(msg) raise Exception(msg) response.update_status('Weather regime clustering done ', 95) ############################################ # set the outputs ############################################ # response.update_status('Set the process outputs ', 96) response.outputs['Routput_graphic'].file = output_graphics response.outputs['output_pca'].file = file_pca response.outputs['output_classification'].file = file_class response.outputs['output_netcdf'].file = model_subset response.update_status('done', 100) return response
def _handler(self, request, response): init_process_logger('log.txt') response.outputs['output_log'].file = 'log.txt' LOGGER.info('Start process') response.update_status('execution started at : {}'.format(dt.now()), 5) process_start_time = time.time() # measure process execution time ... start_time = time.time() # measure init ... ################################ # reading in the input arguments ################################ try: response.update_status('read input parameter : %s ' % dt.now(), 6) refSt = request.inputs['refSt'][0].data refEn = request.inputs['refEn'][0].data dateSt = request.inputs['dateSt'][0].data dateEn = request.inputs['dateEn'][0].data seasonwin = request.inputs['seasonwin'][0].data nanalog = request.inputs['nanalog'][0].data bboxDef = '-20,40,30,70' # in general format bbox = [] bboxStr = request.inputs['BBox'][0].data LOGGER.debug('BBOX selected by user: %s ' % (bboxStr)) bboxStr = bboxStr.split(',') # Checking for wrong cordinates and apply default if nesessary if (abs(float(bboxStr[0])) > 180 or abs(float(bboxStr[1]) > 180) or abs(float(bboxStr[2]) > 90) or abs(float(bboxStr[3])) > 90): bboxStr = bboxDef # request.inputs['BBox'].default # .default doesn't work anymore!!! LOGGER.debug('BBOX is out of the range, using default instead: %s ' % (bboxStr)) bboxStr = bboxStr.split(',') bbox.append(float(bboxStr[0])) bbox.append(float(bboxStr[2])) bbox.append(float(bboxStr[1])) bbox.append(float(bboxStr[3])) LOGGER.debug('BBOX for ocgis: %s ' % (bbox)) LOGGER.debug('BBOX original: %s ' % (bboxStr)) plot = request.inputs['plot'][0].data distance = request.inputs['dist'][0].data outformat = request.inputs['outformat'][0].data timewin = request.inputs['timewin'][0].data model_var = request.inputs['reanalyses'][0].data model, var = model_var.split('_') ref_model_var = request.inputs['Refreanalyses'][0].data ref_model, ref_var = ref_model_var.split('_') LOGGER.info('input parameters set') response.update_status('Read in and convert the arguments', 7) except Exception as e: msg = 'failed to read input prameter %s ' % e LOGGER.exception(msg) raise Exception(msg) ###################################### # convert types and set environment ###################################### try: response.update_status('Preparing enviroment converting arguments', 8) LOGGER.debug('date: %s %s %s %s ' % (type(refSt), refEn, dateSt, dateSt)) # normalize == 'None': seacyc = False if outformat == 'ascii': outformat = '.txt' elif outformat == 'netCDF': outformat = '.nc' else: LOGGER.exception('output format not valid') except Exception as e: msg = 'failed to set environment %s ' % e LOGGER.exception(msg) raise Exception(msg) ########################### # set the environment ########################### response.update_status('fetching data from archive', 9) getlevel = False if 'z' in var: level = var.strip('z') else: level = None ########################################## # fetch Data from original data archive ########################################## try: model_nc = rl(start=dateSt.year, end=dateEn.year, dataset=model, variable=var, getlevel=getlevel) ref_model_nc = rl(start=refSt.year, end=refEn.year, dataset=ref_model, variable=ref_var, getlevel=getlevel) LOGGER.info('reanalyses data fetched') except Exception: msg = 'failed to get reanalyses data' LOGGER.exception(msg) raise Exception(msg) response.update_status('subsetting region of interest', 10) # Checking memory and dataset size model_size = get_files_size(model_nc) ref_model_size = get_files_size(ref_model_nc) m_size = max(model_size, ref_model_size) memory_avail = psutil.virtual_memory().available thrs = 0.2 # 20% if (m_size >= thrs * memory_avail): ser_r = True else: ser_r = False LOGGER.debug('Available Memory: %s ' % (memory_avail)) LOGGER.debug('Dataset size: %s ' % (m_size)) LOGGER.debug('Threshold: %s ' % (thrs * memory_avail)) LOGGER.debug('Serial or at once: %s ' % (ser_r)) # ##################################################### # Construct descriptive filenames for the three files # # listed in config file # # TODO check strftime for years <1900 (!) # # ##################################################### # refDatesString = dt.strftime(refSt, '%Y-%m-%d') + "_" + dt.strftime(refEn, '%Y-%m-%d') # simDatesString = dt.strftime(dateSt, '%Y-%m-%d') + "_" + dt.strftime(dateEn, '%Y-%m-%d') # Fix < 1900 issue... refDatesString = refSt.isoformat().strip().split("T")[0] + "_" + refEn.isoformat().strip().split("T")[0] simDatesString = dateSt.isoformat().strip().split("T")[0] + "_" + dateEn.isoformat().strip().split("T")[0] archiveNameString = "base_" + var + "_" + refDatesString + '_%.1f_%.1f_%.1f_%.1f' \ % (bbox[0], bbox[2], bbox[1], bbox[3]) simNameString = "sim_" + var + "_" + simDatesString + '_%.1f_%.1f_%.1f_%.1f' \ % (bbox[0], bbox[2], bbox[1], bbox[3]) if ('z' in var): # ------------------ NCEP ------------------- tmp_total = [] origvar = get_variable(model_nc) for z in model_nc: b0 = call(resource=z, variable=origvar, level_range=[int(level), int(level)], geom=bbox, spatial_wrapping='wrap', prefix='levdom_' + os.path.basename(z)[0:-3]) tmp_total.append(b0) time_range = [dateSt, dateEn] tmp_total = sorted(tmp_total, key=lambda i: os.path.splitext(os.path.basename(i))[0]) inter_subset_tmp = call(resource=tmp_total, variable=origvar, time_range=time_range) # Clean for i in tmp_total: tbr = 'rm -f %s' % (i) os.system(tbr) # Create new variable ds = Dataset(inter_subset_tmp, mode='a') z_var = ds.variables.pop(origvar) dims = z_var.dimensions new_var = ds.createVariable('z%s' % level, z_var.dtype, dimensions=(dims[0], dims[2], dims[3])) new_var[:, :, :] = squeeze(z_var[:, 0, :, :]) ds.close() simulation = call(inter_subset_tmp, variable='z%s' % level, prefix=simNameString) # ------------------ 20CRV2c ------------------- tmp_total = [] origvar = get_variable(ref_model_nc) for z in ref_model_nc: tmp_n = 'tmp_%s' % (uuid.uuid1()) # select level and regrid b0 = call(resource=z, variable=origvar, level_range=[int(level), int(level)], spatial_wrapping='wrap', cdover='system', regrid_destination=model_nc[0], regrid_options='bil', prefix=tmp_n) # select domain b01 = call(resource=b0, variable=origvar, geom=bbox, spatial_wrapping='wrap', prefix='levregr_' + os.path.basename(z)[0:-3]) tbr = 'rm -f %s' % (b0) os.system(tbr) tbr = 'rm -f %s.nc' % (tmp_n) os.system(tbr) tmp_total.append(b01) time_range = [refSt, refEn] tmp_total = sorted(tmp_total, key=lambda i: os.path.splitext(os.path.basename(i))[0]) ref_inter_subset_tmp = call(resource=tmp_total, variable=origvar, time_range=time_range) # Clean for i in tmp_total: tbr = 'rm -f %s' % (i) os.system(tbr) # Create new variable ds = Dataset(ref_inter_subset_tmp, mode='a') z_var = ds.variables.pop(origvar) dims = z_var.dimensions new_var = ds.createVariable('z%s' % level, z_var.dtype, dimensions=(dims[0], dims[2], dims[3])) new_var[:, :, :] = squeeze(z_var[:, 0, :, :]) ds.close() archive = call(ref_inter_subset_tmp, variable='z%s' % level, prefix=archiveNameString) else: if ser_r: LOGGER.debug('Process reanalysis step-by-step') # ----- NCEP ------ tmp_total = [] for z in model_nc: b0 = call(resource=z, variable=var, geom=bbox, spatial_wrapping='wrap', prefix='Rdom_' + os.path.basename(z)[0:-3]) tmp_total.append(b0) tmp_total = sorted(tmp_total, key=lambda i: os.path.splitext(os.path.basename(i))[0]) simulation = call(resource=tmp_total, variable=var, time_range=[dateSt, dateEn], prefix=simNameString) # Clean for i in tmp_total: tbr = 'rm -f %s' % (i) os.system(tbr) # ----- 20CRV2c ------ tmp_n = 'tmp_%s' % (uuid.uuid1()) tmp_total = [] for z in ref_model_nc: # regrid b0 = call(resource=z, variable=ref_var, spatial_wrapping='wrap', cdover='system', regrid_destination=model_nc[0], regrid_options='bil', prefix=tmp_n) # select domain b01 = call(resource=b0, variable=ref_var, geom=bbox, spatial_wrapping='wrap', prefix='ref_Rdom_' + os.path.basename(z)[0:-3]) tbr = 'rm -f %s' % (b0) os.system(tbr) tbr = 'rm -f %s.nc' % (tmp_n) os.system(tbr) tmp_total.append(b01) tmp_total = sorted(tmp_total, key=lambda i: os.path.splitext(os.path.basename(i))[0]) archive = call(resource=tmp_total, variable=ref_var, time_range=[refSt, refEn], prefix=archiveNameString) # Clean for i in tmp_total: tbr = 'rm -f %s' % (i) os.system(tbr) else: LOGGER.debug('Using whole dataset at once') simulation = call(resource=model_nc, variable=var, geom=bbox, spatial_wrapping='wrap', time_range=[dateSt, dateEn], prefix=simNameString) ref_inter_subset_tmp = call(resource=ref_model_nc, variable=ref_var, spatial_wrapping='wrap', cdover='system', regrid_destination=model_nc[0], regrid_options='bil') archive = call(resource=ref_inter_subset_tmp, geom=bbox, spatial_wrapping='wrap', time_range=[refSt, refEn], prefix=archiveNameString) response.update_status('datasets subsetted', 15) LOGGER.debug("get_input_subset_dataset took %s seconds.", time.time() - start_time) response.update_status('**** Input data fetched', 20) ######################## # input data preperation ######################## response.update_status('Start preparing input data', 30) start_time = time.time() # measure data preperation ... LOGGER.info('archive and simulation files generated: %s, %s' % (archive, simulation)) # Rename variable (TODO: For this specific process we know names: slp and prmsl...) try: if level is not None: out_var = 'z%s' % level else: var_archive = get_variable(archive) var_simulation = get_variable(simulation) if var_archive != var_simulation: rename_variable(archive, oldname=var_archive, newname=var_simulation) out_var = var_simulation LOGGER.info('varname %s in netCDF renamed to %s' % (var_archive, var_simulation)) except: msg = 'failed to rename variable in target files' LOGGER.exception(msg) raise Exception(msg) # seacyc is False: seasoncyc_base = seasoncyc_sim = None output_file = 'output.txt' files = [os.path.abspath(archive), os.path.abspath(simulation), output_file] LOGGER.debug("Data preperation took %s seconds.", time.time() - start_time) ############################ # generate the config file ############################ config_file = analogs.get_configfile( files=files, seasoncyc_base=seasoncyc_base, seasoncyc_sim=seasoncyc_sim, base_id=ref_model, sim_id=model, timewin=timewin, varname=out_var, seacyc=seacyc, cycsmooth=91, nanalog=nanalog, seasonwin=seasonwin, distfun=distance, outformat=outformat, calccor=True, silent=False, # period=[dt.strftime(refSt, '%Y-%m-%d'), dt.strftime(refEn, '%Y-%m-%d')], period=[refSt.isoformat().strip().split("T")[0], refEn.isoformat().strip().split("T")[0]], bbox="{0[0]},{0[2]},{0[1]},{0[3]}".format(bbox)) response.update_status('generated config file', 40) ####################### # CASTf90 call ####################### start_time = time.time() # measure call castf90 # ----------------------- try: import ctypes # TODO: This lib is for linux mkl_rt = ctypes.CDLL('libmkl_rt.so') nth = mkl_rt.mkl_get_max_threads() LOGGER.debug('Current number of threads: %s' % (nth)) mkl_rt.mkl_set_num_threads(ctypes.byref(ctypes.c_int(64))) nth = mkl_rt.mkl_get_max_threads() LOGGER.debug('NEW number of threads: %s' % (nth)) # TODO: Does it \/\/\/ work with default shell=False in subprocess... (?) os.environ['MKL_NUM_THREADS'] = str(nth) os.environ['OMP_NUM_THREADS'] = str(nth) except Exception as e: msg = 'Failed to set THREADS %s ' % e LOGGER.debug(msg) # ----------------------- # ##### TEMPORAL WORKAROUND! With instaled hdf5-1.8.18 in anaconda ############### # ##### MUST be removed after castf90 recompiled with the latest hdf version # ##### NOT safe os.environ['HDF5_DISABLE_VERSION_CHECK'] = '1' # hdflib = os.path.expanduser("~") + '/anaconda/lib' # hdflib = os.getenv("HOME") + '/anaconda/lib' import pwd hdflib = pwd.getpwuid(os.getuid()).pw_dir + '/anaconda/lib' os.environ['LD_LIBRARY_PATH'] = hdflib # ################################################################################ response.update_status('Start CASTf90 call', 50) try: # response.update_status('execution of CASTf90', 50) cmd = ['analogue.out', config_file] LOGGER.debug("castf90 command: %s", cmd) output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) LOGGER.info('analogue output:\n %s', output) response.update_status('**** CASTf90 suceeded', 60) except CalledProcessError as e: msg = 'CASTf90 failed:\n{0}'.format(e.output) LOGGER.exception(msg) raise Exception(msg) LOGGER.debug("castf90 took %s seconds.", time.time() - start_time) # TODO: Add try - except for pdfs if plot == 'Yes': analogs_pdf = analogs.plot_analogs(configfile=config_file) else: analogs_pdf = 'dummy_plot.pdf' with open(analogs_pdf, 'a'): os.utime(analogs_pdf, None) response.update_status('preparing output', 70) response.outputs['analog_pdf'].file = analogs_pdf response.outputs['config'].file = config_file response.outputs['analogs'].file = output_file ######################## # generate analog viewer ######################## formated_analogs_file = analogs.reformat_analogs(output_file) # response.outputs['formated_analogs'].storage = FileStorage() response.outputs['formated_analogs'].file = formated_analogs_file LOGGER.info('analogs reformated') response.update_status('reformatted analog file', 80) viewer_html = analogs.render_viewer( # configfile=response.outputs['config'].get_url(), configfile=config_file, # datafile=response.outputs['formated_analogs'].get_url()) datafile=formated_analogs_file) response.outputs['output'].file = viewer_html response.update_status('Successfully generated analogs viewer', 90) LOGGER.info('rendered pages: %s ', viewer_html) response.update_status('execution ended', 100) LOGGER.debug("total execution took %s seconds.", time.time() - process_start_time) return response