def execute(self): import os from flyingpigeon.utils import archive # import tarfile from tempfile import mkstemp from os import path from numpy import squeeze ncs = self.getInputValues(identifier='resource') indices = self.indices.getValue() polygons = self.polygons.getValue() mosaic = self.mosaic.getValue() groupings = self.groupings.getValue() if polygons==None: self.status.set('No countries selected, entire domain will be calculated' , 10) logger.debug('indices=%s', indices) logger.debug('groupings=%s', groupings) logger.debug('num files=%s', len(ncs)) self.status.set('processing indices : %s' % indices, 12) results = squeeze(calc_indice_simple( resource=ncs, mosaic=mosaic, indices=indices, polygons=polygons, groupings=groupings, dir_output=path.curdir, )) results_list = results.tolist() self.status.set('indices calculated', 90) logger.debug('results type: %s', type(results_list)) logger.debug('indices files: %s ' % results_list ) try: archive_indices = archive(results_list) logger.info('archive prepared') except Exception as e: msg = "archive preparation failed" logger.exception(msg) raise Exception(msg) try: self.output.setValue(archive_indices) if type(results_list) == list: i = next((i for i, x in enumerate(results.tolist()) if x), None) self.output_netcdf.setValue(str(results[i])) elif type(results_list) == str: self.output_netcdf.setValue(results_list) else: logger.debug('results_list type: %s not extractable ' % type(results_list) ) self.output_netcdf.setValue(None) except Exception as e: msg = "extraction of example file failed" logger.exception(msg) self.status.set('done', 100)
def get_indices(resources, indices): from flyingpigeon.utils import sort_by_filename, calc_grouping, drs_filename from flyingpigeon.ocgis_module import call from flyingpigeon.indices import indice_variable, calc_indice_simple #names = [drs_filename(nc, skip_timestamp=False, skip_format=False, # variable=None, rename_file=True, add_file_path=True) for nc in resources] ncs = sort_by_filename(resources, historical_concatination=True) ncs_indices = [] logger.info('resources sorted found %s datasets' % len(ncs.keys()) ) for key in ncs.keys(): for indice in indices: try: name , month = indice.split('_') variable=key.split('_')[0] # print name, month , variable if variable == indice_variable(name): logger.info('calculating indice %s ' % indice) prefix=key.replace(variable, name).replace('_day_','_%s_' % month) nc = calc_indice_simple(resource=ncs[key], variable=variable, prefix=prefix, indices=name, groupings=month, memory_limit=500) #grouping = calc_grouping(month) #calc = [{'func' : 'icclim_' + name, 'name' : name}] #nc = call(resource=ncs[key], variable=variable, calc=calc, calc_grouping=grouping, prefix=prefix , memory_limit=500) #memory_limit=500 ncs_indices.append(nc[0]) logger.info('Successful calculated indice %s %s' % (key, indice)) except Exception as e: logger.exception('failed to calculate indice %s %s' % (key, indice)) return ncs_indices
def test_indice_simple(): # SU expects tasmax resources = [local_path(TESTDATA['cordex_tasmax_2006_nc'])] output = indices.calc_indice_simple( resources, indices=['SU'], groupings='year', dir_output=tempfile.mkdtemp()) assert os.path.basename(output[0]) == 'SU_EUR-44_MPI-M-MPI-ESM-LR_rcp45_r1i1p1_MPI-CSC-REMO2009_v1_mon_20060215-20061216.nc' ds = Dataset(output[0]) # SU variable must be in result assert 'SU' in ds.variables # 1 year assert len(ds.variables['time']) == 1
def test_indice_simple(): # SU expects tasmax resources = [local_path(TESTDATA['cordex_tasmax_2006_nc'])] output = indices.calc_indice_simple( resources, indice='SU', grouping='yr', dir_output=tempfile.mkdtemp()) assert os.path.basename(output[0]) == 'SU_EUR-44_MPI-M-MPI-ESM-LR_rcp45_r1i1p1_MPI-CSC-REMO2009_v1_mon_200602-200612.nc' # noqa ds = Dataset(output[0]) # SU variable must be in result assert 'SU' in ds.variables # 1 year assert len(ds.variables['time']) == 1
def execute(self): import os import tarfile from tempfile import mkstemp from os import path ncs = self.getInputValues(identifier='resource') indices = self.indices.getValue() polygons = self.polygons.getValue() mosaik = self.mosaik.getValue() groupings = self.groupings.getValue() # getInputValues(identifier='groupings') polygons = self.polygons.getValue() # if len(polygons)==0: # polygons = None self.status.set('starting: indices=%s, groupings=%s, countries=%s, num_files=%s' % (indices, groupings, polygons, len(ncs)), 0) results = calc_indice_simple( resource = ncs, mosaik=mosaik, indices = indices, polygons= polygons, groupings = groupings, dir_output = path.curdir, ) if not results: raise Exception("failed to produce results") self.status.set('num results %s' % len(results), 90) try: (fp_tarf, tarf) = mkstemp(dir=".", suffix='.tar') tar = tarfile.open(tarf, "w") for result in results: tar.add( result , arcname = os.path.basename(result)) tar.close() logger.info('Tar file prepared') except Exception as e: msg = "Tar file preparation failed" logger.exception(msg) raise Exception(msg) self.output.setValue( tarf ) self.status.set('done: indice=%s, num_files=%s' % (indices, len(ncs)), 100)
def get_indices(resources, indices): """ calculating indices (netCDF files) defined in _SDMINDICES_ :param resources: :param indices: indices defined in _SDMINDICES_ :return list: list of filepathes to netCDF files """ from flyingpigeon.utils import sort_by_filename, calc_grouping, drs_filename, unrotate_pole # from flyingpigeon.ocgis_module import call from flyingpigeon.indices import indice_variable, calc_indice_simple # names = [drs_filename(nc, skip_timestamp=False, skip_format=False, # variable=None, rename_file=True, add_file_path=True) for nc in resources] ncs = sort_by_filename(resources, historical_concatination=True) ncs_indices = [] logger.info('resources sorted found %s datasets' % len(ncs.keys())) for key in ncs.keys(): for indice in indices: try: name, month = indice.split('_') variable = key.split('_')[0] # print name, month , variable if variable == indice_variable(name): logger.info('calculating indice %s ' % indice) prefix = key.replace(variable, name).replace('_day_', '_%s_' % month) nc = calc_indice_simple(resource=ncs[key], variable=variable, polygons=['Europe', 'Africa', 'Asia', 'North America', 'Oceania', 'South America', 'Antarctica'], mosaic=True, prefix=prefix, indices=name, groupings=month) if nc is not None: coords = unrotate_pole(nc[0], write_to_file=True) ncs_indices.append(nc[0]) logger.info('Successful calculated indice %s %s' % (key, indice)) else: msg = 'failed to calculate indice %s %s' % (key, indice) logger.exception(msg) except: msg = 'failed to calculate indice %s %s' % (key, indice) logger.exception(msg) raise return ncs_indices
def get_indices(resources, indices): from flyingpigeon.utils import sort_by_filename, calc_grouping, drs_filename from flyingpigeon.ocgis_module import call from flyingpigeon.indices import indice_variable, calc_indice_simple #names = [drs_filename(nc, skip_timestamp=False, skip_format=False, # variable=None, rename_file=True, add_file_path=True) for nc in resources] ncs = sort_by_filename(resources, historical_concatination=True) ncs_indices = [] logger.info('resources sorted found %s datasets' % len(ncs.keys())) for key in ncs.keys(): for indice in indices: try: name, month = indice.split('_') variable = key.split('_')[0] # print name, month , variable if variable == indice_variable(name): logger.info('calculating indice %s ' % indice) prefix = key.replace(variable, name).replace('_day_', '_%s_' % month) nc = calc_indice_simple(resource=ncs[key], variable=variable, prefix=prefix, indices=name, groupings=month, memory_limit=500) #grouping = calc_grouping(month) #calc = [{'func' : 'icclim_' + name, 'name' : name}] #nc = call(resource=ncs[key], variable=variable, calc=calc, calc_grouping=grouping, prefix=prefix , memory_limit=500) #memory_limit=500 ncs_indices.append(nc[0]) logger.info('Successful calculated indice %s %s' % (key, indice)) except Exception as e: logger.exception('failed to calculate indice %s %s' % (key, indice)) return ncs_indices
def _handler(self, request, response): init_process_logger('log.txt') response.outputs['output_log'].file = 'log.txt' try: resources = archiveextract( resource=rename_complexinputs(request.inputs['resource'])) indices = [inpt.data for inpt in request.inputs['indices']] grouping = [inpt.data for inpt in request.inputs['grouping']] if 'mosaic' in request.inputs: mosaic = request.inputs['mosaic'][0].data else: mosaic = False if 'region' in request.inputs: region = [inpt.data for inpt in request.inputs['region']] else: region = None LOGGER.debug('grouping: {}'.format(grouping)) LOGGER.debug('mosaic: {}'.format(mosaic)) LOGGER.debug('indices: {}'.format(indices)) LOGGER.debug('region: {}'.format(region)) LOGGER.debug('Nr of input files: {}'.format(len(resources))) except Exception as ex: LOGGER.exception('failed to read in the arguments: {}'.format(str(ex))) response.update_status( 'starting: indices={}, grouping={}, num_files={}'.format(indices, grouping, len(resources)), 2) results = [] from flyingpigeon.utils import sort_by_filename datasets = sort_by_filename(resources, historical_concatination=True) results = [] try: group = grouping[0] # for group in grouping: indice = indices[0] # for indice in indices: for key in datasets.keys(): try: response.update_status('Dataset {}: {}'.format(len(results) + 1, key), 10) LOGGER.debug('grouping: {}'.format(grouping)) LOGGER.debug('mosaic: {}'.format(mosaic)) LOGGER.debug('indice: {}'.format(indice)) LOGGER.debug('region: {}'.format(region)) LOGGER.debug('Nr of input files: {}'.format(len(datasets[key]))) result = calc_indice_simple( resource=datasets[key], mosaic=mosaic, indice=indice, polygons=region, grouping=group, # dir_output=path.curdir, ) LOGGER.debug('result: {}'.format(result)) results.extend(result) except Exception as ex: msg = 'failed for {}: {}'.format(key, str(ex)) LOGGER.exception(msg) raise Exception(msg) except Exception as ex: msg = 'Failed to calculate indices: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) # # if not results: # # raise Exception("failed to produce results") # # response.update_status('num results %s' % len(results), 90) tarf = archive(results) response.outputs['output_archive'].file = tarf i = next((i for i, x in enumerate(results) if x), None) if i is None: i = 'dummy.nc' response.outputs['ncout'].file = results[i] # response.update_status("done", 100) return response
output_format='nc').execute() print geom rd = RequestDataset(ncs) rd.dimension_map.set_bounds(DimensionMapKey.TIME, None) indice = 'TG' geom = OcgOperations(rd, calc=[{ 'func': 'icclim_' + indice, 'name': indice }], calc_grouping=['year', 'month'], prefix='multi_file', output_format='nc').execute() print geom from flyingpigeon.indices import calc_indice_simple fp_indice = calc_indice_simple(resource=ncs, variable=None, prefix=None, indice='TG', polygons=['CMR'], mosaic=True, grouping='yr', dir_output=None, dimension_map=None, memory_limit=None) print fp_indice
def get_indices(resource, indices): """ calculating indices (netCDF files) defined in _SDMINDICES_ :param resources: files containing one Dataset :param indices: List of indices defined in _SDMINDICES_. Index needs to be based on the resource variable :return list: list of filepathes to netCDF files """ from flyingpigeon.utils import sort_by_filename, calc_grouping, drs_filename, unrotate_pole, get_variable # from flyingpigeon.ocgis_module import call from flyingpigeon.indices import indice_variable, calc_indice_simple from flyingpigeon.subset import masking from flyingpigeon.utils import searchfile from flyingpigeon.utils import search_landsea_mask_by_esgf from os.path import basename # names = [drs_filename(nc, skip_timestamp=False, skip_format=False, # variable=None, rename_file=True, add_file_path=True) for nc in resources] variable = get_variable(resource) masked_datasets = [] max_count = len(resource) for ds in resource: ds_name = basename(ds) LOGGER.debug('masking dataset: %s', ds_name) try: landsea_mask = search_landsea_mask_by_esgf(ds) LOGGER.debug("using landsea_mask: %s", landsea_mask) prefix = ds_name.replace('.nc', '') new_ds = masking(ds, landsea_mask, land_area=True, prefix=prefix) masked_datasets.append(new_ds) except: LOGGER.exception("Could not subset dataset.") break else: LOGGER.info("masked: %d/%d", len(masked_datasets), max_count) if not masked_datasets: raise Exception("Could not mask input files.") ncs = sort_by_filename(masked_datasets, historical_concatination=True) key = ncs.keys()[0] ncs_indices = [] LOGGER.info('resources sorted found %s datasets', len(ncs.keys())) for indice in indices: try: name, month = indice.split('_') # print name, month , variable if variable == indice_variable(name): LOGGER.info('calculating indice %s ' % indice) prefix = key.replace(variable, name).replace('_day_', '_%s_' % month) nc = calc_indice_simple(resource=resource, variable=variable, # polygons=['Europe', 'Africa', 'Asia', 'North America', 'Oceania', # 'South America', 'Antarctica'], # mosaic=True, prefix=prefix, indice=name, grouping=month) if nc is not None: # coords = unrotate_pole(nc[0], write_to_file=True) ncs_indices.append(nc[0]) LOGGER.info('Successful calculated indice %s %s' % (key, indice)) else: msg = 'failed to calculate indice %s %s' % (key, indice) LOGGER.exception(msg) except: msg = 'failed to calculate indice %s %s' % (key, indice) LOGGER.exception(msg) return ncs_indices