def execute(self):
import os
from flyingpigeon.utils import archive
# import tarfile
from tempfile import mkstemp
from os import path
from numpy import squeeze
ncs = self.getInputValues(identifier='resource')
indices = self.indices.getValue()
polygons = self.polygons.getValue()
mosaic = self.mosaic.getValue()
groupings = self.groupings.getValue()
if polygons==None:
self.status.set('No countries selected, entire domain will be calculated' , 10)
logger.debug('indices=%s', indices)
logger.debug('groupings=%s', groupings)
logger.debug('num files=%s', len(ncs))
self.status.set('processing indices : %s' % indices, 12)
results = squeeze(calc_indice_simple(
resource=ncs,
mosaic=mosaic,
indices=indices,
polygons=polygons,
groupings=groupings,
dir_output=path.curdir,
))
results_list = results.tolist()
self.status.set('indices calculated', 90)
logger.debug('results type: %s', type(results_list))
logger.debug('indices files: %s ' % results_list )
try:
archive_indices = archive(results_list)
logger.info('archive prepared')
except Exception as e:
msg = "archive preparation failed"
logger.exception(msg)
raise Exception(msg)
try:
self.output.setValue(archive_indices)
if type(results_list) == list:
i = next((i for i, x in enumerate(results.tolist()) if x), None)
self.output_netcdf.setValue(str(results[i]))
elif type(results_list) == str:
self.output_netcdf.setValue(results_list)
else:
logger.debug('results_list type: %s not extractable ' % type(results_list) )
self.output_netcdf.setValue(None)
except Exception as e:
msg = "extraction of example file failed"
logger.exception(msg)
self.status.set('done', 100)
def get_indices(resources, indices):
from flyingpigeon.utils import sort_by_filename, calc_grouping, drs_filename
from flyingpigeon.ocgis_module import call
from flyingpigeon.indices import indice_variable, calc_indice_simple
#names = [drs_filename(nc, skip_timestamp=False, skip_format=False,
# variable=None, rename_file=True, add_file_path=True) for nc in resources]
ncs = sort_by_filename(resources, historical_concatination=True)
ncs_indices = []
logger.info('resources sorted found %s datasets' % len(ncs.keys()) )
for key in ncs.keys():
for indice in indices:
try:
name , month = indice.split('_')
variable=key.split('_')[0]
# print name, month , variable
if variable == indice_variable(name):
logger.info('calculating indice %s ' % indice)
prefix=key.replace(variable, name).replace('_day_','_%s_' % month)
nc = calc_indice_simple(resource=ncs[key], variable=variable, prefix=prefix, indices=name, groupings=month, memory_limit=500)
#grouping = calc_grouping(month)
#calc = [{'func' : 'icclim_' + name, 'name' : name}]
#nc = call(resource=ncs[key], variable=variable, calc=calc, calc_grouping=grouping, prefix=prefix , memory_limit=500) #memory_limit=500
ncs_indices.append(nc[0])
logger.info('Successful calculated indice %s %s' % (key, indice))
except Exception as e:
logger.exception('failed to calculate indice %s %s' % (key, indice))
return ncs_indices
def test_indice_simple():
# SU expects tasmax
resources = [local_path(TESTDATA['cordex_tasmax_2006_nc'])]
output = indices.calc_indice_simple(
resources,
indices=['SU'], groupings='year', dir_output=tempfile.mkdtemp())
assert os.path.basename(output[0]) == 'SU_EUR-44_MPI-M-MPI-ESM-LR_rcp45_r1i1p1_MPI-CSC-REMO2009_v1_mon_20060215-20061216.nc'
ds = Dataset(output[0])
# SU variable must be in result
assert 'SU' in ds.variables
# 1 year
assert len(ds.variables['time']) == 1
def test_indice_simple():
# SU expects tasmax
resources = [local_path(TESTDATA['cordex_tasmax_2006_nc'])]
output = indices.calc_indice_simple(
resources,
indice='SU', grouping='yr', dir_output=tempfile.mkdtemp())
assert os.path.basename(output[0]) == 'SU_EUR-44_MPI-M-MPI-ESM-LR_rcp45_r1i1p1_MPI-CSC-REMO2009_v1_mon_200602-200612.nc' # noqa
ds = Dataset(output[0])
# SU variable must be in result
assert 'SU' in ds.variables
# 1 year
assert len(ds.variables['time']) == 1
def execute(self):
import os
import tarfile
from tempfile import mkstemp
from os import path
ncs = self.getInputValues(identifier='resource')
indices = self.indices.getValue()
polygons = self.polygons.getValue()
mosaik = self.mosaik.getValue()
groupings = self.groupings.getValue() # getInputValues(identifier='groupings')
polygons = self.polygons.getValue()
# if len(polygons)==0:
# polygons = None
self.status.set('starting: indices=%s, groupings=%s, countries=%s, num_files=%s' % (indices,
groupings, polygons, len(ncs)), 0)
results = calc_indice_simple(
resource = ncs,
mosaik=mosaik,
indices = indices,
polygons= polygons,
groupings = groupings,
dir_output = path.curdir,
)
if not results:
raise Exception("failed to produce results")
self.status.set('num results %s' % len(results), 90)
try:
(fp_tarf, tarf) = mkstemp(dir=".", suffix='.tar')
tar = tarfile.open(tarf, "w")
for result in results:
tar.add( result , arcname = os.path.basename(result))
tar.close()
logger.info('Tar file prepared')
except Exception as e:
msg = "Tar file preparation failed"
logger.exception(msg)
raise Exception(msg)
self.output.setValue( tarf )
self.status.set('done: indice=%s, num_files=%s' % (indices, len(ncs)), 100)
def get_indices(resources, indices):
"""
calculating indices (netCDF files) defined in _SDMINDICES_
:param resources:
:param indices: indices defined in _SDMINDICES_
:return list: list of filepathes to netCDF files
"""
from flyingpigeon.utils import sort_by_filename, calc_grouping, drs_filename, unrotate_pole
# from flyingpigeon.ocgis_module import call
from flyingpigeon.indices import indice_variable, calc_indice_simple
# names = [drs_filename(nc, skip_timestamp=False, skip_format=False,
# variable=None, rename_file=True, add_file_path=True) for nc in resources]
ncs = sort_by_filename(resources, historical_concatination=True)
ncs_indices = []
logger.info('resources sorted found %s datasets' % len(ncs.keys()))
for key in ncs.keys():
for indice in indices:
try:
name, month = indice.split('_')
variable = key.split('_')[0]
# print name, month , variable
if variable == indice_variable(name):
logger.info('calculating indice %s ' % indice)
prefix = key.replace(variable, name).replace('_day_', '_%s_' % month)
nc = calc_indice_simple(resource=ncs[key],
variable=variable,
polygons=['Europe', 'Africa', 'Asia', 'North America', 'Oceania',
'South America', 'Antarctica'],
mosaic=True,
prefix=prefix, indices=name, groupings=month)
if nc is not None:
coords = unrotate_pole(nc[0], write_to_file=True)
ncs_indices.append(nc[0])
logger.info('Successful calculated indice %s %s' % (key, indice))
else:
msg = 'failed to calculate indice %s %s' % (key, indice)
logger.exception(msg)
except:
msg = 'failed to calculate indice %s %s' % (key, indice)
logger.exception(msg)
raise
return ncs_indices
def get_indices(resources, indices):
from flyingpigeon.utils import sort_by_filename, calc_grouping, drs_filename
from flyingpigeon.ocgis_module import call
from flyingpigeon.indices import indice_variable, calc_indice_simple
#names = [drs_filename(nc, skip_timestamp=False, skip_format=False,
# variable=None, rename_file=True, add_file_path=True) for nc in resources]
ncs = sort_by_filename(resources, historical_concatination=True)
ncs_indices = []
logger.info('resources sorted found %s datasets' % len(ncs.keys()))
for key in ncs.keys():
for indice in indices:
try:
name, month = indice.split('_')
variable = key.split('_')[0]
# print name, month , variable
if variable == indice_variable(name):
logger.info('calculating indice %s ' % indice)
prefix = key.replace(variable,
name).replace('_day_', '_%s_' % month)
nc = calc_indice_simple(resource=ncs[key],
variable=variable,
prefix=prefix,
indices=name,
groupings=month,
memory_limit=500)
#grouping = calc_grouping(month)
#calc = [{'func' : 'icclim_' + name, 'name' : name}]
#nc = call(resource=ncs[key], variable=variable, calc=calc, calc_grouping=grouping, prefix=prefix , memory_limit=500) #memory_limit=500
ncs_indices.append(nc[0])
logger.info('Successful calculated indice %s %s' %
(key, indice))
except Exception as e:
logger.exception('failed to calculate indice %s %s' %
(key, indice))
return ncs_indices
def _handler(self, request, response):
init_process_logger('log.txt')
response.outputs['output_log'].file = 'log.txt'
try:
resources = archiveextract(
resource=rename_complexinputs(request.inputs['resource']))
indices = [inpt.data for inpt in request.inputs['indices']]
grouping = [inpt.data for inpt in request.inputs['grouping']]
if 'mosaic' in request.inputs:
mosaic = request.inputs['mosaic'][0].data
else:
mosaic = False
if 'region' in request.inputs:
region = [inpt.data for inpt in request.inputs['region']]
else:
region = None
LOGGER.debug('grouping: {}'.format(grouping))
LOGGER.debug('mosaic: {}'.format(mosaic))
LOGGER.debug('indices: {}'.format(indices))
LOGGER.debug('region: {}'.format(region))
LOGGER.debug('Nr of input files: {}'.format(len(resources)))
except Exception as ex:
LOGGER.exception('failed to read in the arguments: {}'.format(str(ex)))
response.update_status(
'starting: indices={}, grouping={}, num_files={}'.format(indices, grouping, len(resources)), 2)
results = []
from flyingpigeon.utils import sort_by_filename
datasets = sort_by_filename(resources, historical_concatination=True)
results = []
try:
group = grouping[0] # for group in grouping:
indice = indices[0] # for indice in indices:
for key in datasets.keys():
try:
response.update_status('Dataset {}: {}'.format(len(results) + 1, key), 10)
LOGGER.debug('grouping: {}'.format(grouping))
LOGGER.debug('mosaic: {}'.format(mosaic))
LOGGER.debug('indice: {}'.format(indice))
LOGGER.debug('region: {}'.format(region))
LOGGER.debug('Nr of input files: {}'.format(len(datasets[key])))
result = calc_indice_simple(
resource=datasets[key],
mosaic=mosaic,
indice=indice,
polygons=region,
grouping=group,
# dir_output=path.curdir,
)
LOGGER.debug('result: {}'.format(result))
results.extend(result)
except Exception as ex:
msg = 'failed for {}: {}'.format(key, str(ex))
LOGGER.exception(msg)
raise Exception(msg)
except Exception as ex:
msg = 'Failed to calculate indices: {}'.format(str(ex))
LOGGER.exception(msg)
raise Exception(msg)
# # if not results:
# # raise Exception("failed to produce results")
# # response.update_status('num results %s' % len(results), 90)
tarf = archive(results)
response.outputs['output_archive'].file = tarf
i = next((i for i, x in enumerate(results) if x), None)
if i is None:
i = 'dummy.nc'
response.outputs['ncout'].file = results[i]
# response.update_status("done", 100)
return response
output_format='nc').execute()
print geom
rd = RequestDataset(ncs)
rd.dimension_map.set_bounds(DimensionMapKey.TIME, None)
indice = 'TG'
geom = OcgOperations(rd,
calc=[{
'func': 'icclim_' + indice,
'name': indice
}],
calc_grouping=['year', 'month'],
prefix='multi_file',
output_format='nc').execute()
print geom
from flyingpigeon.indices import calc_indice_simple
fp_indice = calc_indice_simple(resource=ncs,
variable=None,
prefix=None,
indice='TG',
polygons=['CMR'],
mosaic=True,
grouping='yr',
dir_output=None,
dimension_map=None,
memory_limit=None)
print fp_indice
def get_indices(resource, indices):
"""
calculating indices (netCDF files) defined in _SDMINDICES_
:param resources: files containing one Dataset
:param indices: List of indices defined in _SDMINDICES_. Index needs to be based on the resource variable
:return list: list of filepathes to netCDF files
"""
from flyingpigeon.utils import sort_by_filename, calc_grouping, drs_filename, unrotate_pole, get_variable
# from flyingpigeon.ocgis_module import call
from flyingpigeon.indices import indice_variable, calc_indice_simple
from flyingpigeon.subset import masking
from flyingpigeon.utils import searchfile
from flyingpigeon.utils import search_landsea_mask_by_esgf
from os.path import basename
# names = [drs_filename(nc, skip_timestamp=False, skip_format=False,
# variable=None, rename_file=True, add_file_path=True) for nc in resources]
variable = get_variable(resource)
masked_datasets = []
max_count = len(resource)
for ds in resource:
ds_name = basename(ds)
LOGGER.debug('masking dataset: %s', ds_name)
try:
landsea_mask = search_landsea_mask_by_esgf(ds)
LOGGER.debug("using landsea_mask: %s", landsea_mask)
prefix = ds_name.replace('.nc', '')
new_ds = masking(ds, landsea_mask, land_area=True, prefix=prefix)
masked_datasets.append(new_ds)
except:
LOGGER.exception("Could not subset dataset.")
break
else:
LOGGER.info("masked: %d/%d", len(masked_datasets), max_count)
if not masked_datasets:
raise Exception("Could not mask input files.")
ncs = sort_by_filename(masked_datasets, historical_concatination=True)
key = ncs.keys()[0]
ncs_indices = []
LOGGER.info('resources sorted found %s datasets', len(ncs.keys()))
for indice in indices:
try:
name, month = indice.split('_')
# print name, month , variable
if variable == indice_variable(name):
LOGGER.info('calculating indice %s ' % indice)
prefix = key.replace(variable, name).replace('_day_', '_%s_' % month)
nc = calc_indice_simple(resource=resource,
variable=variable,
# polygons=['Europe', 'Africa', 'Asia', 'North America', 'Oceania',
# 'South America', 'Antarctica'],
# mosaic=True,
prefix=prefix, indice=name, grouping=month)
if nc is not None:
# coords = unrotate_pole(nc[0], write_to_file=True)
ncs_indices.append(nc[0])
LOGGER.info('Successful calculated indice %s %s' % (key, indice))
else:
msg = 'failed to calculate indice %s %s' % (key, indice)
LOGGER.exception(msg)
except:
msg = 'failed to calculate indice %s %s' % (key, indice)
LOGGER.exception(msg)
return ncs_indices
