def rat(self):
uuid = self.request.form.get('uuid')
layer = self.request.form.get('layer')
brain = None
try:
brain = uuidToCatalogBrain(uuid)
except Exception as e:
LOG.error('Caught exception %s', e)
if not brain:
self.record_error('Not Found', 404, 'dataset not found',
{'parameter': 'uuid'})
raise NotFound(self, 'metadata', self.request)
md = IBCCVLMetadata(brain.getObject())
if not layer and layer not in md.get('layers', {}):
self.record_error('Bad Request', 400, 'Missing parameter layer',
{'parameter': 'layer'})
raise BadRequest('Missing parameter layer')
try:
rat = md.get('layers', {}).get(layer, {}).get('rat')
rat = json.loads(unicode(rat))
return rat
except Exception as e:
LOG.warning(
"Couldn't decode Raster Attribute Table from metadata. %s: %s",
self.context, repr(e))
raise NotFound(self, 'rat', self.request)
def dataset_environmental_layer(obj, **kw):
md = IBCCVLMetadata(obj)
# if we have 'layers_used' index it
if 'layers_used' in md:
return md['layers_used']
# otherwise index list of layers provided by dataset
return md.get('layers', None)
def getdatasetparams(uuid):
# return dict with:
# filename
# downloadurl
# dm_accessurl-> maybe add url rewrite to datamover?
# # occurrence specific:
# species
# # raster specific:
# layers ... need to split this up
dsobj = uuidToObject(uuid)
if dsobj is None:
return None
dsinfo = getDatasetInfo(dsobj, uuid)
# if we have species info add it
dsmdr = IBCCVLMetadata(dsobj)
species = dsmdr.get('species', {}).get('scientificName')
if species:
dsinfo['species'] = species
# if we can get layermetadata, let's add it
biomod = getdsmetadata(dsobj)
layers = biomod.get('layers', [])
if len(layers) > 0:
for lk, lv in biomod['layers'].items():
if lv is not None:
dsinfo.setdefault('layers', {})[lk] = {
'filename': lv.get('filename', biomod['filename']),
'datatype': lv.get('datatype', None)
}
# return infoset
return dsinfo
def getdatasetparams(uuid):
# return dict with:
# filename
# downloadurl
# dm_accessurl-> maybe add url rewrite to datamover?
# # occurrence specific:
# species
# # raster specific:
# layers ... need to split this up
dsobj = uuidToObject(uuid)
if dsobj is None:
return None
dsinfo = getDatasetInfo(dsobj, uuid)
# if we have species info add it
dsmdr = IBCCVLMetadata(dsobj)
species = dsmdr.get('species', {}).get('scientificName')
if species:
dsinfo['species'] = species
# if we can get layermetadata, let's add it
biomod = getdsmetadata(dsobj)
layers = biomod.get('layers', [])
if len(layers) > 0:
for lk, lv in biomod['layers'].items():
if lv is not None:
dsinfo.setdefault('layers', {})[lk] = {
'filename': lv.get('filename', biomod['filename']),
'datatype': lv.get('datatype', None)}
# return infoset
return dsinfo
def items(self):
# return dict with keys for experiment
# and subkey 'models' for models within experiment
if self.value:
for experiment_uuid, model_uuids in self.value.items():
item = {}
expbrain = uuidToCatalogBrain(experiment_uuid)
item['title'] = expbrain.Title
item['uuid'] = expbrain.UID
# TODO: what else wolud I need from an experiment?
exp = expbrain.getObject()
expmd = IBCCVLMetadata(exp)
item['resolution'] = expmd.get('resolution')
item['brain'] = expbrain
# now search all models within and add infos
pc = getToolByName(self.context, 'portal_catalog')
brains = pc.searchResults(path=expbrain.getPath(),
BCCDataGenre=self.genre)
# TODO: maybe as generator?
item['datasets'] = [{
'uuid':
brain.UID,
'title':
brain.Title,
'obj':
brain.getObject(),
'md':
IBCCVLMetadata(brain.getObject()),
'selected':
brain.UID in self.value[experiment_uuid]
} for brain in brains]
yield item
def biodiverse_listing_details(expbrain):
details = {}
exp = expbrain.getObject()
species = set()
years = set()
emscs = set()
gcms = set()
for dsuuid in chain.from_iterable(map(lambda x: x.keys(), exp.projection.itervalues())):
dsobj = uuidToObject(dsuuid)
# TODO: should inform user about missing dataset
if dsobj:
md = IBCCVLMetadata(dsobj)
species.add(md.get("species", {}).get("scientificName", ""))
period = md.get("temporal")
if period:
years.add(Period(period).start)
gcm = md.get("gcm")
if gcm:
gcms.add(gcm)
emsc = md.get("emsc")
if emsc:
emscs.add(emsc)
details.update(
{
"type": "BIODIVERSE",
"functions": "endemism, redundancy",
"species_occurrence": ", ".join(sorted(species)),
"species_absence": "{}, {}".format(", ".join(sorted(emscs)), ", ".join(sorted(gcms))),
"years": ", ".join(sorted(years)),
}
)
return details
def details(self, context=None):
# fetch details about dataset, if attributes are unpopulated
# get data from associated collection
if context is None:
context = self.context
coll = context
while not (ISiteRoot.providedBy(coll) or ICollection.providedBy(coll)):
coll = coll.__parent__
# we have either hit siteroot or found a collection
ret = {
'title': context.title,
'description': context.description or coll.description,
'attribution': context.attribution or getattr(coll, 'attribution'),
'rights': context.rights or coll.rights,
'external_description': context.external_description or getattr(coll, 'external_description'),
}
md = IBCCVLMetadata(context)
if 'layers' in md:
layers = []
for layer in sorted(md.get('layers', ())):
try:
layers.append(self.layer_vocab.getTerm(layer))
except:
layers.append(SimpleTerm(layer, layer, layer))
if layers:
ret['layers'] = layers
return ret
def details(self, context=None):
# fetch details about dataset, if attributes are unpopulated
# get data from associated collection
if context is None:
context = self.context
coll = context
while not (ISiteRoot.providedBy(coll) or ICollection.providedBy(coll)):
coll = coll.__parent__
# we have either hit siteroot or found a collection
ret = {
'title':
context.title,
'description':
context.description or coll.description,
'attribution':
context.attribution or getattr(coll, 'attribution'),
'rights':
context.rights or coll.rights,
'external_description':
context.external_description
or getattr(coll, 'external_description'),
}
md = IBCCVLMetadata(context)
if 'layers' in md:
layers = []
for layer in sorted(md.get('layers', ())):
try:
layers.append(self.layer_vocab.getTerm(layer))
except:
layers.append(SimpleTerm(layer, layer, layer))
if layers:
ret['layers'] = layers
return ret
def rat(self):
uuid = self.request.form.get('uuid')
layer = self.request.form.get('layer')
brain = None
try:
brain = uuidToCatalogBrain(uuid)
except Exception as e:
LOG.error('Caught exception %s', e)
if not brain:
self.record_error('Not Found', 404,
'dataset not found',
{'parameter': 'uuid'})
raise NotFound(self, 'metadata', self.request)
md = IBCCVLMetadata(brain.getObject())
if not layer and layer not in md.get('layers', {}):
self.record_error('Bad Request', 400,
'Missing parameter layer',
{'parameter': 'layer'})
raise BadRequest('Missing parameter layer')
try:
rat = md.get('layers', {}).get(layer, {}).get('rat')
rat = json.loads(unicode(rat))
return rat
except Exception as e:
LOG.warning(
"Couldn't decode Raster Attribute Table from metadata. %s: %s",
self.context, repr(e))
raise NotFound(self, 'rat', self.request)
def items(self):
# return dict with keys for experiment
# and subkey 'models' for models within experiment
if self.value:
for experiment_uuid, model_uuids in self.value.items():
item = {}
expbrain = uuidToCatalogBrain(experiment_uuid)
item['title'] = expbrain.Title
item['uuid'] = expbrain.UID
# TODO: what else wolud I need from an experiment?
exp = expbrain.getObject()
expmd = IBCCVLMetadata(exp)
item['resolution'] = expmd.get('resolution')
item['brain'] = expbrain
# now search all models within and add infos
pc = getToolByName(self.context, 'portal_catalog')
brains = pc.searchResults(path=expbrain.getPath(),
BCCDataGenre=self.genre)
# TODO: maybe as generator?
item['datasets'] = [{'uuid': brain.UID,
'title': brain.Title,
'obj': brain.getObject(),
'md': IBCCVLMetadata(brain.getObject()),
'selected': brain.UID in self.value[experiment_uuid]}
for brain in brains]
yield item
def subitems(self, dsbrain):
# return a generator of selectable items within dataset
md = IBCCVLMetadata(dsbrain.getObject())
layer_vocab = self.dstools.layer_vocab
selectedsubitems = self.value.get(dsbrain.UID) or ()
if md.get('genre') != 'DataGenreSpeciesCollection':
for layer in sorted(md.get('layers', ())):
subitem = {
'id':
layer,
'title':
layer_vocab.getTerm(layer).title
if layer in layer_vocab else layer,
'selected':
layer in selectedsubitems,
}
yield subitem
for subdsid in sorted(getattr(dsbrain.getObject(), 'parts', ())):
part = uuidToCatalogBrain(subdsid)
# TODO: should we just ignore it?
if not part:
continue
subitem = {
'id': subdsid,
'title': part.Title,
'selected': subdsid in selectedsubitems
}
yield subitem
def addSpeciesInfo(bccvlmd, result):
if ISDMExperiment.providedBy(result.__parent__):
spds = uuidToObject(result.job_params['species_occurrence_dataset'])
if IProjectionExperiment.providedBy(result.__parent__):
spds = uuidToObject(result.job_params['species_distribution_models'])
speciesmd = IBCCVLMetadata(spds).get('species', None)
if speciesmd:
bccvlmd['species'] = speciesmd.copy()
def addSpeciesInfo(bccvlmd, result):
if ISDMExperiment.providedBy(result.__parent__):
spds = uuidToObject(result.job_params['species_occurrence_dataset'])
if IProjectionExperiment.providedBy(result.__parent__):
spds = uuidToObject(result.job_params['species_distribution_models'])
speciesmd = IBCCVLMetadata(spds).get('species', None)
if speciesmd:
bccvlmd['species'] = speciesmd.copy()
def year(obj, **kw):
# FIXME: this indexer is meant for future projection only ....
# - make sure we don't index any other datasets. i.e. environmental and current datasets, which may have a date attached to it, but it is meaningless for future projections
md = IBCCVLMetadata(obj)
year = md.get('year', None)
if year:
year = str(year)
return year
def test_filemetadata(self):
ds = self.get_dataset(defaults.DATASETS_SPECIES_FOLDER_ID,
'ABT', 'occurrence.csv')
from org.bccvl.site.interfaces import IBCCVLMetadata
md = IBCCVLMetadata(ds)
self.assertEqual(md.get('rows'), 3)
self.assertEqual(md.get('bounds'), {'bottom': 1, 'left': 1, 'top': 3, 'right': 3})
self.assertEqual(md.get('headers'), ['Name', 'lon', 'lat'])
self.assertIn('species', md) # check if species attribute exists
def test_filemetadata(self):
ds = self.get_dataset(defaults.DATASETS_SPECIES_FOLDER_ID,
'ABT', 'occurrence.csv')
md = IBCCVLMetadata(ds)
self.assertEqual(md.get('rows'), 3)
self.assertEqual(md.get('bounds'), {
'bottom': 1, 'left': 1, 'top': 3, 'right': 3})
self.assertEqual(md.get('headers'), ['Name', 'lon', 'lat'])
self.assertIn('species', md) # check if species attribute exists
def scientific_category(obj, **kw):
md = IBCCVLMetadata(obj)
vocab = getUtility(IVocabularyFactory, 'scientific_category_source')(obj)
path = set()
for cat in md.get('categories', ()):
path.update(vocab.getTermPath(cat))
if path:
return tuple(path)
return None
def items(self):
# return dict with keys for experiment
# and subkey 'models' for models within experiment
if self.value:
for experiment_uuid, model_uuids in self.value.items():
item = {}
expbrain = uuidToCatalogBrain(experiment_uuid)
# TODO: we have an experiment_uuid, but can't access the
# experiment (deleted?, access denied?)
# shall we at least try to get some details?
if expbrain is None:
continue
item['title'] = expbrain.Title
item['uuid'] = expbrain.UID
item['brain'] = expbrain
# TODO: what else wolud I need from an experiment?
exp = expbrain.getObject()
expmd = IBCCVLMetadata(exp)
item['resolution'] = expmd.get('resolution')
# now search all datasets within and add infos
pc = getToolByName(self.context, 'portal_catalog')
results = pc.searchResults(path=expbrain.getPath(),
portal_type='Folder',
job_state='COMPLETED')
brains = pc.searchResults(path=[r.getPath() for r in results],
BCCDataGenre=self.genre)
# TODO: maybe as generator?
item['subitems'] = []
for brain in brains:
# FIXME: I need a different list of thresholds for display;
# esp. don't look up threshold, but take vales (threshold
# id and value) from field as is
thresholds = dataset.getThresholds(brain.UID)[brain.UID]
threshold = self.value[experiment_uuid].get(brain.UID)
# is threshold in list?
if threshold and threshold['label'] not in thresholds:
# maybe a custom entered number?
# ... I guess we don't really care as long as we produce the same the user entered. (validate?)
thresholds[threshold['label']] = threshold['label']
dsobj = brain.getObject()
dsmd = IBCCVLMetadata(dsobj)
item['subitems'].append({
'uuid': brain.UID,
'title': brain.Title,
'selected': brain.UID in self.value[experiment_uuid],
'threshold': threshold,
'thresholds': thresholds,
'brain': brain,
'md': dsmd,
'obj': dsobj,
# TODO: this correct? only one layer ever?
'layermd': dsmd['layers'].values()[0]
})
yield item
def items(self):
# return dict with keys for experiment
# and subkey 'models' for models within experiment
if self.value:
for experiment_uuid, model_uuids in self.value.items():
item = {}
expbrain = uuidToCatalogBrain(experiment_uuid)
item['title'] = expbrain.Title
item['uuid'] = expbrain.UID
item['brain'] = expbrain
# TODO: what else wolud I need from an experiment?
exp = expbrain.getObject()
expmd = IBCCVLMetadata(exp)
item['resolution'] = expmd.get('resolution')
# now search all datasets within and add infos
pc = getToolByName(self.context, 'portal_catalog')
brains = pc.searchResults(path=expbrain.getPath(),
BCCDataGenre=self.genre)
# TODO: maybe as generator?
item['datasets'] = []
for brain in brains:
# FIXME: I need a different list of thresholds for display; esp. don't look up threshold, but take vales (threshold id and value) from field as is
thresholds = dataset.getThresholds(brain.UID)[brain.UID]
threshold = self.value[experiment_uuid].get(brain.UID)
# is threshold in list?
if threshold and threshold['label'] not in thresholds:
# maybe a custom entered number?
# ... I guess we don't really care as long as we produce the same the user entered. (validate?)
thresholds[threshold['label']] = threshold['label']
dsobj = brain.getObject()
dsmd = IBCCVLMetadata(dsobj)
item['datasets'].append({
'uuid':
brain.UID,
'title':
brain.Title,
'selected':
brain.UID in self.value[experiment_uuid],
'threshold':
threshold,
'thresholds':
thresholds,
'brain':
brain,
'md':
dsmd,
'obj':
dsobj,
# TODO: this correct? only one layer ever?
'layermd':
dsmd['layers'].values()[0]
})
yield item
def getGenreSchemata(self):
schemata = []
md = IBCCVLMetadata(self.context)
genre = md.get('genre')
if genre in self.genre_interface_map:
schemata.append(self.genre_interface_map[genre])
if IBlobDataset.providedBy(self.context):
schemata.append(IBlobDataset)
if IRemoteDataset.providedBy(self.context):
schemata.append(IRemoteDataset)
return schemata
def getGenreSchemata(self):
schemata = []
md = IBCCVLMetadata(self.context)
genre = md.get('genre')
if genre in self.genre_interface_map:
schemata.append(self.genre_interface_map[genre])
if IBlobDataset.providedBy(self.context):
schemata.append(IBlobDataset)
if IRemoteDataset.providedBy(self.context):
schemata.append(IRemoteDataset)
return schemata
def subitems(self, dsbrain):
# return a generator of selectable items within dataset
md = IBCCVLMetadata(dsbrain.getObject())
layer_vocab = self.dstools.layer_vocab
selectedlayers = self.value.get(dsbrain.UID) or ()
for layer in sorted(md.get('layers', ())):
subitem = {
'id': layer,
'title': layer_vocab.getTerm(layer).title,
'selected': layer in selectedlayers,
}
yield subitem
def subitems(self, dsbrain):
# return a generator of selectable items within dataset
md = IBCCVLMetadata(dsbrain.getObject())
layer_vocab = self.dstools.layer_vocab
selectedlayers = self.value.get(dsbrain.UID) or ()
for layer in sorted(md.get('layers', ())):
subitem = {
'id': layer,
'title': layer_vocab.getTerm(layer).title,
'selected': layer in selectedlayers,
}
yield subitem
def get_project_params(result):
params = deepcopy(result.job_params)
# get metadata for species_distribution_models
uuid = params['species_distribution_models']
params['species_distribution_models'] = getdatasetparams(uuid)
# do biomod name mangling of species name
params['species_distribution_models']['species'] = re.sub(u"[ _'\"/\(\)\{\}\[\]]", u".", params['species_distribution_models'].get('species', u"Unknown"))
# we need the layers from sdm to fetch correct files for climate_models
# TODO: getdatasetparams should fetch 'layers'
sdmobj = uuidToObject(uuid)
sdmmd = IBCCVLMetadata(sdmobj)
params['species_distribution_models']['layers'] = sdmmd.get('layers_used', None)
# do future climate layers
climatelist = []
for uuid, layers in params['future_climate_datasets'].items():
dsinfo = getdatasetparams(uuid)
for layer in layers:
dsdata = {
'uuid': dsinfo['uuid'],
'filename': dsinfo['filename'],
'downloadurl': dsinfo['downloadurl'],
'layer': layer,
'zippath': dsinfo['layers'][layer]['filename'],
# TODO: add year, gcm, emsc here?
'type': dsinfo['layers'][layer]['datatype'],
}
# if this is a zip file we'll have to set zippath as well
# FIXME: poor check whether this is a zip file
if dsinfo['filename'].endswith('.zip'):
dsdata['zippath'] = dsinfo['layers'][layer]['filename']
climatelist.append(dsdata)
# replace climate_models parameter
params['future_climate_datasets'] = climatelist
params['selected_models'] = 'all'
# projection.name from dsinfo
# FIXME: workaround to get future projection name back, but this works only for file naming scheme with current available data
params['projection_name'], _ = os.path.splitext(dsinfo['filename'])
# TODO: quick fix Decimal json encoding through celery (where is my custom json encoder gone?)
for key, item in params.items():
if isinstance(item, Decimal):
params[key] = float(item)
# add hints for worker
workerhints = {
'files': ('species_distribution_models', 'future_climate_datasets')
}
return {'env': {}, 'params': params, 'worker': workerhints}
def items(self):
# return dict with keys for experiment
# and subkey 'models' for models within experiment
if self.value:
for experiment_uuid, model_uuids in self.value.items():
item = {}
expbrain = uuidToCatalogBrain(experiment_uuid)
item['title'] = expbrain.Title
item['uuid'] = expbrain.UID
# TODO: what else wolud I need from an experiment?
exp = expbrain.getObject()
expmd = IBCCVLMetadata(exp)
item['resolution'] = expmd.get('resolution')
item['brain'] = expbrain
# now search all models within and add infos
pc = getToolByName(self.context, 'portal_catalog')
brains = pc.searchResults(path=expbrain.getPath(),
BCCDataGenre=self.genre)
filtered_brains = []
for brain in brains:
# get algorithm term
algoid = getattr(brain.getObject(), 'job_params',
{}).get('function')
# Filter out geographic models
if algoid not in [
'circles', 'convhull', 'geoDist', 'geoIDW',
'voronoiHull'
]:
filtered_brains.append(brain)
brains = filtered_brains
# TODO: maybe as generator?
item['subitems'] = [{
'uuid':
brain.UID,
'title':
brain.Title,
'obj':
brain.getObject(),
'md':
IBCCVLMetadata(brain.getObject()),
'selected':
brain.UID in self.value[experiment_uuid]
} for brain in brains]
yield item
def test_mixed_resolution(self, mock_run_script):
future_1k_uuid = unicode(self.datasets[
defaults.DATASETS_CLIMATE_FOLDER_ID]['future_1k'].UID())
form = self.form.get_form()
form.request.form.update({
'form.buttons.save':
'Create and start',
# select 1k dataset as well
'form.widgets.future_climate_datasets': [future_1k_uuid],
})
form.update()
# setup mock_run_script
mock_run_script.side_effect = self.form.mock_run_script
# run experiment
transaction.commit()
exp = self.experiments['my-cc-experiment']
result = exp.values()[0]
expmd = IBCCVLMetadata(exp)
# We should have the missing layers filled by sdm env layer datasets
self.assertEqual(
result.job_params['future_climate_datasets'], {
future_1k_uuid: set([u'B01']),
self.form.sdmexp.environmental_datasets.keys()[0]: set(
[u'B02'])
})
# resolution should be set to the lowest of selected datasets
self.assertEqual(expmd['resolution'], 'Resolution2_5m')
def getdsmetadata(ds):
# TODO: support brain, obj and uuid string (URI as well?)
# extract info about files
if ICatalogBrain.providedBy(ds):
ds = ds.getObject()
# TODO: try to use brains only here
# url: ds.getURL()
# id: ds.UID,
# description: ds.Description
# start with metadata annotation
md = {
#'@context': { },
'@id': IUUID(ds),
'@type': ds.portal_type,
'url': ds.absolute_url(),
'id': IUUID(ds),
'title': ds.title,
'description': ds.description,
}
md.update(IBCCVLMetadata(ds))
dlinfo = IDownloadInfo(ds)
md.update({
'mimetype': dlinfo['contenttype'],
'filename': dlinfo['filename'],
'file': dlinfo['url'],
'vizurl': dlinfo['alturl'][0]
})
return md
def __iter__(self):
for item in self.previous:
pathkey = self.pathkey(*item.keys())[0]
# no path .. can't do anything
if not pathkey:
yield item
continue
path = item[pathkey]
# Skip the Plone site object itself
if not path:
yield item
continue
obj = self.context.unrestrictedTraverse(path.encode().lstrip('/'),
None)
# path doesn't exist
if obj is None:
yield item
continue
bccvlmd = item.get(self.bccvlmdkey)
if not bccvlmd:
yield item
continue
# apply bccvl metadata
# FIXME: replace or update?
IBCCVLMetadata(obj).update(bccvlmd)
yield item
def __iter__(self):
# exhaust previous iterator
for item in self.previous:
yield item
filename = self.context.file.filename
item = {
self.pathkey: '/'.join(self.context.getPhysicalPath()),
'_type': self.context.portal_type,
'file': {
'file': filename,
},
# TODO: consider deepcopy here (for now it's safe because all are normal dicts; no persistent dicts)
'bccvlmetadata': dict(IBCCVLMetadata(self.context)),
'_files': {
filename: {
# FIXME: there is some chaos here... do I really need name and filename?
'name': self.context.file.filename,
'filename': self.context.file.filename,
'contenttype': self.context.file.contentType,
# data is a readable file like object
# it may be an uncommitted blob file
'data': self.context.file.open('r')
}
}
}
yield item
def validateAction(self, data):
# TODO: check data ...
# ...
datasets = data.get('projection', {})
if not tuple(chain.from_iterable(x for x in datasets.values())):
raise WidgetActionExecutionError(
'projection', Invalid('No projection dataset selected.'))
# check if threshold values are in range
for dataset in (x for x in datasets.values()):
if not dataset:
raise WidgetActionExecutionError(
'projection',
Invalid(
'Please select at least one dataset within experiment')
)
# key: {label, value}
dsuuid = dataset.keys()[0]
ds = uuidToObject(dsuuid)
value = dataset[dsuuid]['value']
md = IBCCVLMetadata(ds)
# ds should be a projection output which has only one layer
# FIXME: error message is not clear enough and
# use widget.errors instead of exception
# also it will only verify if dateset has min/max values in
# metadata
layermd = md['layers'].values()[0]
if 'min' in layermd and 'max' in layermd:
# FIXME: at least layermd['min'] may be a string '0', when
# comparing to decimal from threshold selector, this comparison
# fails and raises the widget validation error
if value <= float(layermd['min']) or value >= float(
layermd['max']):
raise WidgetActionExecutionError(
'projection',
Invalid('Selected threshold is out of range'))
def setUp(self, mock_run_script):
self.portal = self.layer['portal']
self.experiments = self.portal[defaults.EXPERIMENTS_FOLDER_ID]
# create and run sdm experiment
formhelper = SDMExperimentHelper(self.portal)
sdmform = formhelper.get_form()
sdmform.request.form.update({
'form.buttons.save': 'Create and start',
})
# update form with updated request
sdmform.update()
# setup mock_run_script
mock_run_script.side_effect = formhelper.mock_run_script
# We should have only one SDM
sdmexp = self.experiments.values()[0]
transaction.commit()
# setup som threshold values our projection
sdmproj = sdmexp.values()[0]['proj_test.tif']
md = IBCCVLMetadata(sdmproj)
# there is only one layer
layermd = md['layers'].values()[0]
layermd['min'] = 0.0
layermd['max'] = 1.0
transaction.commit()
self.form = BiodiverseExperimentHelper(self.portal, sdmexp)
def item(self):
# return dict with keys for experiment
# and subkey 'models' for models within experiment
item = {}
if self.value:
experiment_uuid = self.value.keys()[0]
expbrain = uuidToCatalogBrain(experiment_uuid)
item['title'] = expbrain.Title
item['uuid'] = expbrain.UID
exp = expbrain.getObject()
item['layers'] = set((chain(*exp.environmental_datasets.values())))
expmd = IBCCVLMetadata(exp)
item['resolution'] = expmd['resolution']
# now search all models within and add infos
pc = getToolByName(self.context, 'portal_catalog')
brains = pc.searchResults(path=expbrain.getPath(),
BCCDataGenre=self.genre)
# TODO: maybe as generator?
item['models'] = [{
'item':
brain,
'uuid':
brain.UID,
'title':
brain.Title,
'selected':
brain.UID in self.value[experiment_uuid]
} for brain in brains]
return item
def get_project_params(result):
params = deepcopy(result.job_params)
# get metadata for species_distribution_models
uuid = params['species_distribution_models']
params['species_distribution_models'] = getdatasetparams(uuid)
# do biomod name mangling of species name
params['species_distribution_models']['species'] = re.sub(
u"[ _]", u".",
params['species_distribution_models'].get('species', u"Unknown"))
# we need the layers from sdm to fetch correct files for climate_models
# TODO: getdatasetparams should fetch 'layers'
sdmobj = uuidToObject(uuid)
sdmmd = IBCCVLMetadata(sdmobj)
params['species_distribution_models']['layers'] = sdmmd.get(
'layers_used', None)
# do future climate layers
climatelist = []
for uuid, layers in params['future_climate_datasets'].items():
dsinfo = getdatasetparams(uuid)
for layer in layers:
dsdata = {
'uuid': dsinfo['uuid'],
'filename': dsinfo['filename'],
'downloadurl': dsinfo['downloadurl'],
'internalurl': dsinfo['internalurl'],
'layer': layer,
'zippath': dsinfo['layers'][layer]['filename'],
# TODO: add year, gcm, emsc here?
'type': dsinfo['layers'][layer]['datatype'],
}
# if this is a zip file we'll have to set zippath as well
# FIXME: poor check whether this is a zip file
if dsinfo['filename'].endswith('.zip'):
dsdata['zippath'] = dsinfo['layers'][layer]['filename']
climatelist.append(dsdata)
# replace climate_models parameter
params['future_climate_datasets'] = climatelist
params['selected_models'] = 'all'
# projection.name from dsinfo
# FIXME: workaround to get future projection name back, but this works only for file naming scheme with current available data
params['projection_name'], _ = os.path.splitext(dsinfo['filename'])
# add hints for worker
workerhints = {
'files': ('species_distribution_models', 'future_climate_datasets')
}
return {'env': {}, 'params': params, 'worker': workerhints}
def test_upload_zip(self):
# upload a zip in bccvl bagit format
view = self.getview()
from ZPublisher.HTTPRequest import FileUpload
from cgi import FieldStorage
from StringIO import StringIO
data = resource_string(__name__, 'spc_obl_merc.zip')
env = {'REQUEST_METHOD': 'PUT'}
headers = {'content-type': 'text/csv',
'content-length': str(len(data)),
'content-disposition': 'attachment; filename=spc_obl_merc.zip'}
fileupload = FileUpload(FieldStorage(fp=StringIO(data),
environ=env, headers=headers))
view.request.form.update({
'climatefuture.buttons.save': u'Save',
'climatefuture.widgets.description': u'some test.tif file',
'climatefuture.widgets.file': fileupload,
'climatefuture.widgets.title': u'test smulti layer title',
'climatefuture.widgets.legalcheckbox': [u'selected'],
'climatefuture.widgets.legalcheckbox-empty-marker': u'1',
'climatefuture.widgets.rightsstatement': u'test rights',
'climatefuture.widgets.rightsstatement.mimeType': u'text/html',
'climatefuture.widgets.emsc': u'SRESB2',
'climatefuture.widgets.gcm': u'cccma-cgcm31',
'climatefuture.widgets.resolution': u'Resolution5m',
'climatefuture.widgets.temporal': u'2015',
})
_ = view()
self.assertEqual(self.portal.REQUEST.response.status, 302)
self.assertEqual(self.portal.REQUEST.response.getHeader('Location'),
'http://nohost/plone/datasets')
ds = self.portal.datasets['spc_obl_merc.zip']
self.assertEqual(ds.rightsstatement.raw, u'test rights')
self.assertEqual(ds.file.data, data)
from org.bccvl.site.interfaces import IBCCVLMetadata
md = IBCCVLMetadata(ds)
self.assertEqual(md['genre'], 'DataGenreFC')
self.assertEqual(md['resolution'], u'Resolution5m')
self.assertEqual(md['temporal'], u'2015')
self.assertEqual(md['emsc'], u'SRESB2')
self.assertEqual(md['gcm'], u'cccma-cgcm31')
layermd = md['layers']['spc_obl_merc/data/spc_obl_merc_1.tif']
self.assertEqual(layermd['filename'], 'spc_obl_merc/data/spc_obl_merc_1.tif')
self.assertEqual(layermd['min'], 19.0)
self.assertEqual(layermd['max'], 128.0)
self.assertEqual(layermd['datatype'], 'continuous')
self.assertEqual(layermd['height'], 200)
self.assertEqual(layermd['width'], 200)
self.assertEqual(layermd['srs'], None)
layermd = md['layers']['spc_obl_merc/data/spc_obl_merc_2.tif']
self.assertEqual(layermd['filename'], 'spc_obl_merc/data/spc_obl_merc_2.tif')
self.assertEqual(layermd['min'], 19.0)
self.assertEqual(layermd['max'], 128.0)
self.assertEqual(layermd['datatype'], 'continuous')
self.assertEqual(layermd['height'], 200)
self.assertEqual(layermd['width'], 200)
self.assertEqual(layermd['srs'], None)
def getRAT(self, datasetid, layer=None):
query = {'UID': datasetid}
dsbrain = dataset.query(self.context, brains=True, **query)
if dsbrain:
# get first brain from list
dsbrain = next(dsbrain, None)
if not dsbrain:
raise NotFound(self.context, datasetid, self.request)
md = IBCCVLMetadata(dsbrain.getObject())
rat = md.get('layers', {}).get(layer, {}).get('rat')
# if we have a rat, let's try and parse it
if rat:
try:
rat = json.loads(unicode(rat))
except Exception as e:
LOG.warning("Couldn't decode Raster Attribute Table from metadata. %s: %s", self.context, repr(e))
rat = None
return rat
def find_projections(ctx, emission_scenarios, climate_models, years, resolution=None):
"""Find Projection datasets for given criteria"""
pc = getToolByName(ctx, 'portal_catalog')
result = []
params = {
'BCCEmissionScenario': emission_scenarios,
'BCCGlobalClimateModel': climate_models,
'BCCDataGenre': 'DataGenreFC'
}
if resolution:
params['BCCResolution'] = resolution
brains = pc.searchResults(**params)
for brain in brains:
md = IBCCVLMetadata(brain.getObject())
year = md.get('temporal', None)
if year in years:
# TODO: yield?
result.append(brain)
return result
def get_project_params(result):
params = deepcopy(result.job_params)
# get metadata for species_distribution_models
uuid = params["species_distribution_models"]
params["species_distribution_models"] = getdatasetparams(uuid)
# do biomod name mangling of species name
params["species_distribution_models"]["species"] = re.sub(
u"[ _'\"/\(\)\{\}\[\]]", u".", params["species_distribution_models"].get("species", u"Unknown")
)
# we need the layers from sdm to fetch correct files for climate_models
# TODO: getdatasetparams should fetch 'layers'
sdmobj = uuidToObject(uuid)
sdmmd = IBCCVLMetadata(sdmobj)
params["species_distribution_models"]["layers"] = sdmmd.get("layers_used", None)
# do future climate layers
climatelist = []
for uuid, layers in params["future_climate_datasets"].items():
dsinfo = getdatasetparams(uuid)
for layer in layers:
dsdata = {
"uuid": dsinfo["uuid"],
"filename": dsinfo["filename"],
"downloadurl": dsinfo["downloadurl"],
"internalurl": dsinfo["internalurl"],
"layer": layer,
"zippath": dsinfo["layers"][layer]["filename"],
# TODO: add year, gcm, emsc here?
"type": dsinfo["layers"][layer]["datatype"],
}
# if this is a zip file we'll have to set zippath as well
# FIXME: poor check whether this is a zip file
if dsinfo["filename"].endswith(".zip"):
dsdata["zippath"] = dsinfo["layers"][layer]["filename"]
climatelist.append(dsdata)
# replace climate_models parameter
params["future_climate_datasets"] = climatelist
params["selected_models"] = "all"
# projection.name from dsinfo
# FIXME: workaround to get future projection name back, but this works only for file naming scheme with current available data
params["projection_name"], _ = os.path.splitext(dsinfo["filename"])
# add hints for worker
workerhints = {"files": ("species_distribution_models", "future_climate_datasets")}
return {"env": {}, "params": params, "worker": workerhints}
def DatasetSearchableText(obj, **kw):
md = IBCCVLMetadata(obj)
entries = [safe_unicode(obj.id), safe_unicode(obj.title) or u"", safe_unicode(obj.description) or u""]
if "layers" in md:
layer_vocab = getUtility(IVocabularyFactory, "layer_source")(obj)
for key in md["layers"]:
if key not in layer_vocab:
continue
entries.append(safe_unicode(layer_vocab.getTerm(key).title) or u"")
if "species" in md:
entries.extend(
(
safe_unicode(md.get("species", {}).get("scientificName")) or u"",
safe_unicode(md.get("species", {}).get("vernacularName")) or u"",
)
)
if md.get("genre") == "DataGenreFC":
# year, gcm, emsc
emsc_vocab = getUtility(IVocabularyFactory, "emsc_source")(obj)
gcm_vocab = getUtility(IVocabularyFactory, "gcm_source")(obj)
year = unicode(md.get("year", u""))
month = unicode(md.get("month", u""))
if md["emsc"] in emsc_vocab:
entries.append(safe_unicode(emsc_vocab.getTerm(md["emsc"]).title) or u"")
if md["gcm"] in gcm_vocab:
entries.append(safe_unicode(gcm_vocab.getTerm(md["gcm"]).title) or u"")
entries.append(year)
entries.append(month)
elif md.get("genre") == "DataGenreCC":
entries.append(u"current")
return u" ".join(entries)
def items(self):
# return dict with keys for experiment
# and subkey 'models' for models within experiment
if self.value:
for experiment_uuid, model_uuids in self.value.items():
item = {}
expbrain = uuidToCatalogBrain(experiment_uuid)
item['title'] = expbrain.Title
item['uuid'] = expbrain.UID
# TODO: what else wolud I need from an experiment?
exp = expbrain.getObject()
expmd = IBCCVLMetadata(exp)
item['resolution'] = expmd.get('resolution')
item['brain'] = expbrain
# now search all models within and add infos
pc = getToolByName(self.context, 'portal_catalog')
brains = pc.searchResults(path=expbrain.getPath(),
BCCDataGenre=self.genre)
filtered_brains = []
for brain in brains:
# get algorithm term
algoid = getattr(brain.getObject(), 'job_params',
{}).get('function')
# Filter out geographic models
if algoid not in ['circles', 'convhull', 'geoDist', 'geoIDW', 'voronoiHull']:
filtered_brains.append(brain)
brains = filtered_brains
# TODO: maybe as generator?
item['subitems'] = [{'uuid': brain.UID,
'title': brain.Title,
'obj': brain.getObject(),
'md': IBCCVLMetadata(brain.getObject()),
'selected': brain.UID in self.value[experiment_uuid]}
for brain in brains]
yield item
def subitems(self, dsbrain):
# return a generator of selectable items within dataset
md = IBCCVLMetadata(dsbrain.getObject())
layer_vocab = self.dstools.layer_vocab
selectedsubitems = self.value.get(dsbrain.UID) or ()
for layer in sorted(md.get('layers', ())):
subitem = {
'id': layer,
'title': layer_vocab.getTerm(layer).title if layer in layer_vocab else layer,
'selected': not selectedsubitems or layer in selectedsubitems,
}
yield subitem
for subdsid in sorted(getattr(dsbrain.getObject(), 'parts', ())):
part = uuidToCatalogBrain(subdsid)
# TODO: should we just ignore it?
if not part:
continue
subitem = {
'id': subdsid,
'title': part.Title,
'selected': not selectedsubitems or subdsid in selectedsubitems
}
yield subitem
def create(self, data):
# Dexterity base AddForm bypasses self.applyData and uses form.applyData directly,
# we'll have to override it to find a place to apply our algo_group data'
newob = super(SDMAdd, self).create(data)
# apply values to algo dict manually to make sure we don't write data on read
new_params = {}
for group in self.param_groups:
if group.toolkit in data['functions']:
content = group.getContent()
applyChanges(group, content, data)
new_params[group.toolkit] = content
newob.parameters = new_params
IBCCVLMetadata(newob)['resolution'] = data['resolution']
return newob
def rat(self):
uuid = self.request.form.get("uuid")
layer = self.request.form.get("layer")
brain = None
try:
brain = uuidToCatalogBrain(uuid)
except Exception as e:
LOG.error("Caught exception %s", e)
if not brain:
self.record_error("Not Found", 404, "dataset not found", {"parameter": "uuid"})
raise NotFound(self, "metadata", self.request)
md = IBCCVLMetadata(brain.getObject())
if not layer and layer not in md.get("layers", {}):
self.record_error("Bad Request", 400, "Missing parameter layer", {"parameter": "layer"})
raise BadRequest("Missing parameter layer")
try:
rat = md.get("layers", {}).get(layer, {}).get("rat")
rat = json.loads(unicode(rat))
return rat
except Exception as e:
LOG.warning("Couldn't decode Raster Attribute Table from metadata. %s: %s", self.context, repr(e))
raise NotFound(self, "rat", self.request)
def DatasetSearchableText(obj, **kw):
md = IBCCVLMetadata(obj)
entries = [
safe_unicode(obj.id),
safe_unicode(obj.title) or u"",
safe_unicode(obj.description) or u""
]
if 'layers' in md:
layer_vocab = getUtility(IVocabularyFactory, 'layer_source')(obj)
for key in md['layers']:
if key not in layer_vocab:
continue
entries.append(
safe_unicode(layer_vocab.getTerm(key).title) or u""
)
if 'species' in md:
entries.extend((
safe_unicode(md.get('species', {}).get('scientificName')) or u"",
safe_unicode(md.get('species', {}).get('vernacularName')) or u"",
))
if md.get('genre') == "DataGenreFC":
# year, gcm, emsc
emsc_vocab = getUtility(IVocabularyFactory, 'emsc_source')(obj)
gcm_vocab = getUtility(IVocabularyFactory, 'gcm_source')(obj)
year = Period(md.get('period','')).start
if md['emsc'] in emsc_vocab:
entries.append(
safe_unicode(emsc_vocab.getTerm(md['emsc']).title) or u""
)
if md['gcm'] in gcm_vocab:
entries.append(
safe_unicode(gcm_vocab.getTerm(md['gcm']).title) or u""
)
entries.append(safe_unicode(year) or u"")
elif md.get('genre') == "DataGenreCC":
entries.append(u"current")
return u" ".join(entries)
def getThresholds(datasets, thresholds=None):
# dataset to get thresholds for
# thresholds a list of threshold names to return (if None return all)
if not isinstance(datasets, list):
datasets = [datasets]
result = {} # we have to return per experiment, per dataset/result
for dataset in datasets:
dataobj = uuidToObject(dataset)
if dataobj is None:
continue
datamd = IBCCVLMetadata(dataobj)
if datamd['genre'] in ('DataGenreFP', 'DataGenreFP_ENVLOP'):
# we have a future projection ... go look for thresholds at SDM result
sdmuuid = dataobj.__parent__.job_params['species_distribution_models']
# get sdm result container
sdmresult = uuidToObject(sdmuuid).__parent__
elif datamd['genre'] in ['DataGenreCP', 'DataGenreCP_ENVLOP', 'DataGenreSDMModel']:
# we have a current projection ...
sdmresult = dataobj.__parent__
else:
continue
# We have the sdm result container ... find thresholds now
pc = getToolByName(dataobj, 'portal_catalog')
# find all model eval datasets
thresholds = {}
for evalbrain in pc.searchResults(path='/'.join(sdmresult.getPhysicalPath()),
BCCDataGenre='DataGenreSDMEval'):
evalmd = IBCCVLMetadata(evalbrain.getObject())
# FIXME: ideally we got only datasets with thresholds back here, but
# at the moment DataGenreSDMEval is aso used for graphs (png files)
# generated by the algorithms
if 'thresholds' not in evalmd:
continue
# TODO: merging of thresholds is random here
thresholds.update(evalmd['thresholds'])
result[dataset] = thresholds
return result
def start_job(self, request):
# split sdm jobs across multiple algorithms,
# and multiple species input datasets
# TODO: rethink and maybe split jobs based on enviro input datasets?
if not self.is_active():
for func in (uuidToObject(f) for f in self.context.functions):
# get utility to execute this experiment
method = queryUtility(IComputeMethod,
name=ISDMExperiment.__identifier__)
if method is None:
return ('error',
u"Can't find method to run SDM Experiment")
# create result object:
# TODO: refactor this out into helper method
title = u'{} - {} {}'.format(
self.context.title, func.getId(),
datetime.now().strftime('%Y-%m-%dT%H:%M:%S'))
result = self._create_result_container(title)
# Build job_params store them on result and submit job
result.job_params = {
'resolution': IBCCVLMetadata(self.context)['resolution'],
'function': func.getId(),
'species_occurrence_dataset':
self.context.species_occurrence_dataset,
'species_absence_dataset':
self.context.species_absence_dataset,
'species_pseudo_absence_points':
self.context.species_pseudo_absence_points,
'species_number_pseudo_absence_points':
self.context.species_number_pseudo_absence_points,
'environmental_datasets':
self.context.environmental_datasets,
}
# add toolkit params:
result.job_params.update(self.context.parameters[IUUID(func)])
self._createProvenance(result)
# submit job
LOG.info("Submit JOB %s to queue", func.getId())
method(result, func)
resultjt = IJobTracker(result)
resultjt.new_job('TODO: generate id',
'generate taskname: sdm_experiment')
resultjt.set_progress('PENDING',
u'{} pending'.format(func.getId()))
return 'info', u'Job submitted {0} - {1}'.format(
self.context.title, self.state)
else:
return 'error', u'Current Job is still running'
def build_traits_import_task(dataset, request):
# creates task chain to import ala dataset
"""
context ... a dictionary with keys:
- context: path to context object
- userid: zope userid
"""
# we need site-path, context-path and lsid for this job
dataset_path = '/'.join(dataset.getPhysicalPath())
member = api.user.get_current()
context = {
'context': dataset_path,
'dataSource': dataset.dataSource,
'user': {
'id': member.getUserName(),
'email': member.getProperty('email'),
'fullname': member.getProperty('fullname')
}
}
results_dir = get_results_dir(dataset, request)
if dataset.dataSource == 'aekos':
md = IBCCVLMetadata(dataset)
return datamover.pull_traits_from_aekos.si(
traits=md['traits'],
species=[sp['scientificName'] for sp in md['species']],
envvars=md['environ'],
dest_url=results_dir,
context=context)
elif dataset.dataSource == 'zoatrack':
md = IBCCVLMetadata(dataset)
return datamover.pull_traits_from_zoatrack.si(
species=[sp['scientificName'] for sp in md['species']],
src_url=md['dataurl'],
dest_url=results_dir,
context=context)
def start_job(self, request):
if not self.is_active():
# get utility to execute this experiment
method = queryUtility(IComputeMethod,
name=IProjectionExperiment.__identifier__)
if method is None:
# TODO: lookup by script type (Perl, Python, etc...)
return ('error',
u"Can't find method to run Projection Experiment")
expuuid = self.context.species_distribution_models.keys()[0]
exp = uuidToObject(expuuid)
# TODO: what if two datasets provide the same layer?
# start a new job for each sdm and future dataset
for sdmuuid in self.context.species_distribution_models[expuuid]:
for dsuuid in self.context.future_climate_datasets:
dsbrain = uuidToCatalogBrain(dsuuid)
dsmd = IBCCVLMetadata(dsbrain.getObject())
futurelayers = set(dsmd['layers'].keys())
# match sdm exp layers with future dataset layers
projlayers = {}
for ds, dslayerset in exp.environmental_datasets.items():
# add matching layers
projlayers.setdefault(dsuuid, set()).update(
dslayerset.intersection(futurelayers))
# remove matching layers
projlayers[ds] = dslayerset - futurelayers
if not projlayers[ds]:
# remove if all layers replaced
del projlayers[ds]
# create result
result = self._create_result_container(
sdmuuid, dsbrain, projlayers)
# update provenance
self._createProvenance(result)
# submit job
LOG.info("Submit JOB project to queue")
method(result, "project") # TODO: wrong interface
resultjt = IJobTracker(result)
resultjt.new_job(
'TODO: generate id',
'generate taskname: projection experiment')
resultjt.set_progress('PENDING', u'projection pending')
return 'info', u'Job submitted {0} - {1}'.format(
self.context.title, self.state)
else:
# TODO: in case there is an error should we abort the transaction
# to cancel previously submitted jobs?
return 'error', u'Current Job is still running'
class DatasetMetadataAdapter(object):
"""
Gives z3c.form datamanagers attribute access to the metadata object.
This class takes care of properly updateing the underlying storage object.
"""
# There is a datamanager problem.
# The Datamanager is looked up for the context (which returns an AttributeField manager)
# but then the Datamanager adapts the context to this adapter and tries attribute
# access which fails.
def __init__(self, context):
self._data = IBCCVLMetadata(context)
def __getattr__(self, name):
ob = self._data
try:
if name in ('scientificName', 'taxonID', 'vernacularName'):
return ob['species'][name]
else:
return ob[name]
except:
raise AttributeError('Attribute %s not found' % name)
def __setattr__(self, name, value):
if name == '_data':
self.__dict__['_data'] = value
# shortcut here to not store _data in metadata dictionary
return
if name in ('scientificName', 'taxonID', 'vernacularName'):
# FIXME: need a new dict here?
ob = self._data.setdefault('species', {})
else:
ob = self._data
ob[name] = value
def __delattr__(self, name):
if name in ('scientificName', 'taxonID', 'vernacularName'):
# FIXME: update dict?
ob = self._data['species']
del ob[name]
if not ob:
del self._data['species']
else:
del self._data[name]
def DatasetSearchableText(obj, **kw):
md = IBCCVLMetadata(obj)
entries = [
safe_unicode(obj.id),
safe_unicode(obj.title) or u"",
safe_unicode(obj.description) or u""
]
if 'layers' in md:
layer_vocab = getUtility(IVocabularyFactory, 'layer_source')(obj)
for key in md['layers']:
if key not in layer_vocab:
continue
entries.append(
safe_unicode(layer_vocab.getTerm(key).title) or u""
)
if 'species' in md:
entries.extend((
safe_unicode(md.get('species', {}).get('scientificName')) or u"",
safe_unicode(md.get('species', {}).get('vernacularName')) or u"",
))
if "Future datasets" in obj.subject:
# year, gcm, emsc
emsc_vocab = getUtility(IVocabularyFactory, 'emsc_source')(obj)
gcm_vocab = getUtility(IVocabularyFactory, 'gcm_source')(obj)
year = unicode(md.get('year', u''))
month = unicode(md.get('month', u''))
if md.get('emsc') in emsc_vocab:
entries.append(
safe_unicode(emsc_vocab.getTerm(md['emsc']).title) or u""
)
if md.get('gcm') in gcm_vocab:
entries.append(
safe_unicode(gcm_vocab.getTerm(md['gcm']).title) or u""
)
entries.append(year)
entries.append(month)
elif "Current datasets" in obj.subject:
entries.append(u"current")
return u" ".join(entries)
def biodiverse_listing_details(expbrain):
details = {}
exp = expbrain.getObject()
species = set()
years = set()
months = set()
emscs = set()
gcms = set()
for dsuuid in chain.from_iterable(map(lambda x: x.keys(), exp.projection.itervalues())):
dsobj = uuidToObject(dsuuid)
# TODO: should inform user about missing dataset
if dsobj:
md = IBCCVLMetadata(dsobj)
species.add(md.get('species', {}).get('scientificName', u'(Unavailable)'))
year = md.get('year')
if year:
years.add(year)
month = md.get('month')
if month:
months.add(month)
gcm = md.get('gcm')
if gcm:
gcms.add(gcm)
emsc = md.get('emsc')
if emsc:
emscs.add(emsc)
details.update({
'type': 'BIODIVERSE',
'functions': 'endemism, redundancy',
'species_occurrence': ', '.join(sorted(species)),
'species_absence': '{}, {}'.format(', '.join(sorted(emscs)),
', '.join(sorted(gcms))),
'years': ', '.join(sorted(years)),
'months': ', '.join(sorted(months))
})
return details
def get_project_params(result):
params = deepcopy(result.job_params)
# get metadata for species_distribution_models
uuid = params['species_distribution_models']
params['species_distribution_models'] = getdatasetparams(uuid)
# do biomod name mangling of species name
params['species_distribution_models']['species'] = re.sub(u"[ _\-'\"/\(\)\{\}\[\]]", u".", params['species_distribution_models'].get('species', u"Unknown"))
# we need the layers from sdm to fetch correct files for climate_models
# TODO: getdatasetparams should fetch 'layers'
sdmobj = uuidToObject(uuid)
sdmmd = IBCCVLMetadata(sdmobj)
params['species_distribution_models']['layers'] = sdmmd.get('layers_used', None)
# do SDM projection results
sdm_projections = []
for resuuid in params['sdm_projections']:
sdm_projections.append(getdatasetparams(resuuid))
params['sdm_projections'] = sdm_projections
# do future climate layers
climatelist = []
for uuid, layers in params['future_climate_datasets'].items():
dsinfo = getdatasetparams(uuid)
for layer in layers:
dsdata = {
'uuid': dsinfo['uuid'],
'filename': dsinfo['filename'],
'downloadurl': dsinfo['downloadurl'],
'layer': layer,
# TODO: add year, gcm, emsc here?
'type': dsinfo['layers'][layer]['datatype'],
}
# if this is a zip file we'll have to set zippath as well
# FIXME: poor check whether this is a zip file
if dsinfo['filename'].endswith('.zip'):
dsdata['zippath'] = dsinfo['layers'][layer]['filename']
# FIXME: workaround to get future projection name back, but this works only for file naming scheme with current available data
if params['selected_future_layers'] and layer in params['selected_future_layers']:
params['projection_name'], _ = os.path.splitext(dsinfo['filename'])
climatelist.append(dsdata)
# replace climate_models parameter
params['future_climate_datasets'] = climatelist
params['selected_models'] = 'all'
# In case no future climate layer is selected
if not params.get('projection_name'):
params['projection_name'], _ = os.path.splitext(dsinfo['filename'])
# TODO: quick fix Decimal json encoding through celery (where is my custom json encoder gone?)
for key, item in params.items():
if isinstance(item, Decimal):
params[key] = float(item)
# Get the content of the projection_region BlobFile.
# Note: deepcopy does not copy the content of BlobFile.
params['projection_region'] = {
'uuid': IUUID(result),
'filename': 'projection_region.json',
'downloadurl': '{0}/API/em/v1/constraintregion?uuid={1}'.format(getSite().absolute_url(), IUUID(result)),
}
# add hints for worker
workerhints = {
'files': ('species_distribution_models', 'future_climate_datasets', 'sdm_projections', 'projection_region',)
}
return {'env': {}, 'params': params, 'worker': workerhints}
def __init__(self, context):
self._data = IBCCVLMetadata(context)
def item(self):
# return dict with keys for experiment
# and subkey 'models' for models within experiment
item = {}
if self.value:
experiment_uuid = self.value.keys()[0]
expbrain = uuidToCatalogBrain(experiment_uuid)
if expbrain is None:
return {
'title': u'Not Available',
'uuid': experiment_uuid,
'subitems': [] # models
}
item['title'] = expbrain.Title
item['uuid'] = expbrain.UID
exp = expbrain.getObject()
# TODO: To get layers of all subsets?
if getattr(exp, 'datasubsets', None):
env_datasets = exp.datasubsets[0].get('environmental_datasets')
item['layers'] = set((chain(*env_datasets.values())))
else:
item['layers'] = set((chain(*exp.environmental_datasets.values())))
expmd = IBCCVLMetadata(exp)
item['resolution'] = expmd.get('resolution')
# now search all models within and add infos
pc = getToolByName(self.context, 'portal_catalog')
# only get result folders that are completed
brains = pc.searchResults(path={'query': expbrain.getPath(), 'depth': 1},
portal_type='Folder',
job_state='COMPLETED')
# TODO: maybe as generator?
item['subitems'] = []
for fldbrain in brains:
# Get the SDM model from result folder
brain = pc.searchResults(path=fldbrain.getPath(),
BCCDataGenre=self.genre)
if not brain:
# ignore this folder, as it does not have a result we want
continue
brain = brain[0]
# get algorithm term
algoid = getattr(brain.getObject(), 'job_params',
{}).get('function')
algobrain = self.algo_dict.get(algoid, None)
# Filter out geographic models
if algobrain.getObject().algorithm_category == 'geographic':
continue
# FIXME: I need a different list of thresholds for display;
# esp. don't look up threshold, but take vales (threshold
# id and value) from field as is
thresholds = dataset.getThresholds(brain.UID)[brain.UID]
threshold = self.value[experiment_uuid].get(brain.UID)
# is threshold in list?
if threshold and threshold['label'] not in thresholds:
# maybe a custom entered number?
# ... I guess we don't really care as long as we produce the same the user entered. (validate?)
thresholds[threshold['label']] = threshold['label']
# current projection tiff file, and its metadata
cpbrains = pc.searchResults(path=expbrain.getPath(),
BCCDataGenre=['DataGenreCP'])
cpmd = IBCCVLMetadata(cpbrains[0].getObject())
item['subitems'].append(
{'item': brain,
'uuid': brain.UID,
'title': brain.Title,
'selected': brain.UID in self.value[experiment_uuid],
'algorithm': algobrain,
'threshold': threshold,
'thresholds': thresholds,
'layermd': cpmd['layers'].values()[0]
}
)
return item
def month(obj, **kw):
# FIXME: see year indexer above
md = IBCCVLMetadata(obj)
return md.get('month', None)
def headers(obj, **kw):
md = IBCCVLMetadata(obj)
return md.get('headers', None)
def export_to_ala(self):
uuid = self.request.form.get("uuid", None)
try:
if uuid:
brain = uuidToCatalogBrain(uuid)
if brain is None:
raise Exception("Brain not found")
obj = brain.getObject()
else:
obj = self.context
# get username
member = ploneapi.user.get_current()
if member.getId():
user = {
"id": member.getUserName(),
"email": member.getProperty("email"),
"fullname": member.getProperty("fullname"),
}
else:
raise Exception("Invalid user")
# verify dataset
if obj.portal_type not in (
"org.bccvl.content.dataset",
"org.bccvl.content.remotedataset",
"org.bccvl.content.multispeciesdataset",
):
raise Exception("Invalid UUID (content type)")
md = IBCCVLMetadata(obj)
if md.get("genre") not in ("DataGenreSpeciesOccurrence", "DataGenreTraits"):
raise Exception("Invalid UUID (data type)")
# get download url
dlinfo = IDownloadInfo(obj)
# download file
from org.bccvl import movelib
from org.bccvl.movelib.utils import build_source, build_destination
import tempfile
destdir = tempfile.mkdtemp(prefix="export_to_ala")
try:
from org.bccvl.tasks.celery import app
settings = app.conf.get("bccvl", {})
dest = os.path.join(destdir, os.path.basename(dlinfo["url"]))
movelib.move(
build_source(dlinfo["url"], user["id"], settings), build_destination("file://{}".format(dest))
)
csvfile = None
if dlinfo["contenttype"] == "application/zip":
# loox at 'layers' to find file within zip
arc = md["layers"].keys()[0]
import zipfile
zf = zipfile.ZipFile(dest, "r")
csvfile = zf.open(arc, "r")
else:
csvfile = open(dest, "rb")
import requests
# "Accept:application/json" "Origin:http://example.com"
res = requests.post(
settings["ala"]["sandboxurl"],
files={"file": csvfile},
headers={"apikey": settings["ala"]["apikey"], "Accept": "application/json"},
)
if res.status_code != 200:
self.record_error(res.reason, res.status_code)
raise Exception("Upload failed")
retval = res.json()
# TODO: do error checking
# keys: sandboxUrl, fileName, message, error: Bool, fileId
return retval
finally:
import shutil
shutil.rmtree(destdir)
except Exception as e:
self.record_error(str(e), 500)
raise
