def items_old(self):
# FIXME importing here to avoid circular import of IDataset
from org.bccvl.site.api.dataset import getdsmetadata
if self.value:
for uuid in self.value:
brain = uuidToCatalogBrain(uuid)
# TODO: could use layer vocab again
md = getdsmetadata(brain)
layers = self.value[uuid]
# FIXME: check if layers or layers_used here
for layer, layeritem in md['layers'].iteritems():
if not layer in layers:
continue
mimetype = 'application/octet-stream'
layerfile = None
if 'filename' in layeritem:
# FIXME: hardcoded mimetype logic for zip files.
# should draw mimetype info from layer metadata
# assumes there are only geotiff in zip files
mimetype = 'image/geotiff'
layerfile = layeritem['filename']
vizurl = '{0}#{1}'.format(md['vizurl'], layerfile)
else:
vizurl = md['vizurl']
mimetype = md['mimetype']
yield {"brain": brain,
"resolution": self.dstools.resolution_vocab.getTerm(brain['BCCResolution']),
"layer": self.dstools.layer_vocab.getTerm(layer),
"vizurl": vizurl,
'mimetype': mimetype,
'vizlayer': layerfile}
def getdatasetparams(uuid):
# return dict with:
# filename
# downloadurl
# dm_accessurl-> maybe add url rewrite to datamover?
# # occurrence specific:
# species
# # raster specific:
# layers ... need to split this up
dsobj = uuidToObject(uuid)
if dsobj is None:
return None
dsinfo = getDatasetInfo(dsobj, uuid)
# if we have species info add it
dsmdr = IBCCVLMetadata(dsobj)
species = dsmdr.get('species', {}).get('scientificName')
if species:
dsinfo['species'] = species
# if we can get layermetadata, let's add it
biomod = getdsmetadata(dsobj)
layers = biomod.get('layers', [])
if len(layers) > 0:
for lk, lv in biomod['layers'].items():
if lv is not None:
dsinfo.setdefault('layers', {})[lk] = {
'filename': lv.get('filename', biomod['filename']),
'datatype': lv.get('datatype', None)
}
# return infoset
return dsinfo
def getdatasetparams(uuid):
# return dict with:
# filename
# downloadurl
# dm_accessurl-> maybe add url rewrite to datamover?
# # occurrence specific:
# species
# # raster specific:
# layers ... need to split this up
dsobj = uuidToObject(uuid)
if dsobj is None:
return None
dsinfo = getDatasetInfo(dsobj, uuid)
# if we have species info add it
dsmdr = IBCCVLMetadata(dsobj)
species = dsmdr.get('species', {}).get('scientificName')
if species:
dsinfo['species'] = species
# if we can get layermetadata, let's add it
biomod = getdsmetadata(dsobj)
layers = biomod.get('layers', [])
if len(layers) > 0:
for lk, lv in biomod['layers'].items():
if lv is not None:
dsinfo.setdefault('layers', {})[lk] = {
'filename': lv.get('filename', biomod['filename']),
'datatype': lv.get('datatype', None)}
# return infoset
return dsinfo
def metadata(self, itemobj=None, uuid=None):
if itemobj is None and uuid is None:
itemobj = self.context
if uuid:
itemobj = uuidToObject(uuid)
if itemobj is None:
return None
return getdsmetadata(itemobj)
def metadata(self, itemobj=None, uuid=None):
if itemobj is None and uuid is None:
itemobj = self.context
if uuid:
itemobj = uuidToObject(uuid)
if itemobj is None:
return None
return getdsmetadata(itemobj)
def __call__(self):
# 1. find all IBlobDataset/ IRemotedDataset/ IDataset objects within context
pc = getToolByName(self.context, 'portal_catalog')
brains = pc.searchResults(path='/'.join(self.context.getPhysicalPath()),
object_provides=IBlobDataset.__identifier__)
fname = None
try:
# create tmp file
fd, fname = tempfile.mkstemp()
fo = os.fdopen(fd, 'w')
zfile = zipfile.ZipFile(fo, 'w')
metadata = {}
# the file/folder name for the zip
zfilename = self.context.title
# iterate over files and add to zip
for brain in brains:
content = brain.getObject()
# ob.file should be a NamedFile ... need to get fs name for that
blobfile = content.file.openDetached()
arcname = '/'.join((zfilename, 'data', content.file.filename))
zfile.write(blobfile.name, arcname)
blobfile.close()
metadata[arcname] = getdsmetadata(content)
# all files are in ....
# TODO: add experiment result metadata
# put metadata into zip
# provenance data stored on result container
provdata = IProvenanceData(self.context)
if not provdata.data is None:
zfile.writestr('/'.join((zfilename, 'prov.ttl')),
provdata.data)
# add mets.xml
metsview = getMultiAdapter((self.context, self.request), name="mets.xml")
zfile.writestr('/'.join((zfilename, 'mets.xml')),
metsview.render())
# finish zip file
zfile.close()
fo.close()
# create response
self.request.response.setHeader('Content-Type', 'application/zip')
self.request.response.setHeader('Content-Disposition', 'attachment; filename="{}.zip"'.format(zfilename))
self.request.response.setHeader('Content-Length', '{}'.format(os.path.getsize(fname)))
return tmpfile_stream_iterator(fname)
except Exception as e:
# something went wrong ...
# clean up and re-raise
if os.path.exists(fname):
os.remove(fname)
raise e
def metadata(self):
uuid = self.request.form.get("uuid")
try:
brain = uuidToCatalogBrain(uuid)
if brain:
return dataset.getdsmetadata(brain)
except Exception as e:
LOG.error("Caught exception %s", e)
self.record_error("Not Found", "404", "dataset not found", {"parameter": "uuid"})
raise NotFound(self, "metadata", self.request)
def metadata(self):
uuid = self.request.form.get('uuid')
try:
brain = uuidToCatalogBrain(uuid)
if brain:
return dataset.getdsmetadata(brain)
except Exception as e:
LOG.error('Caught exception %s', e)
self.record_error('Not Found', '404', 'dataset not found',
{'parameter': 'uuid'})
raise NotFound(self, 'metadata', self.request)
def metadata(self):
uuid = self.request.form.get('uuid')
try:
brain = uuidToCatalogBrain(uuid)
if brain:
return dataset.getdsmetadata(brain)
except Exception as e:
LOG.error('Caught exception %s', e)
self.record_error('Not Found', '404', 'dataset not found', {
'parameter': 'uuid'})
raise NotFound(self, 'metadata', self.request)
def items_old(self):
# FIXME importing here to avoid circular import of IDataset
from org.bccvl.site.api.dataset import getdsmetadata
if self.value:
for uuid in self.value:
brain = uuidToCatalogBrain(uuid)
# TODO: could use layer vocab again
md = getdsmetadata(brain)
layers = self.value[uuid]
# FIXME: check if layers or layers_used here
for layer, layeritem in md['layers'].iteritems():
if not layer in layers:
continue
mimetype = 'application/octet-stream'
layerfile = None
if 'filename' in layeritem:
# FIXME: hardcoded mimetype logic for zip files.
# should draw mimetype info from layer metadata
# assumes there are only geotiff in zip files
mimetype = 'image/geotiff'
layerfile = layeritem['filename']
vizurl = '{0}#{1}'.format(md['vizurl'], layerfile)
else:
vizurl = md['vizurl']
mimetype = md['mimetype']
yield {
"brain":
brain,
"resolution":
self.dstools.resolution_vocab.getTerm(
brain['BCCResolution']),
"layer":
self.dstools.layer_vocab.getTerm(layer),
"vizurl":
vizurl,
'mimetype':
mimetype,
'vizlayer':
layerfile
}
def _download_results(self, context, zfile):
# FIXME: This is a rather lengthy process, and should probably be turned into a background task... (maybe as part of a datamanager service?)
# 1. find all IBlobDataset/ IRemotedDataset/ IDataset objects within context
pc = getToolByName(context, 'portal_catalog')
brains = pc.searchResults(path='/'.join(context.getPhysicalPath()),
object_provides=[
IBlobDataset.__identifier__,
IRemoteDataset.__identifier__
])
metadata = {}
# the file/folder name for the zip
zfilename = context.title
# iterate over files and add to zip
for brain in brains:
content = brain.getObject()
if IBlobDataset.providedBy(content):
# If data is stored locally:
arcname = '/'.join((zfilename, 'data', content.file.filename))
# ob.file should be a NamedFile ... need to get fs name for that
blobfile = content.file.openDetached()
zfile.write(blobfile.name, arcname)
blobfile.close()
elif IRemoteDataset.providedBy(content):
# TODO: duplicate code from
remoteUrl = getattr(content, 'remoteUrl', None)
if remoteUrl is None:
raise NotFound(self, 'remoteUrl', self.request)
# get arcname from remoteUrl
arcname = '/'.join(
(zfilename, 'data', os.path.basename(remoteUrl)))
# FIXME: should check dataset downloaiable flag here,
# but assumption is, that this function can only be called on an experiment result folder....
# TODO: duplicate code in browser/dataset.py:RemoteDatasetDownload.__call__
# TODO: may not work in general... it always uses swift as remote url
tool = getUtility(ISwiftUtility)
try:
url = tool.generate_temp_url(url=remoteUrl)
except:
url = remoteUrl
# url is now the location from which we can fetch the file
temp_file, _ = urlretrieve(url)
zfile.write(temp_file, arcname)
os.remove(temp_file)
else:
# unknown type of Dataset
# just skip it
# TODO: Log warning or debug?
continue
metadata[arcname] = getdsmetadata(content)
# all files are in ....
# TODO: add experiment result metadata
# put metadata into zip
# provenance data stored on result container
provdata = IProvenanceData(context)
if not provdata.data is None:
zfile.writestr('/'.join((zfilename, 'prov.ttl')),
provdata.data.encode('utf-8'))
# add experiment metadata
expmetadata = IExperimentMetadata(context)
if not expmetadata.data is None:
zfile.writestr('/'.join((zfilename, 'expmetadata.txt')),
expmetadata.data.encode('utf-8'))
# add mets.xml
metsview = getMultiAdapter((context, self.request), name="mets.xml")
zfile.writestr('/'.join((zfilename, 'mets.xml')),
metsview.render().encode('utf-8'))
# add experiment parameters
params = IExperimentParameter(context)
if not params.data is None:
zfile.writestr('/'.join((zfilename, 'params.json')),
params.data.encode('utf-8'))
def _download_results(self, context, zfile):
# FIXME: This is a rather lengthy process, and should probably be turned into a background task... (maybe as part of a datamanager service?)
# 1. find all IBlobDataset/ IRemotedDataset/ IDataset objects within context
pc = getToolByName(context, 'portal_catalog')
brains = pc.searchResults(path='/'.join(context.getPhysicalPath()),
object_provides=[IBlobDataset.__identifier__,
IRemoteDataset.__identifier__])
metadata = {}
# the file/folder name for the zip
zfilename = context.title
# iterate over files and add to zip
for brain in brains:
content = brain.getObject()
if IBlobDataset.providedBy(content):
# If data is stored locally:
arcname = '/'.join((zfilename, 'data', content.file.filename))
# ob.file should be a NamedFile ... need to get fs name for that
blobfile = content.file.openDetached()
zfile.write(blobfile.name, arcname)
blobfile.close()
elif IRemoteDataset.providedBy(content):
# TODO: duplicate code from
remoteUrl = getattr(content, 'remoteUrl', None)
if remoteUrl is None:
raise NotFound(self, 'remoteUrl', self.request)
# get arcname from remoteUrl
arcname = '/'.join((zfilename, 'data', os.path.basename(remoteUrl)))
# FIXME: should check dataset downloaiable flag here,
# but assumption is, that this function can only be called on an experiment result folder....
# TODO: duplicate code in browser/dataset.py:RemoteDatasetDownload.__call__
# TODO: may not work in general... it always uses swift as remote url
tool = getUtility(ISwiftUtility)
try:
url = tool.generate_temp_url(url=remoteUrl)
except:
url = remoteUrl
# url is now the location from which we can fetch the file
temp_file, _ = urlretrieve(url)
zfile.write(temp_file, arcname)
os.remove(temp_file)
else:
# unknown type of Dataset
# just skip it
# TODO: Log warning or debug?
continue
metadata[arcname] = getdsmetadata(content)
# all files are in ....
# TODO: add experiment result metadata
# put metadata into zip
# provenance data stored on result container
provdata = IProvenanceData(context)
if not provdata.data is None:
zfile.writestr('/'.join((zfilename, 'prov.ttl')),
provdata.data.encode('utf-8'))
# add experiment metadata
expmetadata = IExperimentMetadata(context)
if not expmetadata.data is None:
zfile.writestr('/'.join((zfilename, 'expmetadata.txt')),
expmetadata.data.encode('utf-8'))
# add mets.xml
metsview = getMultiAdapter((context, self.request), name="mets.xml")
zfile.writestr('/'.join((zfilename, 'mets.xml')),
metsview.render().encode('utf-8'))
# add experiment parameters
params = IExperimentParameter(context)
if not params.data is None:
zfile.writestr('/'.join((zfilename, 'params.json')),
params.data.encode('utf-8'))
def getMetadata(self):
return dataset.getdsmetadata(self.context)
def metadata(self, itemobj=None):
if itemobj is None:
itemobj = self.context
return getdsmetadata(itemobj)
def metadata(self, itemobj=None):
if itemobj is None:
itemobj = self.context
return getdsmetadata(itemobj)
def __call__(self):
# FIXME: This is a rather lengthy process, and should probably be turned into a background task... (maybe as part of a datamanager service?)
# 1. find all IBlobDataset/ IRemotedDataset/ IDataset objects within context
pc = getToolByName(self.context, 'portal_catalog')
brains = pc.searchResults(path='/'.join(self.context.getPhysicalPath()),
object_provides=[IBlobDataset.__identifier__,
IRemoteDataset.__identifier__])
fname = None
try:
# create tmp file
fd, fname = tempfile.mkstemp()
fo = os.fdopen(fd, 'w')
zfile = zipfile.ZipFile(fo, 'w')
metadata = {}
# the file/folder name for the zip
zfilename = self.context.title
# iterate over files and add to zip
for brain in brains:
content = brain.getObject()
if IBlobDataset.providedBy(content):
# If data is stored locally:
arcname = '/'.join((zfilename, 'data', content.file.filename))
# ob.file should be a NamedFile ... need to get fs name for that
blobfile = content.file.openDetached()
zfile.write(blobfile.name, arcname)
blobfile.close()
elif IRemoteDataset.providedBy(content):
# TODO: duplicate code from
remoteUrl = getattr(content, 'remoteUrl', None)
if remoteUrl is None:
raise NotFound(self, 'remoteUrl', self.request)
# get arcname from remoteUrl
arcname = '/'.join((zfilename, 'data', os.path.basename(remoteUrl)))
# FIXME: should check dataset downloaiable flag here,
# but assumption is, that this function can only be called on an experiment result folder....
# TODO: duplicate code in browser/dataset.py:RemoteDatasetDownload.__call__
# TODO: may not work in general... it always uses swift as remote url
tool = getUtility(ISwiftUtility)
try:
url = tool.generate_temp_url(url=remoteUrl)
except:
url = remoteUrl
# url is now the location from which we can fetch the file
temp_file, _ = urlretrieve(url)
zfile.write(temp_file, arcname)
os.remove(temp_file)
else:
# unknown type of Dataset
# just skip it
# TODO: Log warning or debug?
continue
metadata[arcname] = getdsmetadata(content)
# all files are in ....
# TODO: add experiment result metadata
# put metadata into zip
# provenance data stored on result container
provdata = IProvenanceData(self.context)
if not provdata.data is None:
zfile.writestr('/'.join((zfilename, 'prov.ttl')),
provdata.data)
# add mets.xml
metsview = getMultiAdapter((self.context, self.request), name="mets.xml")
zfile.writestr('/'.join((zfilename, 'mets.xml')),
metsview.render())
# finish zip file
zfile.close()
fo.close()
# create response
self.request.response.setHeader('Content-Type', 'application/zip')
self.request.response.setHeader('Content-Disposition', 'attachment; filename="{}.zip"'.format(zfilename))
self.request.response.setHeader('Content-Length', '{}'.format(os.path.getsize(fname)))
return tmpfile_stream_iterator(fname)
except Exception as e:
# something went wrong ...
# clean up and re-raise
if os.path.exists(fname):
os.remove(fname)
raise e
