def items_old(self): # FIXME importing here to avoid circular import of IDataset from org.bccvl.site.api.dataset import getdsmetadata if self.value: for uuid in self.value: brain = uuidToCatalogBrain(uuid) # TODO: could use layer vocab again md = getdsmetadata(brain) layers = self.value[uuid] # FIXME: check if layers or layers_used here for layer, layeritem in md['layers'].iteritems(): if not layer in layers: continue mimetype = 'application/octet-stream' layerfile = None if 'filename' in layeritem: # FIXME: hardcoded mimetype logic for zip files. # should draw mimetype info from layer metadata # assumes there are only geotiff in zip files mimetype = 'image/geotiff' layerfile = layeritem['filename'] vizurl = '{0}#{1}'.format(md['vizurl'], layerfile) else: vizurl = md['vizurl'] mimetype = md['mimetype'] yield {"brain": brain, "resolution": self.dstools.resolution_vocab.getTerm(brain['BCCResolution']), "layer": self.dstools.layer_vocab.getTerm(layer), "vizurl": vizurl, 'mimetype': mimetype, 'vizlayer': layerfile}
def getdatasetparams(uuid): # return dict with: # filename # downloadurl # dm_accessurl-> maybe add url rewrite to datamover? # # occurrence specific: # species # # raster specific: # layers ... need to split this up dsobj = uuidToObject(uuid) if dsobj is None: return None dsinfo = getDatasetInfo(dsobj, uuid) # if we have species info add it dsmdr = IBCCVLMetadata(dsobj) species = dsmdr.get('species', {}).get('scientificName') if species: dsinfo['species'] = species # if we can get layermetadata, let's add it biomod = getdsmetadata(dsobj) layers = biomod.get('layers', []) if len(layers) > 0: for lk, lv in biomod['layers'].items(): if lv is not None: dsinfo.setdefault('layers', {})[lk] = { 'filename': lv.get('filename', biomod['filename']), 'datatype': lv.get('datatype', None) } # return infoset return dsinfo
def getdatasetparams(uuid): # return dict with: # filename # downloadurl # dm_accessurl-> maybe add url rewrite to datamover? # # occurrence specific: # species # # raster specific: # layers ... need to split this up dsobj = uuidToObject(uuid) if dsobj is None: return None dsinfo = getDatasetInfo(dsobj, uuid) # if we have species info add it dsmdr = IBCCVLMetadata(dsobj) species = dsmdr.get('species', {}).get('scientificName') if species: dsinfo['species'] = species # if we can get layermetadata, let's add it biomod = getdsmetadata(dsobj) layers = biomod.get('layers', []) if len(layers) > 0: for lk, lv in biomod['layers'].items(): if lv is not None: dsinfo.setdefault('layers', {})[lk] = { 'filename': lv.get('filename', biomod['filename']), 'datatype': lv.get('datatype', None)} # return infoset return dsinfo
def metadata(self, itemobj=None, uuid=None): if itemobj is None and uuid is None: itemobj = self.context if uuid: itemobj = uuidToObject(uuid) if itemobj is None: return None return getdsmetadata(itemobj)
def __call__(self): # 1. find all IBlobDataset/ IRemotedDataset/ IDataset objects within context pc = getToolByName(self.context, 'portal_catalog') brains = pc.searchResults(path='/'.join(self.context.getPhysicalPath()), object_provides=IBlobDataset.__identifier__) fname = None try: # create tmp file fd, fname = tempfile.mkstemp() fo = os.fdopen(fd, 'w') zfile = zipfile.ZipFile(fo, 'w') metadata = {} # the file/folder name for the zip zfilename = self.context.title # iterate over files and add to zip for brain in brains: content = brain.getObject() # ob.file should be a NamedFile ... need to get fs name for that blobfile = content.file.openDetached() arcname = '/'.join((zfilename, 'data', content.file.filename)) zfile.write(blobfile.name, arcname) blobfile.close() metadata[arcname] = getdsmetadata(content) # all files are in .... # TODO: add experiment result metadata # put metadata into zip # provenance data stored on result container provdata = IProvenanceData(self.context) if not provdata.data is None: zfile.writestr('/'.join((zfilename, 'prov.ttl')), provdata.data) # add mets.xml metsview = getMultiAdapter((self.context, self.request), name="mets.xml") zfile.writestr('/'.join((zfilename, 'mets.xml')), metsview.render()) # finish zip file zfile.close() fo.close() # create response self.request.response.setHeader('Content-Type', 'application/zip') self.request.response.setHeader('Content-Disposition', 'attachment; filename="{}.zip"'.format(zfilename)) self.request.response.setHeader('Content-Length', '{}'.format(os.path.getsize(fname))) return tmpfile_stream_iterator(fname) except Exception as e: # something went wrong ... # clean up and re-raise if os.path.exists(fname): os.remove(fname) raise e
def metadata(self): uuid = self.request.form.get("uuid") try: brain = uuidToCatalogBrain(uuid) if brain: return dataset.getdsmetadata(brain) except Exception as e: LOG.error("Caught exception %s", e) self.record_error("Not Found", "404", "dataset not found", {"parameter": "uuid"}) raise NotFound(self, "metadata", self.request)
def metadata(self): uuid = self.request.form.get('uuid') try: brain = uuidToCatalogBrain(uuid) if brain: return dataset.getdsmetadata(brain) except Exception as e: LOG.error('Caught exception %s', e) self.record_error('Not Found', '404', 'dataset not found', {'parameter': 'uuid'}) raise NotFound(self, 'metadata', self.request)
def metadata(self): uuid = self.request.form.get('uuid') try: brain = uuidToCatalogBrain(uuid) if brain: return dataset.getdsmetadata(brain) except Exception as e: LOG.error('Caught exception %s', e) self.record_error('Not Found', '404', 'dataset not found', { 'parameter': 'uuid'}) raise NotFound(self, 'metadata', self.request)
def items_old(self): # FIXME importing here to avoid circular import of IDataset from org.bccvl.site.api.dataset import getdsmetadata if self.value: for uuid in self.value: brain = uuidToCatalogBrain(uuid) # TODO: could use layer vocab again md = getdsmetadata(brain) layers = self.value[uuid] # FIXME: check if layers or layers_used here for layer, layeritem in md['layers'].iteritems(): if not layer in layers: continue mimetype = 'application/octet-stream' layerfile = None if 'filename' in layeritem: # FIXME: hardcoded mimetype logic for zip files. # should draw mimetype info from layer metadata # assumes there are only geotiff in zip files mimetype = 'image/geotiff' layerfile = layeritem['filename'] vizurl = '{0}#{1}'.format(md['vizurl'], layerfile) else: vizurl = md['vizurl'] mimetype = md['mimetype'] yield { "brain": brain, "resolution": self.dstools.resolution_vocab.getTerm( brain['BCCResolution']), "layer": self.dstools.layer_vocab.getTerm(layer), "vizurl": vizurl, 'mimetype': mimetype, 'vizlayer': layerfile }
def _download_results(self, context, zfile): # FIXME: This is a rather lengthy process, and should probably be turned into a background task... (maybe as part of a datamanager service?) # 1. find all IBlobDataset/ IRemotedDataset/ IDataset objects within context pc = getToolByName(context, 'portal_catalog') brains = pc.searchResults(path='/'.join(context.getPhysicalPath()), object_provides=[ IBlobDataset.__identifier__, IRemoteDataset.__identifier__ ]) metadata = {} # the file/folder name for the zip zfilename = context.title # iterate over files and add to zip for brain in brains: content = brain.getObject() if IBlobDataset.providedBy(content): # If data is stored locally: arcname = '/'.join((zfilename, 'data', content.file.filename)) # ob.file should be a NamedFile ... need to get fs name for that blobfile = content.file.openDetached() zfile.write(blobfile.name, arcname) blobfile.close() elif IRemoteDataset.providedBy(content): # TODO: duplicate code from remoteUrl = getattr(content, 'remoteUrl', None) if remoteUrl is None: raise NotFound(self, 'remoteUrl', self.request) # get arcname from remoteUrl arcname = '/'.join( (zfilename, 'data', os.path.basename(remoteUrl))) # FIXME: should check dataset downloaiable flag here, # but assumption is, that this function can only be called on an experiment result folder.... # TODO: duplicate code in browser/dataset.py:RemoteDatasetDownload.__call__ # TODO: may not work in general... it always uses swift as remote url tool = getUtility(ISwiftUtility) try: url = tool.generate_temp_url(url=remoteUrl) except: url = remoteUrl # url is now the location from which we can fetch the file temp_file, _ = urlretrieve(url) zfile.write(temp_file, arcname) os.remove(temp_file) else: # unknown type of Dataset # just skip it # TODO: Log warning or debug? continue metadata[arcname] = getdsmetadata(content) # all files are in .... # TODO: add experiment result metadata # put metadata into zip # provenance data stored on result container provdata = IProvenanceData(context) if not provdata.data is None: zfile.writestr('/'.join((zfilename, 'prov.ttl')), provdata.data.encode('utf-8')) # add experiment metadata expmetadata = IExperimentMetadata(context) if not expmetadata.data is None: zfile.writestr('/'.join((zfilename, 'expmetadata.txt')), expmetadata.data.encode('utf-8')) # add mets.xml metsview = getMultiAdapter((context, self.request), name="mets.xml") zfile.writestr('/'.join((zfilename, 'mets.xml')), metsview.render().encode('utf-8')) # add experiment parameters params = IExperimentParameter(context) if not params.data is None: zfile.writestr('/'.join((zfilename, 'params.json')), params.data.encode('utf-8'))
def _download_results(self, context, zfile): # FIXME: This is a rather lengthy process, and should probably be turned into a background task... (maybe as part of a datamanager service?) # 1. find all IBlobDataset/ IRemotedDataset/ IDataset objects within context pc = getToolByName(context, 'portal_catalog') brains = pc.searchResults(path='/'.join(context.getPhysicalPath()), object_provides=[IBlobDataset.__identifier__, IRemoteDataset.__identifier__]) metadata = {} # the file/folder name for the zip zfilename = context.title # iterate over files and add to zip for brain in brains: content = brain.getObject() if IBlobDataset.providedBy(content): # If data is stored locally: arcname = '/'.join((zfilename, 'data', content.file.filename)) # ob.file should be a NamedFile ... need to get fs name for that blobfile = content.file.openDetached() zfile.write(blobfile.name, arcname) blobfile.close() elif IRemoteDataset.providedBy(content): # TODO: duplicate code from remoteUrl = getattr(content, 'remoteUrl', None) if remoteUrl is None: raise NotFound(self, 'remoteUrl', self.request) # get arcname from remoteUrl arcname = '/'.join((zfilename, 'data', os.path.basename(remoteUrl))) # FIXME: should check dataset downloaiable flag here, # but assumption is, that this function can only be called on an experiment result folder.... # TODO: duplicate code in browser/dataset.py:RemoteDatasetDownload.__call__ # TODO: may not work in general... it always uses swift as remote url tool = getUtility(ISwiftUtility) try: url = tool.generate_temp_url(url=remoteUrl) except: url = remoteUrl # url is now the location from which we can fetch the file temp_file, _ = urlretrieve(url) zfile.write(temp_file, arcname) os.remove(temp_file) else: # unknown type of Dataset # just skip it # TODO: Log warning or debug? continue metadata[arcname] = getdsmetadata(content) # all files are in .... # TODO: add experiment result metadata # put metadata into zip # provenance data stored on result container provdata = IProvenanceData(context) if not provdata.data is None: zfile.writestr('/'.join((zfilename, 'prov.ttl')), provdata.data.encode('utf-8')) # add experiment metadata expmetadata = IExperimentMetadata(context) if not expmetadata.data is None: zfile.writestr('/'.join((zfilename, 'expmetadata.txt')), expmetadata.data.encode('utf-8')) # add mets.xml metsview = getMultiAdapter((context, self.request), name="mets.xml") zfile.writestr('/'.join((zfilename, 'mets.xml')), metsview.render().encode('utf-8')) # add experiment parameters params = IExperimentParameter(context) if not params.data is None: zfile.writestr('/'.join((zfilename, 'params.json')), params.data.encode('utf-8'))
def getMetadata(self): return dataset.getdsmetadata(self.context)
def metadata(self, itemobj=None): if itemobj is None: itemobj = self.context return getdsmetadata(itemobj)
def __call__(self): # FIXME: This is a rather lengthy process, and should probably be turned into a background task... (maybe as part of a datamanager service?) # 1. find all IBlobDataset/ IRemotedDataset/ IDataset objects within context pc = getToolByName(self.context, 'portal_catalog') brains = pc.searchResults(path='/'.join(self.context.getPhysicalPath()), object_provides=[IBlobDataset.__identifier__, IRemoteDataset.__identifier__]) fname = None try: # create tmp file fd, fname = tempfile.mkstemp() fo = os.fdopen(fd, 'w') zfile = zipfile.ZipFile(fo, 'w') metadata = {} # the file/folder name for the zip zfilename = self.context.title # iterate over files and add to zip for brain in brains: content = brain.getObject() if IBlobDataset.providedBy(content): # If data is stored locally: arcname = '/'.join((zfilename, 'data', content.file.filename)) # ob.file should be a NamedFile ... need to get fs name for that blobfile = content.file.openDetached() zfile.write(blobfile.name, arcname) blobfile.close() elif IRemoteDataset.providedBy(content): # TODO: duplicate code from remoteUrl = getattr(content, 'remoteUrl', None) if remoteUrl is None: raise NotFound(self, 'remoteUrl', self.request) # get arcname from remoteUrl arcname = '/'.join((zfilename, 'data', os.path.basename(remoteUrl))) # FIXME: should check dataset downloaiable flag here, # but assumption is, that this function can only be called on an experiment result folder.... # TODO: duplicate code in browser/dataset.py:RemoteDatasetDownload.__call__ # TODO: may not work in general... it always uses swift as remote url tool = getUtility(ISwiftUtility) try: url = tool.generate_temp_url(url=remoteUrl) except: url = remoteUrl # url is now the location from which we can fetch the file temp_file, _ = urlretrieve(url) zfile.write(temp_file, arcname) os.remove(temp_file) else: # unknown type of Dataset # just skip it # TODO: Log warning or debug? continue metadata[arcname] = getdsmetadata(content) # all files are in .... # TODO: add experiment result metadata # put metadata into zip # provenance data stored on result container provdata = IProvenanceData(self.context) if not provdata.data is None: zfile.writestr('/'.join((zfilename, 'prov.ttl')), provdata.data) # add mets.xml metsview = getMultiAdapter((self.context, self.request), name="mets.xml") zfile.writestr('/'.join((zfilename, 'mets.xml')), metsview.render()) # finish zip file zfile.close() fo.close() # create response self.request.response.setHeader('Content-Type', 'application/zip') self.request.response.setHeader('Content-Disposition', 'attachment; filename="{}.zip"'.format(zfilename)) self.request.response.setHeader('Content-Length', '{}'.format(os.path.getsize(fname))) return tmpfile_stream_iterator(fname) except Exception as e: # something went wrong ... # clean up and re-raise if os.path.exists(fname): os.remove(fname) raise e