def createGeojsonDataset(self, item, params): user = self.getCurrentUser() folder = findDatasetFolder(user, user, create=True) if folder is None: raise RestException('User has no Minerva Dataset folder.') if folder['_id'] != item['folderId']: raise RestException("Items need to be in user's Minerva Dataset " + "folder.") minerva_metadata = { 'original_type': 'geojson', 'dataset_type': 'geojson', } # Use the first geojson or json file found as the dataset. for file in self.model('item').childFiles(item=item, limit=0): if ('geojson' in file['exts'] or 'json' in file['exts'] or file.get('mimeType') in ( 'application/json', 'application/vnd.geo+json', )): minerva_metadata['original_files'] = [{ 'name': file['name'], '_id': file['_id']}] minerva_metadata['geojson_file'] = { 'name': file['name'], '_id': file['_id']} minerva_metadata['geo_render'] = { 'type': 'geojson', 'file_id': file['_id']} minerva_metadata['original_type'] = 'geojson' minerva_metadata['source'] = { 'layer_source': 'GeoJSON'} minerva_metadata['source_type'] = 'item' break if 'geojson_file' not in minerva_metadata: raise RestException('Item contains no geojson file.') updateMinervaMetadata(item, minerva_metadata) return item
def createBsveDataset(user, bsveRoot, extraHeaders, params, layer_type): typeName = params['type_name'] try: if params['metadata']: layer_info = params['metadata'] else: layer_info = BsveWmsStyle( typeName, bsveRoot=bsveRoot, extraHeaders=extraHeaders).get_layer_info(layer_type) except TypeError: layer_info = "" # TODO: Add the legend url here once it is # ready on bsve side name = params['name'] params['layer_info'] = layer_info params['adapter'] = 'bsve' legend_url = bsveRoot + "/data/v2/sources/geotiles/data/result?" legend_qs = quote( "$filter=name eq {} and request eq getlegendgraphic and height eq 20 and width eq 20" .format(typeName), safe='= ').replace(' ', '+') r = requests.get(legend_url + legend_qs, headers=extraHeaders) legend = b64encode(r.content) params['legend'] = legend # dataset = self.constructDataset(name, params) folder = findDatasetFolder(user, user, create=True) dataset = Item().createItem(name, user, folder, '') updateMinervaMetadata(dataset, params) return dataset
def createBsveDataset(user, bsveRoot, extraHeaders, params, layer_type): typeName = params['type_name'] try: if params['metadata']: layer_info = params['metadata'] else: layer_info = BsveWmsStyle(typeName, bsveRoot=bsveRoot, extraHeaders=extraHeaders).get_layer_info(layer_type) except TypeError: layer_info = "" # TODO: Add the legend url here once it is # ready on bsve side name = params['name'] params['layer_info'] = layer_info params['adapter'] = 'bsve' legend_url = bsveRoot + "/data/v2/sources/geotiles/data/result?" legend_qs = quote("$filter=name eq {} and request eq getlegendgraphic and height eq 20 and width eq 20".format( typeName), safe='= ').replace(' ', '+') r = requests.get(legend_url + legend_qs, headers=extraHeaders) legend = b64encode(r.content) params['legend'] = legend # dataset = self.constructDataset(name, params) folder = findDatasetFolder(user, user, create=True) dataset = Item().createItem(name, user, folder, '') updateMinervaMetadata(dataset, params) return dataset
def _update_metadata(self, item_id, params, meta_key): """Adds a new field to metadata""" item = self.model('item').load(item_id, user=self.getCurrentUser()) item['meta']['minerva'][meta_key] = params updateMinervaMetadata(item, item['meta']['minerva'])
def _update_metadata(self, item_id, params, meta_key): """Adds a new field to metadata""" item = self.model('item').load(item_id, user=self.getCurrentUser()) item['meta']['minerva'][meta_key] = params updateMinervaMetadata(item, item['meta']['minerva'])
def constructDataset(self, name, minerva_metadata, desc=''): user = self.getCurrentUser() folder = findDatasetFolder(user, user, create=True) if folder is None: raise Exception('User has no Minerva Dataset folder.') dataset = self.model('item').createItem(name, user, folder, desc) updateMinervaMetadata(dataset, minerva_metadata) return dataset
def createSource(self, name, minerva_metadata, desc=''): user = self.getCurrentUser() folder = findSourceFolder(user, user, create=True) if folder is None: raise RestException('User has no Minerva Source folder.') source = self.model('item').createItem(name, user, folder, desc) updateMinervaMetadata(source, minerva_metadata) return source
def constructDataset(self, name, minerva_metadata, desc=''): user = self.getCurrentUser() folder = findDatasetFolder(user, user, create=True) if folder is None: raise Exception('User has no Minerva Dataset folder.') dataset = self.model('item').createItem(name, user, folder, desc) updateMinervaMetadata(dataset, minerva_metadata) return dataset
def createGeojsonDataset(self, item, params, postgresGeojson=None): user = self.getCurrentUser() folder = findDatasetFolder(user, user, create=True) if folder is None: raise RestException('User has no Minerva Dataset folder.') if folder['_id'] != item['folderId']: raise RestException("Items need to be in user's Minerva Dataset " + "folder.") minerva_metadata = { 'original_type': 'geojson', 'dataset_type': 'geojson', } # Use the first geojson or json file found as the dataset. for file in self.model('item').childFiles(item=item, limit=0): if ('geojson' in file['exts'] or 'json' in file['exts'] or file.get('mimeType') in ( 'application/json', 'application/vnd.geo+json', )): minerva_metadata['original_files'] = [{ 'name': file['name'], '_id': file['_id'] }] minerva_metadata['geojson_file'] = { 'name': file['name'], '_id': file['_id'] } minerva_metadata['geo_render'] = { 'type': 'geojson', 'file_id': file['_id'] } minerva_metadata['original_type'] = 'geojson' minerva_metadata['source'] = {'layer_source': 'GeoJSON'} minerva_metadata['source_type'] = 'item' if postgresGeojson is not None: if postgresGeojson['field'] is not None: minerva_metadata['visProperties'] = { 'line': { "fillColorKey": postgresGeojson['field'] }, 'polygon': { "fillColorKey": postgresGeojson['field'] }, 'point': { "fillColorKey": postgresGeojson['field'] } } minerva_metadata['postgresGeojson'] = postgresGeojson break if 'geojson_file' not in minerva_metadata: raise RestException('Item contains no geojson file.') updateMinervaMetadata(item, minerva_metadata) return item
def promoteItemToDataset(self, item, params): """ Take an Item in the user's Minerva Dataset folder, and promote it to a Minerva Dataset by adding proper Minerva metadata. """ user = self.getCurrentUser() folder = findDatasetFolder(user, user, create=True) if folder is None: raise RestException('User has no Minerva Dataset folder.') if folder['_id'] != item['folderId']: raise RestException("Items need to be in user's Minerva Dataset " + "folder.") # Don't overwrite if minerva metadata already exists. if 'meta' in item and 'minerva' in item['meta']: return item minerva_metadata = {'source_type': 'item'} for file in self.model('item').childFiles(item=item, limit=0): # TODO This switching based on which file is found first is # fairly brittle and should only be called after first upload. if 'geojson' in file['exts']: # we found a geojson, assume this is geojson original minerva_metadata['original_type'] = 'geojson' minerva_metadata['dataset_type'] = 'geojson' minerva_metadata['original_files'] = [{ 'name': file['name'], '_id': file['_id'] }] minerva_metadata['geojson_file'] = { 'name': file['name'], '_id': file['_id'] } minerva_metadata['source'] = {'layer_source': 'GeoJSON'} break elif 'json' in file['exts']: minerva_metadata['original_type'] = 'json' minerva_metadata['dataset_type'] = 'json' minerva_metadata['original_files'] = [{ 'name': file['name'], '_id': file['_id'] }] break elif 'csv' in file['exts']: minerva_metadata['original_type'] = 'csv' minerva_metadata['dataset_type'] = 'csv' minerva_metadata['original_files'] = [{ 'name': file['name'], '_id': file['_id'] }] break updateMinervaMetadata(item, minerva_metadata) return item
def promoteItemToDataset(self, item, params): """ Take an Item in the user's Minerva Dataset folder, and promote it to a Minerva Dataset by adding proper Minerva metadata. """ user = self.getCurrentUser() folder = findDatasetFolder(user, user, create=True) if folder is None: raise RestException('User has no Minerva Dataset folder.') if folder['_id'] != item['folderId']: raise RestException("Items need to be in user's Minerva Dataset " + "folder.") # Don't overwrite if minerva metadata already exists. if 'meta' in item and 'minerva' in item['meta']: return item minerva_metadata = { 'source_type': 'item' } for file in self.model('item').childFiles(item=item, limit=0): # TODO This switching based on which file is found first is # fairly brittle and should only be called after first upload. if 'geojson' in file['exts']: # we found a geojson, assume this is geojson original minerva_metadata['original_type'] = 'geojson' minerva_metadata['dataset_type'] = 'geojson' minerva_metadata['original_files'] = [{ 'name': file['name'], '_id': file['_id']}] minerva_metadata['geojson_file'] = { 'name': file['name'], '_id': file['_id']} break elif 'json' in file['exts']: minerva_metadata['original_type'] = 'json' minerva_metadata['dataset_type'] = 'json' minerva_metadata['original_files'] = [{ 'name': file['name'], '_id': file['_id']}] break elif 'csv' in file['exts']: minerva_metadata['original_type'] = 'csv' minerva_metadata['dataset_type'] = 'csv' minerva_metadata['original_files'] = [{ 'name': file['name'], '_id': file['_id']}] break updateMinervaMetadata(item, minerva_metadata) return item
def queryElasticsearch(self, params): """ Creates a local job to run the elasticsearch_worker, the job will store the results of the elastic search query in a dataset. """ currentUser = self.getCurrentUser() datasetName = params['datasetName'] elasticsearchParams = params['searchParams'] datasetFolder = findDatasetFolder(currentUser, currentUser) dataset = (self.model('item').createItem( datasetName, currentUser, datasetFolder, 'created by elasticsearch query')) user, token = self.getCurrentUser(returnToken=True) kwargs = { 'params': params, 'user': currentUser, 'dataset': dataset, 'token': token, 'sourceId': params['sourceId'] } job = self.model('job', 'jobs').createLocalJob( title='elasticsearch: %s' % datasetName, user=currentUser, type='elasticsearch', public=False, kwargs=kwargs, module='girder.plugins.minerva.jobs.elasticsearch_worker', async=True) minerva_metadata = { 'dataset_type': 'json', 'source_id': params['sourceId'], 'source': 'elasticsearch', 'elasticsearch_params': elasticsearchParams } updateMinervaMetadata(dataset, minerva_metadata) self.model('job', 'jobs').scheduleJob(job) return job
def promoteItemToDataset(self, item, params): """ Take an Item in the user's Minerva Dataset folder, and promote it to a Minerva Dataset by adding proper Minerva metadata. """ user = self.getCurrentUser() folder = findDatasetFolder(user, user, create=True) if folder is None: raise RestException('User has no Minerva Dataset folder.') if folder['_id'] != item['folderId']: raise RestException("Items need to be in user's Minerva Dataset " + "folder.") # Don't overwrite if minerva metadata already exists. if 'meta' in item and 'minerva' in item['meta']: return item minerva_metadata = self._updateMinervaMetadata(item) bounds = self._getBound(item) if bounds: minerva_metadata['bounds'] = bounds updateMinervaMetadata(item, minerva_metadata) return item
def _updateMinervaMetadata(self, item): minerva_metadata = { 'source_type': 'item' } for file in self.model('item').childFiles(item=item, limit=0): # Check the first few k of a file to see if this might be a # geojson timeseries. Crudely, we expect this to be a json array # which contains objects, each of which has at least a geojson # element. This test will fail if there are other elements in the # first object that push the geojson element beyond the tested # header length. It could give a false positive, too. The correct # way would be to download and parse the whole file, but that would # be more expensive in memory and time. headerLen = 2048 fileHeader = '' for headerData in self.model('file').download(file, headers=False, endByte=headerLen)(): fileHeader = (fileHeader + headerData)[:headerLen] if len(fileHeader) >= headerLen: break if (fileHeader.lstrip()[:1] == '[' and fileHeader.lstrip()[1:].lstrip()[:1] == '{' and '"geojson"' in fileHeader): minerva_metadata['original_type'] = 'geojson-timeseries' minerva_metadata['dataset_type'] = 'geojson-timeseries' minerva_metadata['original_files'] = [{ 'name': file['name'], '_id': file['_id']}] minerva_metadata['geojson_file'] = { 'name': file['name'], '_id': file['_id']} minerva_metadata['source'] = { 'layer_source': 'GeoJSON'} break # TODO This switching based on which file is found first is # fairly brittle and should only be called after first upload. if 'geojson' in file['exts']: # we found a geojson, assume this is geojson original minerva_metadata['original_type'] = 'geojson' minerva_metadata['dataset_type'] = 'geojson' minerva_metadata['original_files'] = [{ 'name': file['name'], '_id': file['_id']}] minerva_metadata['geojson_file'] = { 'name': file['name'], '_id': file['_id']} minerva_metadata['source'] = { 'layer_source': 'GeoJSON'} break elif 'json' in file['exts']: minerva_metadata['original_type'] = 'json' minerva_metadata['dataset_type'] = 'json' minerva_metadata['original_files'] = [{ 'name': file['name'], '_id': file['_id']}] break elif 'csv' in file['exts']: minerva_metadata['original_type'] = 'csv' minerva_metadata['dataset_type'] = 'csv' minerva_metadata['original_files'] = [{ 'name': file['name'], '_id': file['_id']}] break elif ({'tif', 'tiff'}.intersection(file['exts']) and file['mimeType'] == 'image/tiff'): info = ImageItem().tileSource(item).getMetadata() if 'srs' in info['sourceBounds'] and info['sourceBounds']['srs']: minerva_metadata['original_type'] = 'tiff' minerva_metadata['dataset_type'] = 'geotiff' minerva_metadata['original_files'] = [{ 'name': file['name'], '_id': file['_id']}] minerva_metadata['source'] = { 'layer_source': 'Tiff'} break updateMinervaMetadata(item, minerva_metadata) return minerva_metadata
def promoteItemToDataset(self, item, params): """ Take an Item in the user's Minerva Dataset folder, and promote it to a Minerva Dataset by adding proper Minerva metadata. """ user = self.getCurrentUser() folder = findDatasetFolder(user, user, create=True) if folder is None: raise RestException('User has no Minerva Dataset folder.') if folder['_id'] != item['folderId']: raise RestException("Items need to be in user's Minerva Dataset " + "folder.") # Don't overwrite if minerva metadata already exists. if 'meta' in item and 'minerva' in item['meta']: return item minerva_metadata = {'source_type': 'item'} for file in self.model('item').childFiles(item=item, limit=0): # Check the first few k of a file to see if this might be a # geojson timeseries. Crudely, we expect this to be a json array # which contains objects, each of which has at least a geojson # element. This test will fail if there are other elements in the # first object that push the geojson element beyond the tested # header length. It could give a false positive, too. The correct # way would be to download and parse the whole file, but that would # be more expensive in memory and time. headerLen = 2048 fileHeader = '' for headerData in self.model('file').download(file, headers=False, endByte=headerLen)(): fileHeader = (fileHeader + headerData)[:headerLen] if len(fileHeader) >= headerLen: break if (fileHeader.lstrip()[:1] == '[' and fileHeader.lstrip()[1:].lstrip()[:1] == '{' and '"geojson"' in fileHeader): minerva_metadata['original_type'] = 'geojson-timeseries' minerva_metadata['dataset_type'] = 'geojson-timeseries' minerva_metadata['original_files'] = [{ 'name': file['name'], '_id': file['_id'] }] minerva_metadata['geojson_file'] = { 'name': file['name'], '_id': file['_id'] } minerva_metadata['source'] = {'layer_source': 'GeoJSON'} break # TODO This switching based on which file is found first is # fairly brittle and should only be called after first upload. if 'geojson' in file['exts']: # we found a geojson, assume this is geojson original minerva_metadata['original_type'] = 'geojson' minerva_metadata['dataset_type'] = 'geojson' minerva_metadata['original_files'] = [{ 'name': file['name'], '_id': file['_id'] }] minerva_metadata['geojson_file'] = { 'name': file['name'], '_id': file['_id'] } minerva_metadata['source'] = {'layer_source': 'GeoJSON'} break elif 'json' in file['exts']: minerva_metadata['original_type'] = 'json' minerva_metadata['dataset_type'] = 'json' minerva_metadata['original_files'] = [{ 'name': file['name'], '_id': file['_id'] }] break elif 'csv' in file['exts']: minerva_metadata['original_type'] = 'csv' minerva_metadata['dataset_type'] = 'csv' minerva_metadata['original_files'] = [{ 'name': file['name'], '_id': file['_id'] }] break elif ({'tif', 'tiff'}.intersection(file['exts']) and file['mimeType'] == 'image/tiff'): info = getInfo(file) if 'srs' in info and info['srs']: minerva_metadata['original_type'] = 'tiff' minerva_metadata['dataset_type'] = 'geotiff' minerva_metadata['original_files'] = [{ 'name': file['name'], '_id': file['_id'] }] minerva_metadata['source'] = {'layer_source': 'Tiff'} break updateMinervaMetadata(item, minerva_metadata) bounds = self._getBound(item) if bounds: minerva_metadata['bounds'] = bounds updateMinervaMetadata(item, minerva_metadata) return item