def createGeojsonDataset(self, item, params):
user = self.getCurrentUser()
folder = findDatasetFolder(user, user, create=True)
if folder is None:
raise RestException('User has no Minerva Dataset folder.')
if folder['_id'] != item['folderId']:
raise RestException("Items need to be in user's Minerva Dataset " +
"folder.")
minerva_metadata = {
'original_type': 'geojson',
'dataset_type': 'geojson',
}
# Use the first geojson or json file found as the dataset.
for file in self.model('item').childFiles(item=item, limit=0):
if ('geojson' in file['exts'] or 'json' in file['exts'] or
file.get('mimeType') in (
'application/json', 'application/vnd.geo+json',
)):
minerva_metadata['original_files'] = [{
'name': file['name'], '_id': file['_id']}]
minerva_metadata['geojson_file'] = {
'name': file['name'], '_id': file['_id']}
minerva_metadata['geo_render'] = {
'type': 'geojson', 'file_id': file['_id']}
minerva_metadata['original_type'] = 'geojson'
minerva_metadata['source'] = {
'layer_source': 'GeoJSON'}
minerva_metadata['source_type'] = 'item'
break
if 'geojson_file' not in minerva_metadata:
raise RestException('Item contains no geojson file.')
updateMinervaMetadata(item, minerva_metadata)
return item
def createBsveDataset(user, bsveRoot, extraHeaders, params, layer_type):
typeName = params['type_name']
try:
if params['metadata']:
layer_info = params['metadata']
else:
layer_info = BsveWmsStyle(
typeName, bsveRoot=bsveRoot,
extraHeaders=extraHeaders).get_layer_info(layer_type)
except TypeError:
layer_info = ""
# TODO: Add the legend url here once it is
# ready on bsve side
name = params['name']
params['layer_info'] = layer_info
params['adapter'] = 'bsve'
legend_url = bsveRoot + "/data/v2/sources/geotiles/data/result?"
legend_qs = quote(
"$filter=name eq {} and request eq getlegendgraphic and height eq 20 and width eq 20"
.format(typeName),
safe='= ').replace(' ', '+')
r = requests.get(legend_url + legend_qs, headers=extraHeaders)
legend = b64encode(r.content)
params['legend'] = legend
# dataset = self.constructDataset(name, params)
folder = findDatasetFolder(user, user, create=True)
dataset = Item().createItem(name, user, folder, '')
updateMinervaMetadata(dataset, params)
return dataset
def createBsveDataset(user, bsveRoot, extraHeaders, params, layer_type):
typeName = params['type_name']
try:
if params['metadata']:
layer_info = params['metadata']
else:
layer_info = BsveWmsStyle(typeName, bsveRoot=bsveRoot,
extraHeaders=extraHeaders).get_layer_info(layer_type)
except TypeError:
layer_info = ""
# TODO: Add the legend url here once it is
# ready on bsve side
name = params['name']
params['layer_info'] = layer_info
params['adapter'] = 'bsve'
legend_url = bsveRoot + "/data/v2/sources/geotiles/data/result?"
legend_qs = quote("$filter=name eq {} and request eq getlegendgraphic and height eq 20 and width eq 20".format(
typeName), safe='= ').replace(' ', '+')
r = requests.get(legend_url + legend_qs, headers=extraHeaders)
legend = b64encode(r.content)
params['legend'] = legend
# dataset = self.constructDataset(name, params)
folder = findDatasetFolder(user, user, create=True)
dataset = Item().createItem(name, user, folder, '')
updateMinervaMetadata(dataset, params)
return dataset
def _update_metadata(self, item_id, params, meta_key):
"""Adds a new field to metadata"""
item = self.model('item').load(item_id, user=self.getCurrentUser())
item['meta']['minerva'][meta_key] = params
updateMinervaMetadata(item, item['meta']['minerva'])
def _update_metadata(self, item_id, params, meta_key):
"""Adds a new field to metadata"""
item = self.model('item').load(item_id, user=self.getCurrentUser())
item['meta']['minerva'][meta_key] = params
updateMinervaMetadata(item, item['meta']['minerva'])
def constructDataset(self, name, minerva_metadata, desc=''):
user = self.getCurrentUser()
folder = findDatasetFolder(user, user, create=True)
if folder is None:
raise Exception('User has no Minerva Dataset folder.')
dataset = self.model('item').createItem(name, user, folder, desc)
updateMinervaMetadata(dataset, minerva_metadata)
return dataset
def createSource(self, name, minerva_metadata, desc=''):
user = self.getCurrentUser()
folder = findSourceFolder(user, user, create=True)
if folder is None:
raise RestException('User has no Minerva Source folder.')
source = self.model('item').createItem(name, user, folder, desc)
updateMinervaMetadata(source, minerva_metadata)
return source
def constructDataset(self, name, minerva_metadata, desc=''):
user = self.getCurrentUser()
folder = findDatasetFolder(user, user, create=True)
if folder is None:
raise Exception('User has no Minerva Dataset folder.')
dataset = self.model('item').createItem(name, user, folder, desc)
updateMinervaMetadata(dataset, minerva_metadata)
return dataset
def createGeojsonDataset(self, item, params, postgresGeojson=None):
user = self.getCurrentUser()
folder = findDatasetFolder(user, user, create=True)
if folder is None:
raise RestException('User has no Minerva Dataset folder.')
if folder['_id'] != item['folderId']:
raise RestException("Items need to be in user's Minerva Dataset " +
"folder.")
minerva_metadata = {
'original_type': 'geojson',
'dataset_type': 'geojson',
}
# Use the first geojson or json file found as the dataset.
for file in self.model('item').childFiles(item=item, limit=0):
if ('geojson' in file['exts'] or 'json' in file['exts']
or file.get('mimeType') in (
'application/json',
'application/vnd.geo+json',
)):
minerva_metadata['original_files'] = [{
'name': file['name'],
'_id': file['_id']
}]
minerva_metadata['geojson_file'] = {
'name': file['name'],
'_id': file['_id']
}
minerva_metadata['geo_render'] = {
'type': 'geojson',
'file_id': file['_id']
}
minerva_metadata['original_type'] = 'geojson'
minerva_metadata['source'] = {'layer_source': 'GeoJSON'}
minerva_metadata['source_type'] = 'item'
if postgresGeojson is not None:
if postgresGeojson['field'] is not None:
minerva_metadata['visProperties'] = {
'line': {
"fillColorKey": postgresGeojson['field']
},
'polygon': {
"fillColorKey": postgresGeojson['field']
},
'point': {
"fillColorKey": postgresGeojson['field']
}
}
minerva_metadata['postgresGeojson'] = postgresGeojson
break
if 'geojson_file' not in minerva_metadata:
raise RestException('Item contains no geojson file.')
updateMinervaMetadata(item, minerva_metadata)
return item
def promoteItemToDataset(self, item, params):
"""
Take an Item in the user's Minerva Dataset folder, and promote
it to a Minerva Dataset by adding proper Minerva metadata.
"""
user = self.getCurrentUser()
folder = findDatasetFolder(user, user, create=True)
if folder is None:
raise RestException('User has no Minerva Dataset folder.')
if folder['_id'] != item['folderId']:
raise RestException("Items need to be in user's Minerva Dataset " +
"folder.")
# Don't overwrite if minerva metadata already exists.
if 'meta' in item and 'minerva' in item['meta']:
return item
minerva_metadata = {'source_type': 'item'}
for file in self.model('item').childFiles(item=item, limit=0):
# TODO This switching based on which file is found first is
# fairly brittle and should only be called after first upload.
if 'geojson' in file['exts']:
# we found a geojson, assume this is geojson original
minerva_metadata['original_type'] = 'geojson'
minerva_metadata['dataset_type'] = 'geojson'
minerva_metadata['original_files'] = [{
'name': file['name'],
'_id': file['_id']
}]
minerva_metadata['geojson_file'] = {
'name': file['name'],
'_id': file['_id']
}
minerva_metadata['source'] = {'layer_source': 'GeoJSON'}
break
elif 'json' in file['exts']:
minerva_metadata['original_type'] = 'json'
minerva_metadata['dataset_type'] = 'json'
minerva_metadata['original_files'] = [{
'name': file['name'],
'_id': file['_id']
}]
break
elif 'csv' in file['exts']:
minerva_metadata['original_type'] = 'csv'
minerva_metadata['dataset_type'] = 'csv'
minerva_metadata['original_files'] = [{
'name': file['name'],
'_id': file['_id']
}]
break
updateMinervaMetadata(item, minerva_metadata)
return item
def promoteItemToDataset(self, item, params):
"""
Take an Item in the user's Minerva Dataset folder, and promote
it to a Minerva Dataset by adding proper Minerva metadata.
"""
user = self.getCurrentUser()
folder = findDatasetFolder(user, user, create=True)
if folder is None:
raise RestException('User has no Minerva Dataset folder.')
if folder['_id'] != item['folderId']:
raise RestException("Items need to be in user's Minerva Dataset " +
"folder.")
# Don't overwrite if minerva metadata already exists.
if 'meta' in item and 'minerva' in item['meta']:
return item
minerva_metadata = {
'source_type': 'item'
}
for file in self.model('item').childFiles(item=item, limit=0):
# TODO This switching based on which file is found first is
# fairly brittle and should only be called after first upload.
if 'geojson' in file['exts']:
# we found a geojson, assume this is geojson original
minerva_metadata['original_type'] = 'geojson'
minerva_metadata['dataset_type'] = 'geojson'
minerva_metadata['original_files'] = [{
'name': file['name'], '_id': file['_id']}]
minerva_metadata['geojson_file'] = {
'name': file['name'], '_id': file['_id']}
break
elif 'json' in file['exts']:
minerva_metadata['original_type'] = 'json'
minerva_metadata['dataset_type'] = 'json'
minerva_metadata['original_files'] = [{
'name': file['name'], '_id': file['_id']}]
break
elif 'csv' in file['exts']:
minerva_metadata['original_type'] = 'csv'
minerva_metadata['dataset_type'] = 'csv'
minerva_metadata['original_files'] = [{
'name': file['name'], '_id': file['_id']}]
break
updateMinervaMetadata(item, minerva_metadata)
return item
def queryElasticsearch(self, params):
"""
Creates a local job to run the elasticsearch_worker, the job will store
the results of the elastic search query in a dataset.
"""
currentUser = self.getCurrentUser()
datasetName = params['datasetName']
elasticsearchParams = params['searchParams']
datasetFolder = findDatasetFolder(currentUser, currentUser)
dataset = (self.model('item').createItem(
datasetName,
currentUser,
datasetFolder,
'created by elasticsearch query'))
user, token = self.getCurrentUser(returnToken=True)
kwargs = {
'params': params,
'user': currentUser,
'dataset': dataset,
'token': token,
'sourceId': params['sourceId']
}
job = self.model('job', 'jobs').createLocalJob(
title='elasticsearch: %s' % datasetName,
user=currentUser,
type='elasticsearch',
public=False,
kwargs=kwargs,
module='girder.plugins.minerva.jobs.elasticsearch_worker',
async=True)
minerva_metadata = {
'dataset_type': 'json',
'source_id': params['sourceId'],
'source': 'elasticsearch',
'elasticsearch_params': elasticsearchParams
}
updateMinervaMetadata(dataset, minerva_metadata)
self.model('job', 'jobs').scheduleJob(job)
return job
def promoteItemToDataset(self, item, params):
"""
Take an Item in the user's Minerva Dataset folder, and promote
it to a Minerva Dataset by adding proper Minerva metadata.
"""
user = self.getCurrentUser()
folder = findDatasetFolder(user, user, create=True)
if folder is None:
raise RestException('User has no Minerva Dataset folder.')
if folder['_id'] != item['folderId']:
raise RestException("Items need to be in user's Minerva Dataset " +
"folder.")
# Don't overwrite if minerva metadata already exists.
if 'meta' in item and 'minerva' in item['meta']:
return item
minerva_metadata = self._updateMinervaMetadata(item)
bounds = self._getBound(item)
if bounds:
minerva_metadata['bounds'] = bounds
updateMinervaMetadata(item, minerva_metadata)
return item
def _updateMinervaMetadata(self, item):
minerva_metadata = {
'source_type': 'item'
}
for file in self.model('item').childFiles(item=item, limit=0):
# Check the first few k of a file to see if this might be a
# geojson timeseries. Crudely, we expect this to be a json array
# which contains objects, each of which has at least a geojson
# element. This test will fail if there are other elements in the
# first object that push the geojson element beyond the tested
# header length. It could give a false positive, too. The correct
# way would be to download and parse the whole file, but that would
# be more expensive in memory and time.
headerLen = 2048
fileHeader = ''
for headerData in self.model('file').download(file, headers=False, endByte=headerLen)():
fileHeader = (fileHeader + headerData)[:headerLen]
if len(fileHeader) >= headerLen:
break
if (fileHeader.lstrip()[:1] == '[' and
fileHeader.lstrip()[1:].lstrip()[:1] == '{' and
'"geojson"' in fileHeader):
minerva_metadata['original_type'] = 'geojson-timeseries'
minerva_metadata['dataset_type'] = 'geojson-timeseries'
minerva_metadata['original_files'] = [{
'name': file['name'], '_id': file['_id']}]
minerva_metadata['geojson_file'] = {
'name': file['name'], '_id': file['_id']}
minerva_metadata['source'] = {
'layer_source': 'GeoJSON'}
break
# TODO This switching based on which file is found first is
# fairly brittle and should only be called after first upload.
if 'geojson' in file['exts']:
# we found a geojson, assume this is geojson original
minerva_metadata['original_type'] = 'geojson'
minerva_metadata['dataset_type'] = 'geojson'
minerva_metadata['original_files'] = [{
'name': file['name'], '_id': file['_id']}]
minerva_metadata['geojson_file'] = {
'name': file['name'], '_id': file['_id']}
minerva_metadata['source'] = {
'layer_source': 'GeoJSON'}
break
elif 'json' in file['exts']:
minerva_metadata['original_type'] = 'json'
minerva_metadata['dataset_type'] = 'json'
minerva_metadata['original_files'] = [{
'name': file['name'], '_id': file['_id']}]
break
elif 'csv' in file['exts']:
minerva_metadata['original_type'] = 'csv'
minerva_metadata['dataset_type'] = 'csv'
minerva_metadata['original_files'] = [{
'name': file['name'], '_id': file['_id']}]
break
elif ({'tif', 'tiff'}.intersection(file['exts']) and
file['mimeType'] == 'image/tiff'):
info = ImageItem().tileSource(item).getMetadata()
if 'srs' in info['sourceBounds'] and info['sourceBounds']['srs']:
minerva_metadata['original_type'] = 'tiff'
minerva_metadata['dataset_type'] = 'geotiff'
minerva_metadata['original_files'] = [{
'name': file['name'], '_id': file['_id']}]
minerva_metadata['source'] = {
'layer_source': 'Tiff'}
break
updateMinervaMetadata(item, minerva_metadata)
return minerva_metadata
def promoteItemToDataset(self, item, params):
"""
Take an Item in the user's Minerva Dataset folder, and promote
it to a Minerva Dataset by adding proper Minerva metadata.
"""
user = self.getCurrentUser()
folder = findDatasetFolder(user, user, create=True)
if folder is None:
raise RestException('User has no Minerva Dataset folder.')
if folder['_id'] != item['folderId']:
raise RestException("Items need to be in user's Minerva Dataset " +
"folder.")
# Don't overwrite if minerva metadata already exists.
if 'meta' in item and 'minerva' in item['meta']:
return item
minerva_metadata = {'source_type': 'item'}
for file in self.model('item').childFiles(item=item, limit=0):
# Check the first few k of a file to see if this might be a
# geojson timeseries. Crudely, we expect this to be a json array
# which contains objects, each of which has at least a geojson
# element. This test will fail if there are other elements in the
# first object that push the geojson element beyond the tested
# header length. It could give a false positive, too. The correct
# way would be to download and parse the whole file, but that would
# be more expensive in memory and time.
headerLen = 2048
fileHeader = ''
for headerData in self.model('file').download(file,
headers=False,
endByte=headerLen)():
fileHeader = (fileHeader + headerData)[:headerLen]
if len(fileHeader) >= headerLen:
break
if (fileHeader.lstrip()[:1] == '['
and fileHeader.lstrip()[1:].lstrip()[:1] == '{'
and '"geojson"' in fileHeader):
minerva_metadata['original_type'] = 'geojson-timeseries'
minerva_metadata['dataset_type'] = 'geojson-timeseries'
minerva_metadata['original_files'] = [{
'name': file['name'],
'_id': file['_id']
}]
minerva_metadata['geojson_file'] = {
'name': file['name'],
'_id': file['_id']
}
minerva_metadata['source'] = {'layer_source': 'GeoJSON'}
break
# TODO This switching based on which file is found first is
# fairly brittle and should only be called after first upload.
if 'geojson' in file['exts']:
# we found a geojson, assume this is geojson original
minerva_metadata['original_type'] = 'geojson'
minerva_metadata['dataset_type'] = 'geojson'
minerva_metadata['original_files'] = [{
'name': file['name'],
'_id': file['_id']
}]
minerva_metadata['geojson_file'] = {
'name': file['name'],
'_id': file['_id']
}
minerva_metadata['source'] = {'layer_source': 'GeoJSON'}
break
elif 'json' in file['exts']:
minerva_metadata['original_type'] = 'json'
minerva_metadata['dataset_type'] = 'json'
minerva_metadata['original_files'] = [{
'name': file['name'],
'_id': file['_id']
}]
break
elif 'csv' in file['exts']:
minerva_metadata['original_type'] = 'csv'
minerva_metadata['dataset_type'] = 'csv'
minerva_metadata['original_files'] = [{
'name': file['name'],
'_id': file['_id']
}]
break
elif ({'tif', 'tiff'}.intersection(file['exts'])
and file['mimeType'] == 'image/tiff'):
info = getInfo(file)
if 'srs' in info and info['srs']:
minerva_metadata['original_type'] = 'tiff'
minerva_metadata['dataset_type'] = 'geotiff'
minerva_metadata['original_files'] = [{
'name': file['name'],
'_id': file['_id']
}]
minerva_metadata['source'] = {'layer_source': 'Tiff'}
break
updateMinervaMetadata(item, minerva_metadata)
bounds = self._getBound(item)
if bounds:
minerva_metadata['bounds'] = bounds
updateMinervaMetadata(item, minerva_metadata)
return item