def createBsveDataset(user, bsveRoot, extraHeaders, params, layer_type): typeName = params['type_name'] try: if params['metadata']: layer_info = params['metadata'] else: layer_info = BsveWmsStyle(typeName, bsveRoot=bsveRoot, extraHeaders=extraHeaders).get_layer_info(layer_type) except TypeError: layer_info = "" # TODO: Add the legend url here once it is # ready on bsve side name = params['name'] params['layer_info'] = layer_info params['adapter'] = 'bsve' legend_url = bsveRoot + "/data/v2/sources/geotiles/data/result?" legend_qs = quote("$filter=name eq {} and request eq getlegendgraphic and height eq 20 and width eq 20".format( typeName), safe='= ').replace(' ', '+') r = requests.get(legend_url + legend_qs, headers=extraHeaders) legend = b64encode(r.content) params['legend'] = legend # dataset = self.constructDataset(name, params) folder = findDatasetFolder(user, user, create=True) dataset = Item().createItem(name, user, folder, '') updateMinervaMetadata(dataset, params) return dataset
def createGeojsonDataset(self, item, params): user = self.getCurrentUser() folder = findDatasetFolder(user, user, create=True) if folder is None: raise RestException('User has no Minerva Dataset folder.') if folder['_id'] != item['folderId']: raise RestException("Items need to be in user's Minerva Dataset " + "folder.") minerva_metadata = { 'original_type': 'geojson', 'dataset_type': 'geojson', } # Use the first geojson or json file found as the dataset. for file in self.model('item').childFiles(item=item, limit=0): if ('geojson' in file['exts'] or 'json' in file['exts'] or file.get('mimeType') in ( 'application/json', 'application/vnd.geo+json', )): minerva_metadata['original_files'] = [{ 'name': file['name'], '_id': file['_id']}] minerva_metadata['geojson_file'] = { 'name': file['name'], '_id': file['_id']} minerva_metadata['geo_render'] = { 'type': 'geojson', 'file_id': file['_id']} minerva_metadata['original_type'] = 'geojson' minerva_metadata['source'] = { 'layer_source': 'GeoJSON'} minerva_metadata['source_type'] = 'item' break if 'geojson_file' not in minerva_metadata: raise RestException('Item contains no geojson file.') updateMinervaMetadata(item, minerva_metadata) return item
def testCheckDatasetsExistInDatabase(self): with HTTMock(wms_mock): response = self.request(path=self._path, method='POST', params=self._params, user=self._user) self.assertStatusOk(response) from girder.plugins.minerva.utility.minerva_utility import findDatasetFolder dataset_folder = findDatasetFolder(self._user, self._user) items = self.model('folder').childItems(dataset_folder) db_datasets = [i['meta']['minerva']['type_name'] for i in items] wmsSource = response.json response_datasets = [ d['meta']['minerva']['type_name'] for d in wmsSource ] self.assertEquals(len(db_datasets), len(response_datasets), 'Number of datasets do not match with thedb') self.assertEquals(set(db_datasets), set(response_datasets), 'Dataset type_names do not math with the db')
def createBsveDataset(user, bsveRoot, extraHeaders, params, layer_type): typeName = params['type_name'] try: if params['metadata']: layer_info = params['metadata'] else: layer_info = BsveWmsStyle( typeName, bsveRoot=bsveRoot, extraHeaders=extraHeaders).get_layer_info(layer_type) except TypeError: layer_info = "" # TODO: Add the legend url here once it is # ready on bsve side name = params['name'] params['layer_info'] = layer_info params['adapter'] = 'bsve' legend_url = bsveRoot + "/data/v2/sources/geotiles/data/result?" legend_qs = quote( "$filter=name eq {} and request eq getlegendgraphic and height eq 20 and width eq 20" .format(typeName), safe='= ').replace(' ', '+') r = requests.get(legend_url + legend_qs, headers=extraHeaders) legend = b64encode(r.content) params['legend'] = legend # dataset = self.constructDataset(name, params) folder = findDatasetFolder(user, user, create=True) dataset = Item().createItem(name, user, folder, '') updateMinervaMetadata(dataset, params) return dataset
def createExternalMongo(self, name, dbConnectionUri, collectionName): # assuming to create in the user space of the current user user = self.getCurrentUser() folder = findDatasetFolder(user, user) desc = 'external mongo dataset for %s' % name item = self.model('item').createItem(name, user, folder, desc) minerva_metadata = { 'dataset_id': item['_id'], 'original_type': 'mongo', 'mongo_connection': { 'db_uri': dbConnectionUri, 'collection_name': collectionName } } # get the first entry in the collection, set as json_row # TODO integrate this with the methods for taking a row from a JSON # array in a file collection = self.mongoCollection(dbConnectionUri, collectionName) collectionList = list(collection.find(limit=1)) if len(collectionList) > 0: minerva_metadata['json_row'] = collectionList[0] else: minerva_metadata['json_row'] = None if 'meta' not in item: item['meta'] = {} item['meta']['minerva'] = minerva_metadata self.model('item').setMetadata(item, item['meta']) return item['meta']['minerva']
def unshareDataset(self, item, params): currentUser = self.getCurrentUser() datasetFolder = findDatasetFolder(currentUser, currentUser, create=True) self.model('item').move(item, datasetFolder) return self.model('item').filter(item, currentUser)
def getGeometryLinkTarget(self, params): currentUser = self.getCurrentUser() folder = findDatasetFolder(currentUser, currentUser) items = list(self.model('item').find( query={'folderId': folder['_id'], 'meta.minerva.dataset_type': 'geojson'}, fields=['name'])) return items
def constructDataset(self, name, minerva_metadata, desc=''): user = self.getCurrentUser() folder = findDatasetFolder(user, user, create=True) if folder is None: raise Exception('User has no Minerva Dataset folder.') dataset = self.model('item').createItem(name, user, folder, desc) updateMinervaMetadata(dataset, minerva_metadata) return dataset
def constructDataset(self, name, minerva_metadata, desc=''): user = self.getCurrentUser() folder = findDatasetFolder(user, user, create=True) if folder is None: raise Exception('User has no Minerva Dataset folder.') dataset = self.model('item').createItem(name, user, folder, desc) updateMinervaMetadata(dataset, minerva_metadata) return dataset
def getGeometryLinkTarget(self, params): currentUser = self.getCurrentUser() folder = findDatasetFolder(currentUser, currentUser) items = list( self.model('item').find(query={ 'folderId': folder['_id'], 'meta.minerva.dataset_type': 'geojson' }, fields=['name'])) return items
def listDatasets(self, user, params): folder = findDatasetFolder(self.getCurrentUser(), user) if folder is None: return [] else: limit, offset, sort = \ self.getPagingParameters(params, defaultSortDir=pymongo.DESCENDING) items = [self.model('item').filter(item, self.getCurrentUser()) for item in self.model('folder').childItems(folder, limit=limit, offset=offset, sort=sort)] return items
def createGeojsonDataset(self, item, params, postgresGeojson=None): user = self.getCurrentUser() folder = findDatasetFolder(user, user, create=True) if folder is None: raise RestException('User has no Minerva Dataset folder.') if folder['_id'] != item['folderId']: raise RestException("Items need to be in user's Minerva Dataset " + "folder.") minerva_metadata = { 'original_type': 'geojson', 'dataset_type': 'geojson', } # Use the first geojson or json file found as the dataset. for file in self.model('item').childFiles(item=item, limit=0): if ('geojson' in file['exts'] or 'json' in file['exts'] or file.get('mimeType') in ( 'application/json', 'application/vnd.geo+json', )): minerva_metadata['original_files'] = [{ 'name': file['name'], '_id': file['_id'] }] minerva_metadata['geojson_file'] = { 'name': file['name'], '_id': file['_id'] } minerva_metadata['geo_render'] = { 'type': 'geojson', 'file_id': file['_id'] } minerva_metadata['original_type'] = 'geojson' minerva_metadata['source'] = {'layer_source': 'GeoJSON'} minerva_metadata['source_type'] = 'item' if postgresGeojson is not None: if postgresGeojson['field'] is not None: minerva_metadata['visProperties'] = { 'line': { "fillColorKey": postgresGeojson['field'] }, 'polygon': { "fillColorKey": postgresGeojson['field'] }, 'point': { "fillColorKey": postgresGeojson['field'] } } minerva_metadata['postgresGeojson'] = postgresGeojson break if 'geojson_file' not in minerva_metadata: raise RestException('Item contains no geojson file.') updateMinervaMetadata(item, minerva_metadata) return item
def promoteItemToDataset(self, item, params): """ Take an Item in the user's Minerva Dataset folder, and promote it to a Minerva Dataset by adding proper Minerva metadata. """ user = self.getCurrentUser() folder = findDatasetFolder(user, user, create=True) if folder is None: raise RestException('User has no Minerva Dataset folder.') if folder['_id'] != item['folderId']: raise RestException("Items need to be in user's Minerva Dataset " + "folder.") # Don't overwrite if minerva metadata already exists. if 'meta' in item and 'minerva' in item['meta']: return item minerva_metadata = {'source_type': 'item'} for file in self.model('item').childFiles(item=item, limit=0): # TODO This switching based on which file is found first is # fairly brittle and should only be called after first upload. if 'geojson' in file['exts']: # we found a geojson, assume this is geojson original minerva_metadata['original_type'] = 'geojson' minerva_metadata['dataset_type'] = 'geojson' minerva_metadata['original_files'] = [{ 'name': file['name'], '_id': file['_id'] }] minerva_metadata['geojson_file'] = { 'name': file['name'], '_id': file['_id'] } minerva_metadata['source'] = {'layer_source': 'GeoJSON'} break elif 'json' in file['exts']: minerva_metadata['original_type'] = 'json' minerva_metadata['dataset_type'] = 'json' minerva_metadata['original_files'] = [{ 'name': file['name'], '_id': file['_id'] }] break elif 'csv' in file['exts']: minerva_metadata['original_type'] = 'csv' minerva_metadata['dataset_type'] = 'csv' minerva_metadata['original_files'] = [{ 'name': file['name'], '_id': file['_id'] }] break updateMinervaMetadata(item, minerva_metadata) return item
def listDatasets(self, user, params): folder = findDatasetFolder(self.getCurrentUser(), user) if folder is None: return [] else: limit, offset, sort = \ self.getPagingParameters(params, defaultSortDir=pymongo.DESCENDING) items = [self.model('item').filter(item, self.getCurrentUser()) for item in self.model('folder').childItems(folder, limit=limit, offset=offset, sort=sort)] return items
def createMinervaDataset(self, geojsonString, name): """Creates a dataset from a geojson string""" output = StringIO.StringIO(json.dumps(geojsonString)) outputSize = output.len user = self.getCurrentUser() datasetFolder = findDatasetFolder(user, user, create=True) itemModel = ModelImporter.model('item') uploadModel = ModelImporter.model('upload') item = itemModel.createItem(name, user, datasetFolder) geojsonFile = uploadModel.uploadFromFile(output, outputSize, name, 'item', item, user) GeojsonDataset().createGeojsonDataset(itemId=geojsonFile['itemId'], params={}) return geojsonFile
def createMinervaDataset(self, geojsonString, name): """Creates a dataset from a geojson string""" output = StringIO.StringIO(json.dumps(geojsonString)) outputSize = output.len user = self.getCurrentUser() datasetFolder = findDatasetFolder(user, user, create=True) itemModel = ModelImporter.model('item') uploadModel = ModelImporter.model('upload') item = itemModel.createItem(name, user, datasetFolder) geojsonFile = uploadModel.uploadFromFile(output, outputSize, name, 'item', item, user) GeojsonDataset().createGeojsonDataset(itemId=geojsonFile['itemId'], params={}) return geojsonFile
def promoteItemToDataset(self, item, params): """ Take an Item in the user's Minerva Dataset folder, and promote it to a Minerva Dataset by adding proper Minerva metadata. """ user = self.getCurrentUser() folder = findDatasetFolder(user, user, create=True) if folder is None: raise RestException('User has no Minerva Dataset folder.') if folder['_id'] != item['folderId']: raise RestException("Items need to be in user's Minerva Dataset " + "folder.") # Don't overwrite if minerva metadata already exists. if 'meta' in item and 'minerva' in item['meta']: return item minerva_metadata = { 'source_type': 'item' } for file in self.model('item').childFiles(item=item, limit=0): # TODO This switching based on which file is found first is # fairly brittle and should only be called after first upload. if 'geojson' in file['exts']: # we found a geojson, assume this is geojson original minerva_metadata['original_type'] = 'geojson' minerva_metadata['dataset_type'] = 'geojson' minerva_metadata['original_files'] = [{ 'name': file['name'], '_id': file['_id']}] minerva_metadata['geojson_file'] = { 'name': file['name'], '_id': file['_id']} break elif 'json' in file['exts']: minerva_metadata['original_type'] = 'json' minerva_metadata['dataset_type'] = 'json' minerva_metadata['original_files'] = [{ 'name': file['name'], '_id': file['_id']}] break elif 'csv' in file['exts']: minerva_metadata['original_type'] = 'csv' minerva_metadata['dataset_type'] = 'csv' minerva_metadata['original_files'] = [{ 'name': file['name'], '_id': file['_id']}] break updateMinervaMetadata(item, minerva_metadata) return item
def queryElasticsearch(self, params): """ Creates a local job to run the elasticsearch_worker, the job will store the results of the elastic search query in a dataset. """ currentUser = self.getCurrentUser() datasetName = params['datasetName'] elasticsearchParams = params['searchParams'] datasetFolder = findDatasetFolder(currentUser, currentUser) dataset = (self.model('item').createItem( datasetName, currentUser, datasetFolder, 'created by elasticsearch query')) user, token = self.getCurrentUser(returnToken=True) kwargs = { 'params': params, 'user': currentUser, 'dataset': dataset, 'token': token, 'sourceId': params['sourceId'] } job = self.model('job', 'jobs').createLocalJob( title='elasticsearch: %s' % datasetName, user=currentUser, type='elasticsearch', public=False, kwargs=kwargs, module='girder.plugins.minerva.jobs.elasticsearch_worker', async=True) minerva_metadata = { 'dataset_type': 'json', 'source_id': params['sourceId'], 'source': 'elasticsearch', 'elasticsearch_params': elasticsearchParams } updateMinervaMetadata(dataset, minerva_metadata) self.model('job', 'jobs').scheduleJob(job) return job
def getLayers(user): folder = findDatasetFolder(user, user) if not folder: return [] items = Folder().childItems(folder) layers = {} for item in items: adapter = item.get('meta', {}).get('minerva', {}).get('adapter') name = item.get('meta', {}).get('minerva', {}).get('type_name') if adapter == 'bsve': # delete duplicates if they exist if name in layers: Item().remove(item) else: layers[name] = item return layers
def getLayers(user): folder = findDatasetFolder( user, user ) if not folder: return [] items = Folder().childItems(folder) layers = {} for item in items: adapter = item.get('meta', {}).get('minerva', {}).get('adapter') name = item.get('meta', {}).get('minerva', {}).get('type_name') if adapter == 'bsve': # delete duplicates if they exist if name in layers: Item().remove(item) else: layers[name] = item return layers
def promoteItemToDataset(self, item, params): """ Take an Item in the user's Minerva Dataset folder, and promote it to a Minerva Dataset by adding proper Minerva metadata. """ user = self.getCurrentUser() folder = findDatasetFolder(user, user, create=True) if folder is None: raise RestException('User has no Minerva Dataset folder.') if folder['_id'] != item['folderId']: raise RestException("Items need to be in user's Minerva Dataset " + "folder.") # Don't overwrite if minerva metadata already exists. if 'meta' in item and 'minerva' in item['meta']: return item minerva_metadata = self._updateMinervaMetadata(item) bounds = self._getBound(item) if bounds: minerva_metadata['bounds'] = bounds updateMinervaMetadata(item, minerva_metadata) return item
def testCheckDatasetsExistInDatabase(self): with HTTMock(wms_mock): response = self.request(path=self._path, method='POST', params=self._params, user=self._user) self.assertStatusOk(response) from girder.plugins.minerva.utility.minerva_utility import findDatasetFolder dataset_folder = findDatasetFolder(self._user, self._user) items = self.model('folder').childItems(dataset_folder) db_datasets = [i['meta']['minerva']['type_name'] for i in items] wmsSource = response.json response_datasets = [d['meta']['minerva']['type_name'] for d in wmsSource] self.assertEquals(len(db_datasets), len(response_datasets), 'Number of datasets do not match with thedb') self.assertEquals(set(db_datasets), set(response_datasets), 'Dataset type_names do not math with the db')
def unshareDataset(self, item, params): currentUser = self.getCurrentUser() datasetFolder = findDatasetFolder(currentUser, currentUser, create=True) self.model('item').move(item, datasetFolder) return self.model('item').filter(item, currentUser)
def createDatasetFolder(self, user, params): folder = findDatasetFolder(self.getCurrentUser(), user, create=True) return {'folder': folder}
def getDatasetFolder(self, user, params): folder = findDatasetFolder(self.getCurrentUser(), user) return {'folder': folder}
def promoteItemToDataset(self, item, params): """ Take an Item in the user's Minerva Dataset folder, and promote it to a Minerva Dataset by adding proper Minerva metadata. """ user = self.getCurrentUser() folder = findDatasetFolder(user, user, create=True) if folder is None: raise RestException('User has no Minerva Dataset folder.') if folder['_id'] != item['folderId']: raise RestException("Items need to be in user's Minerva Dataset " + "folder.") # Don't overwrite if minerva metadata already exists. if 'meta' in item and 'minerva' in item['meta']: return item minerva_metadata = {'source_type': 'item'} for file in self.model('item').childFiles(item=item, limit=0): # Check the first few k of a file to see if this might be a # geojson timeseries. Crudely, we expect this to be a json array # which contains objects, each of which has at least a geojson # element. This test will fail if there are other elements in the # first object that push the geojson element beyond the tested # header length. It could give a false positive, too. The correct # way would be to download and parse the whole file, but that would # be more expensive in memory and time. headerLen = 2048 fileHeader = '' for headerData in self.model('file').download(file, headers=False, endByte=headerLen)(): fileHeader = (fileHeader + headerData)[:headerLen] if len(fileHeader) >= headerLen: break if (fileHeader.lstrip()[:1] == '[' and fileHeader.lstrip()[1:].lstrip()[:1] == '{' and '"geojson"' in fileHeader): minerva_metadata['original_type'] = 'geojson-timeseries' minerva_metadata['dataset_type'] = 'geojson-timeseries' minerva_metadata['original_files'] = [{ 'name': file['name'], '_id': file['_id'] }] minerva_metadata['geojson_file'] = { 'name': file['name'], '_id': file['_id'] } minerva_metadata['source'] = {'layer_source': 'GeoJSON'} break # TODO This switching based on which file is found first is # fairly brittle and should only be called after first upload. if 'geojson' in file['exts']: # we found a geojson, assume this is geojson original minerva_metadata['original_type'] = 'geojson' minerva_metadata['dataset_type'] = 'geojson' minerva_metadata['original_files'] = [{ 'name': file['name'], '_id': file['_id'] }] minerva_metadata['geojson_file'] = { 'name': file['name'], '_id': file['_id'] } minerva_metadata['source'] = {'layer_source': 'GeoJSON'} break elif 'json' in file['exts']: minerva_metadata['original_type'] = 'json' minerva_metadata['dataset_type'] = 'json' minerva_metadata['original_files'] = [{ 'name': file['name'], '_id': file['_id'] }] break elif 'csv' in file['exts']: minerva_metadata['original_type'] = 'csv' minerva_metadata['dataset_type'] = 'csv' minerva_metadata['original_files'] = [{ 'name': file['name'], '_id': file['_id'] }] break elif ({'tif', 'tiff'}.intersection(file['exts']) and file['mimeType'] == 'image/tiff'): info = getInfo(file) if 'srs' in info and info['srs']: minerva_metadata['original_type'] = 'tiff' minerva_metadata['dataset_type'] = 'geotiff' minerva_metadata['original_files'] = [{ 'name': file['name'], '_id': file['_id'] }] minerva_metadata['source'] = {'layer_source': 'Tiff'} break updateMinervaMetadata(item, minerva_metadata) bounds = self._getBound(item) if bounds: minerva_metadata['bounds'] = bounds updateMinervaMetadata(item, minerva_metadata) return item
def bsveSearchAnalysis(self, params): currentUser = self.getCurrentUser() datasetName = params['datasetName'] bsveSearchParams = params['bsveSearchParams'] analysis = findAnalysisByName(currentUser, 'bsve search') # TODO in case can't find analysis? try: bsveSearchParams = json.loads(bsveSearchParams) except ValueError: raise RestException('bsveSearchParams is invalid JSON.') datasetFolder = findDatasetFolder(currentUser, currentUser) # TODO # try findOne earlier # all throughout utility # create a new dataset in the dataset folder with this name # TODO in case of duplicates? dataset = (self.model('item').createItem(datasetName, currentUser, datasetFolder, 'created by bsve search')) params = { 'bsveSearchParams': bsveSearchParams } # create a local job with bsve search # tie in the dataset id with the local job # TODO would we rather create the dataset at the end of the bsve search? # TODO change token to job token user, token = self.getCurrentUser(returnToken=True) kwargs = { 'params': params, 'user': currentUser, 'dataset': dataset, 'analysis': analysis, 'token': token } job = self.model('job', 'jobs').createLocalJob( title='bsve search: %s' % datasetName, user=currentUser, type='bsve.search', public=False, kwargs=kwargs, module='girder.plugins.minerva.jobs.bsve_search_worker', async=True) if 'meta' in dataset: metadata = dataset['meta'] else: metadata = {} minerva_metadata = { 'dataset_id': dataset['_id'], 'source': 'bsve_search', 'bsve_search_params': bsveSearchParams, 'original_type': 'json' } metadata['minerva'] = minerva_metadata self.model('item').setMetadata(dataset, metadata) self.model('job', 'jobs').scheduleJob(job) return minerva_metadata
def createPostgresGeojsonDataset(self, assetstore, params): filter = params['filter'] table = params['table'] field = params['field'] aggregateFunction = params['aggregateFunction'] geometryField = json.loads(params['geometryField']) if geometryField['type'] == 'built-in': buildInGeomField = geometryField['field'] properties = [ field, { 'func': aggregateFunction, 'param': { 'field': field } } ] # add string fields with concat aggregate function and in the format for # json_build_object for i in self._getColumns(assetstore, {'table': params['table']}): if i['datatype'] == 'string' and i['name'] != field: properties.extend((i['name'], { 'func': 'string_agg', 'param': [{ 'func': 'distinct', 'param': { 'field': i['name'] } }, '|'], 'reference': i['name'] })) fields = [{ 'func': 'json_build_object', 'param': [ 'type', 'Feature', 'geometry', { 'func': 'cast', 'param': [{ 'func': 'st_asgeojson', 'param': [{ 'func': 'st_transform', 'param': [{ 'field': buildInGeomField }, 4326] }] }, 'JSON'] }, 'properties', { 'func': 'json_build_object', 'param': properties } ] }] group = [buildInGeomField] elif geometryField['type'] == 'link': fields = [{ 'func': aggregateFunction, 'param': { 'field': field }, 'reference': field }] group = [x['value'] for x in geometryField['links']] # add string fields with concat aggregate function for i in self._getColumns(assetstore, {'table': params['table']}): if i['datatype'] in ('string', 'number', 'date') and i['name'] != field: if i['datatype'] == 'string': fields.append({ 'func': 'string_agg', 'param': [{ 'func': 'distinct', 'param': { 'field': i['name'] } }, '|'], 'reference': i['name'] }) datasetName = params['datasetName'] # TODO: schema should be read from the listed table, not set explicitly schema = 'public' hash = hashlib.md5(filter).hexdigest() if datasetName: output_name = datasetName else: output_name = '{0}.{1}.{2}.geojson'.format(table, field, hash[-6:]) currentUser = self.getCurrentUser() datasetFolder = findDatasetFolder(currentUser, currentUser) adapter = assetstore_utilities.getAssetstoreAdapter(assetstore) # Create the item dbParams = self._getQueryParams( schema, table, fields, group, filter, 'GeoJSON' if geometryField['type'] == 'built-in' else 'json') dbParams['tables'][0]['name'] = output_name result = adapter.importData(datasetFolder, 'folder', dbParams, progress.noProgress, currentUser) resItem = result[0]['item'] GeojsonDataset().createGeojsonDataset(itemId=resItem['_id'], postgresGeojson={ 'geometryField': geometryField, 'field': field, 'aggregateFunction': aggregateFunction }, params={}) return resItem['_id']
def createDatasetFolder(self, user, params): folder = findDatasetFolder(self.getCurrentUser(), user, create=True) return {'folder': folder}
def createPostgresGeojsonDataset(self, assetstore, params): filter = params['filter'] table = params['table'] field = params['field'] aggregateFunction = params['aggregateFunction'] geometryField = json.loads(params['geometryField']) if geometryField['type'] == 'built-in': buildInGeomField = geometryField['field'] properties = [field, { 'func': aggregateFunction, 'param': {'field': field} }] # add string fields with concat aggregate function and in the format for # json_build_object for i in self._getColumns(assetstore, {'table': params['table']}): if i['datatype'] == 'string' and i['name'] != field: properties.extend((i['name'], { 'func': 'string_agg', 'param': [{ 'func': 'distinct', 'param': {'field': i['name']} }, '|'], 'reference': i['name'] })) fields = [{ 'func': 'json_build_object', 'param': [ 'type', 'Feature', 'geometry', { 'func': 'cast', 'param': [{ 'func': 'st_asgeojson', 'param': [{ 'func': 'st_transform', 'param': [{'field': buildInGeomField}, 4326] }]}, 'JSON'] }, 'properties', { 'func': 'json_build_object', 'param': properties } ] }] group = [buildInGeomField] elif geometryField['type'] == 'link': fields = [{ 'func': aggregateFunction, 'param': {'field': field}, 'reference': field }] group = [x['value'] for x in geometryField['links']] # add string fields with concat aggregate function for i in self._getColumns(assetstore, {'table': params['table']}): if i['datatype'] in ('string', 'number', 'date') and i['name'] != field: if i['datatype'] == 'string': fields.append({ 'func': 'string_agg', 'param': [{ 'func': 'distinct', 'param': {'field': i['name']} }, '|'], 'reference': i['name'] }) datasetName = params['datasetName'] # TODO: schema should be read from the listed table, not set explicitly schema = 'public' hash = hashlib.md5(filter).hexdigest() if datasetName: output_name = datasetName else: output_name = '{0}.{1}.{2}.geojson'.format( table, field, hash[-6:]) currentUser = self.getCurrentUser() datasetFolder = findDatasetFolder(currentUser, currentUser) adapter = assetstore_utilities.getAssetstoreAdapter(assetstore) # Create the item dbParams = self._getQueryParams( schema, table, fields, group, filter, 'GeoJSON' if geometryField['type'] == 'built-in' else 'json') dbParams['tables'][0]['name'] = output_name result = adapter.importData(datasetFolder, 'folder', dbParams, progress.noProgress, currentUser) resItem = result[0]['item'] GeojsonDataset().createGeojsonDataset( itemId=resItem['_id'], postgresGeojson={ 'geometryField': geometryField, 'field': field, 'aggregateFunction': aggregateFunction }, params={}) return resItem['_id']