def _convertJsonfileToGeoJson(self, item, tmpdir):
# use the first filename with json ext found in original_files
filename = None
files = item['meta']['minerva']['original_files']
for f in files:
if f['name'].endswith('.json'):
filename = f['name']
if filename is None:
raise RestException('Dataset %s has no json files' % item['name'])
jsonFilepath = os.path.join(tmpdir, item['name'], filename)
geoJsonFilename = item['name'] + PluginSettings.GEOJSON_EXTENSION
geoJsonFilepath = os.path.join(tmpdir, item['name'], geoJsonFilename)
mapping = item['meta']['minerva']['mapper']
geoJsonMapper = GeoJsonMapper(objConverter=None, mapping=mapping)
objects = jsonObjectReader(jsonFilepath)
geoJsonMapper.mapToJsonFile(tmpdir, objects, geoJsonFilepath)
return geoJsonFilepath
def _convertJsonfileToGeoJson(self, item, tmpdir):
# use the first filename with json ext found in original_files
filename = None
files = item['meta']['minerva']['original_files']
for f in files:
if f['name'].endswith('.json'):
filename = f['name']
if filename is None:
raise RestException('Dataset %s has no json files' % item['name'])
jsonFilepath = os.path.join(tmpdir, item['name'],
filename)
geoJsonFilename = item['name'] + PluginSettings.GEOJSON_EXTENSION
geoJsonFilepath = os.path.join(tmpdir, item['name'], geoJsonFilename)
mapping = item['meta']['minerva']['mapper']
geoJsonMapper = GeoJsonMapper(objConverter=None,
mapping=mapping)
objects = jsonObjectReader(jsonFilepath)
geoJsonMapper.mapToJsonFile(tmpdir, objects, geoJsonFilepath)
return geoJsonFilepath
def _convertMongoToGeoJson(self, item, params):
minerva_metadata = item['meta']['minerva']
# TODO time to break this code out to another method
# TODO maybe here we can store limit and offset?
# set a placeholder for geojson file, even though
# there is no actual geojson file, rather we will
# generate it on the fly
# in this case we don't actually want to store a file
# but store the metadata we used to create the geojson
if 'geojson' not in minerva_metadata:
minerva_metadata['geojson'] = {}
# TODO no reason couldn't have query and limit/offset
# add the geojson to the minerva metadata returned
# but don't save it
# TODO think on caching implications
connection = minerva_metadata['mongo_connection']
dbConnectionUri = connection['db_uri']
collectionName = connection['collection_name']
collection = self.mongoCollection(dbConnectionUri, collectionName)
query_count = collection.find().count()
minerva_metadata['geojson']['query_count'] = query_count
objects = collection.find()
mapping = item['meta']['minerva']['mapper']
geoJsonMapper = GeoJsonMapper(objConverter=None,
mapping=mapping)
import cStringIO
writer = cStringIO.StringIO()
geoJsonMapper.mapToJson(objects, writer)
item['meta']['minerva'] = minerva_metadata
item['meta']['minerva']['geojson']['data'] = writer.getvalue()
self.model('item').setMetadata(item, item['meta'])
return minerva_metadata
def _convertMongoToGeoJson(self, item, params):
minerva_metadata = item['meta']['minerva']
# TODO time to break this code out to another method
# TODO maybe here we can store limit and offset?
# set a placeholder for geojson file, even though
# there is no actual geojson file, rather we will
# generate it on the fly
# in this case we don't actually want to store a file
# but store the metadata we used to create the geojson
if 'geojson' not in minerva_metadata:
minerva_metadata['geojson'] = {}
# TODO no reason couldn't have query and limit/offset
# add the geojson to the minerva metadata returned
# but don't save it
# TODO think on caching implications
connection = minerva_metadata['mongo_connection']
dbConnectionUri = connection['db_uri']
collectionName = connection['collection_name']
collection = self.mongoCollection(dbConnectionUri, collectionName)
query_count = collection.find().count()
minerva_metadata['geojson']['query_count'] = query_count
objects = collection.find()
mapping = item['meta']['minerva']['mapper']
geoJsonMapper = GeoJsonMapper(objConverter=None,
mapping=mapping)
import cStringIO
writer = cStringIO.StringIO()
geoJsonMapper.mapToJson(objects, writer)
item['meta']['minerva'] = minerva_metadata
item['meta']['minerva']['geojson']['data'] = writer.getvalue()
self.model('item').setMetadata(item, item['meta'])
return minerva_metadata
def _convertMongoToGeoJson(self, item, params):
minerva_metadata = item['meta']['minerva']
# TODO time to break this code out to another method
# TODO maybe here we can store limit and offset?
# set a placeholder for geojson file, even though
# there is no actual geojson file, rather we will
# generate it on the fly
# in this case we don't actually want to store a file
# but store the metadata we used to create the geojson
# Look for datetime limits, if none, look for limit and offset
# use 50/0 as defaults
metadataQuery = {}
if 'dateField' in params:
dateField = params['dateField']
metadataQuery = {dateField: {}}
else:
if 'startTime' in params or 'endTime' in params:
raise RestException('dateField param required for startTime ' +
'or endTime param')
query = {}
if 'startTime' in params:
startTime = params['startTime']
query[dateField] = {'$gte': int(startTime)}
metadataQuery[dateField]['startTime'] = startTime
if 'endTime' in params:
endTime = params['endTime']
dateFieldQuery = query.get(dateField, {})
dateFieldQuery['$lte'] = int(endTime)
query[dateField] = dateFieldQuery
metadataQuery[dateField]['endTime'] = endTime
minerva_metadata['geojson'] = {}
minerva_metadata['geojson']['query'] = metadataQuery
# TODO no reason couldn't have query and limit/offset
# add the geojson to the minerva metadata returned
# but don't save it
# TODO think on caching implications
connection = minerva_metadata['mongo_connection']
dbConnectionUri = connection['db_uri']
collectionName = connection['collection_name']
collection = self.mongoCollection(dbConnectionUri, collectionName)
query_count = collection.find(query).count()
minerva_metadata['geojson']['query_count'] = query_count
objects = collection.find(query)
mapping = item['meta']['minerva']['mapper']
geoJsonMapper = GeoJsonMapper(objConverter=None,
mapping=mapping)
import cStringIO
writer = cStringIO.StringIO()
geoJsonMapper.mapToJson(objects, writer)
item['meta']['minerva'] = minerva_metadata
self.model('item').setMetadata(item, item['meta'])
item['meta']['minerva']['geojson']['data'] = writer.getvalue()
return minerva_metadata
def run(job):
job_model = ModelImporter.model('job', 'jobs')
job_model.updateJob(job, status=JobStatus.RUNNING)
try:
configFile = os.path.join(os.path.dirname(__file__), "bsve.json")
if os.path.exists(configFile):
bsveConfig = json.load(open(configFile))['bsve']
else:
bsveConfig = {}
kwargs = job['kwargs']
bsveSearchParams = kwargs['params']['bsveSearchParams']
datasetId = str(kwargs['dataset']['_id'])
# TODO better to create a job token rather than a user token?
token = kwargs['token']
bsveUtility = BsveUtility(
user=bsveConfig.get(
'USER_NAME', os.environ.get('BSVE_USERNAME')),
apikey=bsveConfig.get(
'API_KEY', os.environ.get('BSVE_APIKEY')),
secret=bsveConfig.get(
'SECRET_KEY', os.environ.get('BSVE_SECRETKEY')),
base=bsveConfig.get('BASE_URL')
)
# TODO sleeping in async thread, probably starving other tasks
# would be better to split this into two or more parts, creating
# additional jobs as needed
searchResult = bsveUtility.search(bsveSearchParams)
# write the output to a json file
tmpdir = tempfile.mkdtemp()
outFilepath = tempfile.mkstemp(suffix='.json', dir=tmpdir)[1]
writer = open(outFilepath, 'w')
writer.write(json.dumps(searchResult))
writer.close()
# rename the file so it will have the right name when uploaded
# could probably be done post upload
outFilename = 'search.json'
humanFilepath = os.path.join(tmpdir, outFilename)
shutil.move(outFilepath, humanFilepath)
# connect to girder and upload the file
# TODO will probably have to change this from local to girder worker
# so that can work on worker machine
# at least need host connection info
girderPort = config.getConfig()['server.socket_port']
client = girder_client.GirderClient(port=girderPort)
client.token = token['_id']
client.uploadFileToItem(datasetId, humanFilepath)
# TODO some stuff here using models will only work on a local job
# will have to be rewritten using girder client to work in girder worker
# non-locally
user_model = ModelImporter.model('user')
user = user_model.load(job['userId'], force=True)
item_model = ModelImporter.model('item')
dataset = item_model.load(datasetId, level=AccessType.WRITE, user=user)
minerva_metadata = mM(dataset)
file_model = ModelImporter.model('file')
existing = file_model.findOne({
'itemId': dataset['_id'],
'name': outFilename
})
if existing:
minerva_metadata['original_files'] = [{
'_id': existing['_id'],
'name': outFilename
}]
else:
raise (Exception('Cannot find file %s in dataset %s' %
(outFilename, datasetId)))
jsonRow = jsonArrayHead(humanFilepath, limit=1)[0]
minerva_metadata['json_row'] = jsonRow
# Generate the geojson for this dataset and set
# dataset_type = geojson
geojsonFilename = 'search.geojson'
geojsonFilepath = os.path.join(tmpdir, geojsonFilename)
mapping = {
"dateKeypath": "",
"latitudeKeypath": "data.Latitude",
"longitudeKeypath": "data.Longitude"
}
geojsonMapper = GeoJsonMapper(objConverter=None, mapping=mapping)
objects = jsonObjectReader(humanFilepath)
geojsonMapper.mapToJsonFile(tmpdir, objects, geojsonFilepath)
client.uploadFileToItem(datasetId, geojsonFilepath)
shutil.rmtree(tmpdir)
minerva_metadata['mapper'] = mapping
minerva_metadata['dataset_type'] = 'geojson'
existing = file_model.findOne({
'itemId': dataset['_id'],
'name': geojsonFilename
})
if existing:
minerva_metadata['geojson_file'] = {
'_id': existing['_id'],
'name': geojsonFilename
}
else:
raise (Exception('Cannot find file %s in dataset %s' %
(geojsonFilename, datasetId)))
mM(dataset, minerva_metadata)
job_model.updateJob(job, status=JobStatus.SUCCESS)
except Exception:
t, val, tb = sys.exc_info()
log = '%s: %s\n%s' % (t.__name__, repr(val), traceback.extract_tb(tb))
# TODO only works locally
job_model.updateJob(job, status=JobStatus.ERROR, log=log)
raise
