def _convertJsonfileToGeoJson(self, item, tmpdir): # use the first filename with json ext found in original_files filename = None files = item['meta']['minerva']['original_files'] for f in files: if f['name'].endswith('.json'): filename = f['name'] if filename is None: raise RestException('Dataset %s has no json files' % item['name']) jsonFilepath = os.path.join(tmpdir, item['name'], filename) geoJsonFilename = item['name'] + PluginSettings.GEOJSON_EXTENSION geoJsonFilepath = os.path.join(tmpdir, item['name'], geoJsonFilename) mapping = item['meta']['minerva']['mapper'] geoJsonMapper = GeoJsonMapper(objConverter=None, mapping=mapping) objects = jsonObjectReader(jsonFilepath) geoJsonMapper.mapToJsonFile(tmpdir, objects, geoJsonFilepath) return geoJsonFilepath
def _convertJsonfileToGeoJson(self, item, tmpdir): # use the first filename with json ext found in original_files filename = None files = item['meta']['minerva']['original_files'] for f in files: if f['name'].endswith('.json'): filename = f['name'] if filename is None: raise RestException('Dataset %s has no json files' % item['name']) jsonFilepath = os.path.join(tmpdir, item['name'], filename) geoJsonFilename = item['name'] + PluginSettings.GEOJSON_EXTENSION geoJsonFilepath = os.path.join(tmpdir, item['name'], geoJsonFilename) mapping = item['meta']['minerva']['mapper'] geoJsonMapper = GeoJsonMapper(objConverter=None, mapping=mapping) objects = jsonObjectReader(jsonFilepath) geoJsonMapper.mapToJsonFile(tmpdir, objects, geoJsonFilepath) return geoJsonFilepath
def _convertMongoToGeoJson(self, item, params): minerva_metadata = item['meta']['minerva'] # TODO time to break this code out to another method # TODO maybe here we can store limit and offset? # set a placeholder for geojson file, even though # there is no actual geojson file, rather we will # generate it on the fly # in this case we don't actually want to store a file # but store the metadata we used to create the geojson if 'geojson' not in minerva_metadata: minerva_metadata['geojson'] = {} # TODO no reason couldn't have query and limit/offset # add the geojson to the minerva metadata returned # but don't save it # TODO think on caching implications connection = minerva_metadata['mongo_connection'] dbConnectionUri = connection['db_uri'] collectionName = connection['collection_name'] collection = self.mongoCollection(dbConnectionUri, collectionName) query_count = collection.find().count() minerva_metadata['geojson']['query_count'] = query_count objects = collection.find() mapping = item['meta']['minerva']['mapper'] geoJsonMapper = GeoJsonMapper(objConverter=None, mapping=mapping) import cStringIO writer = cStringIO.StringIO() geoJsonMapper.mapToJson(objects, writer) item['meta']['minerva'] = minerva_metadata item['meta']['minerva']['geojson']['data'] = writer.getvalue() self.model('item').setMetadata(item, item['meta']) return minerva_metadata
def _convertMongoToGeoJson(self, item, params): minerva_metadata = item['meta']['minerva'] # TODO time to break this code out to another method # TODO maybe here we can store limit and offset? # set a placeholder for geojson file, even though # there is no actual geojson file, rather we will # generate it on the fly # in this case we don't actually want to store a file # but store the metadata we used to create the geojson if 'geojson' not in minerva_metadata: minerva_metadata['geojson'] = {} # TODO no reason couldn't have query and limit/offset # add the geojson to the minerva metadata returned # but don't save it # TODO think on caching implications connection = minerva_metadata['mongo_connection'] dbConnectionUri = connection['db_uri'] collectionName = connection['collection_name'] collection = self.mongoCollection(dbConnectionUri, collectionName) query_count = collection.find().count() minerva_metadata['geojson']['query_count'] = query_count objects = collection.find() mapping = item['meta']['minerva']['mapper'] geoJsonMapper = GeoJsonMapper(objConverter=None, mapping=mapping) import cStringIO writer = cStringIO.StringIO() geoJsonMapper.mapToJson(objects, writer) item['meta']['minerva'] = minerva_metadata item['meta']['minerva']['geojson']['data'] = writer.getvalue() self.model('item').setMetadata(item, item['meta']) return minerva_metadata
def _convertMongoToGeoJson(self, item, params): minerva_metadata = item['meta']['minerva'] # TODO time to break this code out to another method # TODO maybe here we can store limit and offset? # set a placeholder for geojson file, even though # there is no actual geojson file, rather we will # generate it on the fly # in this case we don't actually want to store a file # but store the metadata we used to create the geojson # Look for datetime limits, if none, look for limit and offset # use 50/0 as defaults metadataQuery = {} if 'dateField' in params: dateField = params['dateField'] metadataQuery = {dateField: {}} else: if 'startTime' in params or 'endTime' in params: raise RestException('dateField param required for startTime ' + 'or endTime param') query = {} if 'startTime' in params: startTime = params['startTime'] query[dateField] = {'$gte': int(startTime)} metadataQuery[dateField]['startTime'] = startTime if 'endTime' in params: endTime = params['endTime'] dateFieldQuery = query.get(dateField, {}) dateFieldQuery['$lte'] = int(endTime) query[dateField] = dateFieldQuery metadataQuery[dateField]['endTime'] = endTime minerva_metadata['geojson'] = {} minerva_metadata['geojson']['query'] = metadataQuery # TODO no reason couldn't have query and limit/offset # add the geojson to the minerva metadata returned # but don't save it # TODO think on caching implications connection = minerva_metadata['mongo_connection'] dbConnectionUri = connection['db_uri'] collectionName = connection['collection_name'] collection = self.mongoCollection(dbConnectionUri, collectionName) query_count = collection.find(query).count() minerva_metadata['geojson']['query_count'] = query_count objects = collection.find(query) mapping = item['meta']['minerva']['mapper'] geoJsonMapper = GeoJsonMapper(objConverter=None, mapping=mapping) import cStringIO writer = cStringIO.StringIO() geoJsonMapper.mapToJson(objects, writer) item['meta']['minerva'] = minerva_metadata self.model('item').setMetadata(item, item['meta']) item['meta']['minerva']['geojson']['data'] = writer.getvalue() return minerva_metadata
def run(job): job_model = ModelImporter.model('job', 'jobs') job_model.updateJob(job, status=JobStatus.RUNNING) try: configFile = os.path.join(os.path.dirname(__file__), "bsve.json") if os.path.exists(configFile): bsveConfig = json.load(open(configFile))['bsve'] else: bsveConfig = {} kwargs = job['kwargs'] bsveSearchParams = kwargs['params']['bsveSearchParams'] datasetId = str(kwargs['dataset']['_id']) # TODO better to create a job token rather than a user token? token = kwargs['token'] bsveUtility = BsveUtility( user=bsveConfig.get( 'USER_NAME', os.environ.get('BSVE_USERNAME')), apikey=bsveConfig.get( 'API_KEY', os.environ.get('BSVE_APIKEY')), secret=bsveConfig.get( 'SECRET_KEY', os.environ.get('BSVE_SECRETKEY')), base=bsveConfig.get('BASE_URL') ) # TODO sleeping in async thread, probably starving other tasks # would be better to split this into two or more parts, creating # additional jobs as needed searchResult = bsveUtility.search(bsveSearchParams) # write the output to a json file tmpdir = tempfile.mkdtemp() outFilepath = tempfile.mkstemp(suffix='.json', dir=tmpdir)[1] writer = open(outFilepath, 'w') writer.write(json.dumps(searchResult)) writer.close() # rename the file so it will have the right name when uploaded # could probably be done post upload outFilename = 'search.json' humanFilepath = os.path.join(tmpdir, outFilename) shutil.move(outFilepath, humanFilepath) # connect to girder and upload the file # TODO will probably have to change this from local to girder worker # so that can work on worker machine # at least need host connection info girderPort = config.getConfig()['server.socket_port'] client = girder_client.GirderClient(port=girderPort) client.token = token['_id'] client.uploadFileToItem(datasetId, humanFilepath) # TODO some stuff here using models will only work on a local job # will have to be rewritten using girder client to work in girder worker # non-locally user_model = ModelImporter.model('user') user = user_model.load(job['userId'], force=True) item_model = ModelImporter.model('item') dataset = item_model.load(datasetId, level=AccessType.WRITE, user=user) minerva_metadata = mM(dataset) file_model = ModelImporter.model('file') existing = file_model.findOne({ 'itemId': dataset['_id'], 'name': outFilename }) if existing: minerva_metadata['original_files'] = [{ '_id': existing['_id'], 'name': outFilename }] else: raise (Exception('Cannot find file %s in dataset %s' % (outFilename, datasetId))) jsonRow = jsonArrayHead(humanFilepath, limit=1)[0] minerva_metadata['json_row'] = jsonRow # Generate the geojson for this dataset and set # dataset_type = geojson geojsonFilename = 'search.geojson' geojsonFilepath = os.path.join(tmpdir, geojsonFilename) mapping = { "dateKeypath": "", "latitudeKeypath": "data.Latitude", "longitudeKeypath": "data.Longitude" } geojsonMapper = GeoJsonMapper(objConverter=None, mapping=mapping) objects = jsonObjectReader(humanFilepath) geojsonMapper.mapToJsonFile(tmpdir, objects, geojsonFilepath) client.uploadFileToItem(datasetId, geojsonFilepath) shutil.rmtree(tmpdir) minerva_metadata['mapper'] = mapping minerva_metadata['dataset_type'] = 'geojson' existing = file_model.findOne({ 'itemId': dataset['_id'], 'name': geojsonFilename }) if existing: minerva_metadata['geojson_file'] = { '_id': existing['_id'], 'name': geojsonFilename } else: raise (Exception('Cannot find file %s in dataset %s' % (geojsonFilename, datasetId))) mM(dataset, minerva_metadata) job_model.updateJob(job, status=JobStatus.SUCCESS) except Exception: t, val, tb = sys.exc_info() log = '%s: %s\n%s' % (t.__name__, repr(val), traceback.extract_tb(tb)) # TODO only works locally job_model.updateJob(job, status=JobStatus.ERROR, log=log) raise