def register_dataset():
"""Register GEE Dataset Endpoint"""
logging.info('Registering new GEE Dataset')
# Build query
table_name = request.get_json().get('connector').get('table_name')
table_type = QueryService.get_type(table_name=table_name)
sql = '?sql=SELECT * FROM \"' + table_name + '\" LIMIT 1'
if table_type is 'raster':
sql = '?sql=SELECT ST_METADATA() from \"' + table_name + '\"'
# Convert query
query, json_sql = QueryService.convert(sql, query_type='sql')
try:
response = EarthEngineService.execute_query(query)
status = 1
except GEEQueryError as error:
logging.error(error.message)
status = 2
config = {
'uri': '/dataset/' + request.get_json().get('connector').get('id'),
'method': 'PATCH',
'body': {
'status': status
}
}
response = request_to_microservice(config)
return jsonify(response), 200
def register_dataset():
"""Register GEE Dataset Endpoint"""
logging.info('Registering new GEE Dataset')
table_name = request.get_json().get('connector').get('table_name')
sql = '?sql=SELECT * FROM \"' + table_name + '\" LIMIT 1'
# Convert query
json_sql = QueryService.convert(sql, query_type='sql')
try:
response = EarthEngineService.execute_query(json_sql).metadata
status = 1
except GEEQueryError as e:
logging.error('[ROUTER] /rest-datasets/gee - GEE Query error in GEE query execution: ' + e.message)
status = 2
except Exception as e:
logging.error('[ROUTER] /rest-datasets/gee - Generic error in GEE query execution: ' + str(e))
status = 2
config = {
'uri': '/dataset/' + request.get_json().get('connector').get('id'),
'method': 'PATCH',
'body': {'status': status}
}
response = request_to_microservice(config)
return jsonify(data=serialize_fields(response, table_name)), 200
def download(dataset_id):
"""Download GEE Dataset Endpoint"""
logging.info('Downloading GEE Dataset')
sql = request.args.get('sql', None) or request.get_json().get('sql', None)
result_query = build_query(request)
try:
if sql:
query_type = 'sql'
else:
query_type = 'fs'
json_sql = QueryService.convert(result_query, query_type=query_type)
except SqlFormatError as e:
logging.error('[ROUTER] /download/<dataset_id> - SQL Format error in query conversion: ' + e.message)
return error(status=400, detail=e.message)
except Exception as e:
logging.error('[ROUTER] /download/<dataset_id> - Generic error in query conversion: ' + str(e))
return error(status=500, detail='Generic Error')
try:
response = EarthEngineService.execute_query(json_sql).response()
except GEEQueryError as e:
logging.error('[ROUTER] /download/<dataset_id> - GEE Query error in GEE query execution: ' + e.message)
return error(status=400, detail=e.message)
except Exception as e:
logging.error('[ROUTER] /download/<dataset_id> - Generic error in GEE query execution: ' + str(e))
return error(status=500, detail='Generic Error')
# @TODO
meta = {}
# @TODO download content-type
return jsonify(data=serialize_query(response, meta)), 200
def generate_geojson():
yield '{"cloneUrl": ' + json.dumps(
QueryService.get_clone_url(dataset.get('id'), query)) + ','
if table_type is 'ft':
f_len = len(features)
yield '"data": [{ "type": "FeatureCollection", "features": ['
for idx, feature in enumerate(features):
geo_feature = {
"type": "Feature",
"properties": {},
"geometry": {}
}
for prop in feature:
if prop == 'the_geom':
geo_feature['geometry'] = feature['the_geom']
else:
geo_feature['properties'][prop] = feature[prop]
if idx != f_len - 1:
yield json.dumps(geo_feature) + ','
else:
yield json.dumps(geo_feature)
yield ']}]}'
elif table_type is 'raster':
yield '"data": ['
yield json.dumps(response)
yield ']}'
def query(dataset_id):
"""Query GEE Dataset Endpoint"""
logging.info('Doing GEE Query')
sql = request.args.get('sql', None) or request.get_json().get('sql', None)
result_query = build_query(request)
try:
if sql:
query_type = 'sql'
else:
query_type = 'fs'
json_sql = QueryService.convert(result_query, query_type=query_type)
except SqlFormatError as e:
logging.error('[ROUTER]: ' + e.message)
return error(status=400, detail=e.message)
except Exception as e:
logging.error('[ROUTER]: ' + str(e))
return error(status=500, detail='Generic Error')
try:
response = EarthEngineService.execute_query(json_sql).response()
except GEEQueryError as e:
logging.error('[ROUTER]: ' + e.message)
return error(status=400, detail=e.message)
except Exception as e:
logging.error('[ROUTER]: ' + str(e))
return error(status=500, detail='Generic Error')
# @TODO
meta = {}
return jsonify(serialize_query(response, meta)), 200
def query(dataset_id):
"""Query GEE Dataset Endpoint"""
logging.info('Doing GEE Query')
sql = request.args.get('sql', None) or request.get_json().get('sql', None)
result_query = build_query(request)
try:
if sql:
query_type = 'sql'
else:
query_type = 'fs'
json_sql = QueryService.convert(result_query, query_type=query_type)
except SqlFormatError as e:
logging.error('[ROUTER]: '+e.message)
return error(status=400, detail=e.message)
except Exception as e:
logging.error('[ROUTER]: '+str(e))
return error(status=500, detail="Error performing query")
try:
response = EarthEngineService.execute_query(json_sql).response()
except GEEQueryError as e:
logging.error('[ROUTER]: '+e.message)
return error(status=400, detail=e.message)
except Exception as e:
# hotfix until library developments
if str(e) == "'_init_cols'": return error(status=500, detail="Query not supported. Please visit https://doc.apihighways.org/ to obtain more information on supported queries.")
logging.error('[ROUTER]: '+str(e))
return error(status=500, detail=str(e))
# @TODO
meta = {}
return jsonify(serialize_query(response, meta)), 200
def download(dataset_id):
"""Download GEE Dataset Endpoint"""
logging.info('Downloading GEE Dataset')
sql = request.args.get('sql', None) or request.get_json().get('sql', None)
try:
result_query = build_query(request)
except TypeError as e:
return error(status=501, detail="Download not supported for this dataset. Please visit https://doc.apihighways.org/ to obtain more information on supported queries.")
try:
if sql:
query_type = 'sql'
else:
query_type = 'fs'
json_sql = QueryService.convert(result_query, query_type=query_type)
except SqlFormatError as e:
logging.error('[ROUTER]: '+e.message)
return error(status=400, detail=e.message)
except Exception as e:
logging.error('[ROUTER]: '+str(e))
return error(status=500, detail='Generic Error')
try:
response = EarthEngineService.execute_query(json_sql).response()
except GEEQueryError as e:
logging.error('[ROUTER]: '+e.message)
return error(status=400, detail=e.message)
except Exception as e:
logging.error('[ROUTER]: '+str(e))
# @TODO
meta = {}
# @TODO download content-type
return jsonify(data=serialize_query(response, meta)), 200
def fields(dataset_id):
"""Get GEE Dataset Fields Endpoint"""
logging.info('Getting fields of a GEE Dataset')
# Get and deserialize
dataset = DatasetResponder().deserialize(request.get_json())
# Build query
table_name = dataset.get('attributes').get('tableName')
table_type = QueryService.get_type(
dataset.get('attributes').get('tableName'))
if table_type is 'raster':
response = {'tableName': table_name, 'fields': {}}
return jsonify(response), 200
sql = '?sql=SELECT * FROM \"' + table_name + '\" LIMIT 1'
# Convert query
query, json_sql = QueryService.convert(sql, query_type='sql')
try:
response = EarthEngineService.execute_query(query)
except GEEQueryError as error:
logging.error(error.message)
response = ErrorResponder.build({
'status': 500,
'message': error.message
})
return jsonify(response), 500
fields = FieldsResponder().serialize(response.get('columns', {}))
response = FieldsResponder.build({
'tableName': table_name,
'fields': fields
})
return jsonify(response), 200
def generate_json():
yield '{"cloneUrl": ' + json.dumps(
QueryService.get_clone_url(dataset.get('id'), query)) + ','
if table_type is 'ft':
f_len = len(features)
yield '"data": ['
for idx, feature in enumerate(features):
if idx != f_len - 1:
yield json.dumps(feature) + ', '
else:
yield json.dumps(feature)
yield ']}'
elif table_type is 'raster':
yield '"data": ['
yield json.dumps(response)
yield ']}'
def fields(dataset_id):
"""Get GEE Dataset Fields Endpoint"""
logging.info('Getting fields of a GEE Dataset')
dataset = request.get_json().get('dataset').get('data')
table_name = dataset.get('attributes').get('tableName')
sql = '?sql=SELECT * FROM \"' + table_name + '\" LIMIT 1'
# Convert query
json_sql = QueryService.convert(sql, query_type='sql')
try:
response = EarthEngineService.execute_query(json_sql).metadata
except GEEQueryError as e:
logging.error('[ROUTER] /fields/<dataset_id> - GEE Query error in GEE query execution: ' + e.message)
return error(status=400, detail=e.message)
except Exception as e:
logging.error('[ROUTER] /fields/<dataset_id> - Generic error in GEE query execution: ' + str(e))
return error(status=500, detail='Generic Error')
return jsonify(data=serialize_fields(response, table_name)), 200
def query(dataset_id):
"""Query GEE Dataset Endpoint"""
logging.info('Doing GEE Query')
# Get and deserialize
dataset = DatasetResponder().deserialize(request.get_json())
table_type = QueryService.get_type(
dataset.get('attributes').get('tableName'))
sql = request.args.get('sql', None) or request.get_json().get('sql', None)
# request
result_query = form_query(sql, dataset, request)
# convert
try:
if sql:
query_type = 'sql'
else:
query_type = 'fs'
query, json_sql = QueryService.convert(result_query,
query_type=query_type)
except SqlFormatError as error:
logging.error(error.message)
response = ErrorResponder.build({
'status': 400,
'message': error.message
})
return jsonify(response), 400
except Exception as error:
response = ErrorResponder.build({
'status': 500,
'message': 'Generic Error'
})
return jsonify(response), 500
# geojson
geojson = request.get_json().get('geojson', None)
if not geojson:
try:
if query.index('ST_INTERSECTS'):
geojson = QueryService.get_geojson(json_sql)
except:
pass
# query
try:
response = EarthEngineService.execute_query(query, geojson=geojson)
except GEEQueryError as error:
logging.error(error.message)
response = ErrorResponder.build({
'status': 500,
'message': error.message
})
return jsonify(response), 500
except Exception as error:
response = ErrorResponder.build({
'status': 500,
'message': 'Generic Error'
})
return jsonify(response), 500
if table_type is 'ft':
if not isinstance(response, dict):
value = response
response = {'features': [{'FUNCTION': value}]}
features = QueryResponder().serialize(response.get('features', {}))
def generate_json():
yield '{"cloneUrl": ' + json.dumps(
QueryService.get_clone_url(dataset.get('id'), query)) + ','
if table_type is 'ft':
f_len = len(features)
yield '"data": ['
for idx, feature in enumerate(features):
if idx != f_len - 1:
yield json.dumps(feature) + ', '
else:
yield json.dumps(feature)
yield ']}'
elif table_type is 'raster':
yield '"data": ['
yield json.dumps(response)
yield ']}'
def generate_geojson():
yield '{"cloneUrl": ' + json.dumps(
QueryService.get_clone_url(dataset.get('id'), query)) + ','
if table_type is 'ft':
f_len = len(features)
yield '"data": [{ "type": "FeatureCollection", "features": ['
for idx, feature in enumerate(features):
geo_feature = {
"type": "Feature",
"properties": {},
"geometry": {}
}
for prop in feature:
if prop == 'the_geom':
geo_feature['geometry'] = feature['the_geom']
else:
geo_feature['properties'][prop] = feature[prop]
if idx != f_len - 1:
yield json.dumps(geo_feature) + ','
else:
yield json.dumps(geo_feature)
yield ']}]}'
elif table_type is 'raster':
yield '"data": ['
yield json.dumps(response)
yield ']}'
format = request.args.get('format', None)
if format == 'geojson':
return Response(
stream_with_context(generate_geojson()),
mimetype='application/json',
)
else:
return Response(
stream_with_context(generate_json()),
mimetype='application/json',
)
def download(dataset_id):
"""Download GEE Dataset Endpoint"""
logging.info('Downloading GEE Dataset')
# Get and deserialize
dataset = DatasetResponder().deserialize(request.get_json())
table_type = QueryService.get_type(
dataset.get('attributes').get('tableName'))
sql = request.args.get('sql', None) or request.get_json().get('sql', None)
# request
result_query = form_query(sql, dataset, request)
# convert
try:
if sql:
query_type = 'sql'
else:
query_type = 'fs'
query, json_sql = QueryService.convert(result_query,
query_type=query_type)
except SqlFormatError as error:
logging.error(error.message)
response = ErrorResponder.build({
'status': 400,
'message': error.message
})
return jsonify(response), 400
except Exception as error:
response = ErrorResponder.build({
'status': 500,
'message': 'Generic Error'
})
return jsonify(response), 500
# geojson
geojson = request.get_json().get('geojson', None)
if not geojson:
try:
if query.index('ST_INTERSECTS'):
geojson = QueryService.get_geojson(json_sql)
except:
pass
# query
try:
response = EarthEngineService.execute_query(query, geojson=geojson)
except GEEQueryError as error:
logging.error(error.message)
response = ErrorResponder.build({
'status': 500,
'message': error.message
})
return jsonify(response), 500
except Exception as error:
response = ErrorResponder.build({
'status': 500,
'message': 'Generic Error'
})
return jsonify(response), 500
if table_type is 'ft':
if not isinstance(response, dict):
value = response
response = {'features': [{'FUNCTION': value}]}
features = QueryResponder().serialize(response.get('features', {}))
class Row(object):
def __init__(self):
self.row = None
def write(self, row):
self.row = row
def read(self):
return self.row
def generate_csv():
row = Row()
writer = csv.writer(row)
if table_type is 'ft':
writer.writerow(features[0].keys())
elif table_type is 'raster':
writer.writerow(response.keys())
yield row.read()
def encode_feature_values(value):
if isinstance(value, basestring):
return value.encode('utf-8')
else:
return value
if table_type is 'ft':
for feature in features:
writer.writerow(map(encode_feature_values, feature.values()))
yield row.read()
elif table_type is 'raster':
for key in response.keys():
writer.writerow([response[key]])
yield row.read()
def generate_json():
yield '{'
if table_type is 'ft':
f_len = len(features)
yield '"data": ['
for idx, feature in enumerate(features):
if idx != f_len - 1:
yield json.dumps(feature) + ', '
else:
yield json.dumps(feature)
yield ']}'
elif table_type is 'raster':
k_len = len(response.keys())
yield '"data": ['
for idx, key in enumerate(response.keys()):
if idx != k_len - 1:
yield json.dumps(response[key]) + ', '
else:
yield json.dumps(response[key])
yield ']}'
def generate_geojson():
yield '{'
if table_type is 'ft':
f_len = len(features)
yield '"data": [{ "type": "FeatureCollection", "features": ['
for idx, feature in enumerate(features):
geo_feature = {
"type": "Feature",
"properties": {},
"geometry": {}
}
for prop in feature:
if prop == 'the_geom':
geo_feature['geometry'] = feature['the_geom']
else:
geo_feature['properties'][prop] = feature[prop]
if idx != f_len - 1:
yield json.dumps(geo_feature) + ','
else:
yield json.dumps(geo_feature)
yield ']}]}'
elif table_type is 'raster':
yield '"data": ['
yield json.dumps(response)
yield ']}'
format = request.args.get('format', None)
if format == 'csv':
return Response(stream_with_context(generate_csv()),
mimetype='text/csv',
headers={
'Content-Disposition':
'attachment; filename=export.csv',
'Content-type': 'text/csv'
})
elif format == 'geojson':
return Response(stream_with_context(generate_geojson()),
mimetype='application/json',
headers={
'Content-Disposition':
'attachment; filename=export.json',
'Content-type': 'application/json'
})
else:
return Response(stream_with_context(generate_json()),
mimetype='application/json',
headers={
'Content-Disposition':
'attachment; filename=export.json',
'Content-type': 'application/json'
})
