def post(self, data_id=None):
# TODO: Rewrite this for environments
if not data_id:
desc = "ID is missing from the request."
response["success"] = False
response["response"] = "MissingParametersError"
response["description"] = desc
response["code"] = 400
wrap_response(self, response)
return
data = APIData.get_by_id(normalize_id(data_id))
if not data:
desc = "Cannot find the package."
response["success"] = False
response["response"] = "InvalidIDError"
response["description"] = desc
response["code"] = 400
wrap_response(self, response)
return
if self.user.key != data.user:
if not self.user.teams:
msg = "You have insufficient rights to access this application."
response["success"] = False
response["response"] = "InvalidIDError"
response["description"] = msg
response["code"] = 400
wrap_response(self, response)
return
has_access = False
for team in self.user.teams:
if team in data.access_lock:
has_access = True
if not has_access:
msg = "You have insufficient rights to access this application."
response["success"] = False
response["response"] = "InvalidIDError"
response["description"] = msg
response["code"] = 400
else:
data.access_lock.remove("PRIVATE")
data.access_lock.append("PUBLIC")
data.access_lock = uniquify(data.access_lock)
data.put()
msg = "The data has been published and is now public."
response["success"] = True
response["response"] = "Success"
response["description"] = msg
response["code"] = 200
wrap_response(self, response)
def get(self, data_id=None):
"""
Handles the /api/v1/data/[data_id] endpoint.
Returns the details of the data_id provided.
"""
response = {
"code": 200,
"type": "Data details.",
"method": "GET",
"response": "OK",
"data": {}
}
try:
data_id = int(data_id)
except ValueError:
data_id = str(data_id)
except Exception as e:
logging.exception(e)
wrap_response(self, response)
return
data = APIData.get_by_id(normalize_id(data_id))
if data:
# TODO: Add check for authorization
response["data"] = data.to_api_object()
if self.request.get('show_environments'):
public = False
for environment_key in data.environment:
logging.debug(str(data.environment))
if environment_key.id() == 'PUBLIC':
public = True
else:
environment = environment_key.get()
if environment:
response["data"][
"environment_object"] = environment.to_api_object(
)
if public:
response["data"]["public"] = True
else:
response["data"]["public"] = False
if self.GET("callback"):
callback = self.GET("callback")
d = json.dumps(response)
self.response.out.write(callback + "(" + d + ");")
else:
wrap_response(self, response)
def get(self):
if not self.request.get('project_code') and not self.request.get('parent_code'):
logging.error('missing project code')
self.error(400)
return
project_code = self.request.get('project_code')
parent_code = self.request.get('parent_code')
kml_id = self.request.get('kml_id')
if not kml_id:
kml = APIData.query()
kml = kml.filter(APIData.indexed_data == 'TYPE->KML')
kml = kml.filter(APIData.indexed_data == 'PROJECT_CODE->' + project_code.upper())
kml = kml.get()
kml_id = str(kml.key.id())
else:
kml = APIData.get_by_id(normalize_id(kml_id))
if not kml:
logging.error('cannot find kml_id: ' + kml_id)
self.error(404)
return
# check if has GCSFile
if self.request.get('output') and self.request.get('output') == 'json':
file_type = 'JSON'
else:
file_type = 'KML'
gcs_file = GCSFile.query(GCSFile.kml_id == kml_id, GCSFile.file_type == file_type).order(-GCSFile.created_time).get()
if gcs_file:
# check if classification
classification_check = APIData.query(APIData.indexed_data == create_indexed_tag('TYPE', 'CLASSIFICATION')).order(-APIData.created_time).get()
if classification_check and classification_check.created_time < gcs_file.created_time:
# has file and still updated
self.redirect(gcs_file.link.encode('utf-8'))
return
output = self.request.get('output')
deferred.defer(merge_kml, parent_code, project_code, output, kml)
self.tv['project_code'] = project_code
self.render('kml-download.html')
return
def compute_kml_length(kml_id):
kml_id = normalize_id(kml_id)
kml = APIData.get_by_id(kml_id)
location = urllib.unquote(kml.file_url).replace("https://storage.googleapis.com", '')
location = location.replace("http://storage.googleapis.com", '')
response = {}
try:
with gcs.open(location) as f:
if kml.file_url.endswith('.kmz'):
zf = ZipFile(StringIO(f.read()))
xml_data = zf.read('doc.kml')
else:
xml_data = f.read()
response['points'] = get_kml_points(xml_data)
except Exception as e:
response['error'] = 'Error reading KML/KMZ.'
logging.debug(e)
response['length'] = []
for points in response['points']:
p = 0
for i in range(1, len(points)):
p += get_distance_in_km(points[i - 1], points[i])
response['length'].append(p)
return response
def post(self):
if self.request.get('target') and self.request.get('id') and self.request.get('permission'):
if self.request.get('permission') not in ['PUBLIC', 'PRIVATE']:
logging.error('Permission not valid')
logging.error(self.request.get('permission'))
self.response.write('Invalid Permission Value')
self.error(400)
return
target = self.request.get('target')
if target == 'DATA':
# edit data permission setting
data = APIData.get_by_id(normalize_id(self.request.get('id')))
if data:
if data.additional_data['type'] not in ['PROJECT', 'DATASET']:
data.permission = self.request.get('permission')
data.put()
return
else:
self.response.write('Cannot modify permission for Project or Dataset')
self.error(400)
return
else:
self.response.write('Data not found')
logging.error('data not found')
logging.error(self.request.get('id'))
self.error(404)
return
elif target == 'DATASET':
# edit dataset
if self.user.role in ["CLUSTERDIRECTOR", 'GEOSTOREADMIN']:
data = APIData.get_by_id(normalize_id(self.request.get('id')))
if data:
if data.additional_data['type'] == 'DATASET':
# modify environment key
if self.request.get('permission') == 'PUBLIC':
if ndb.Key('Environment', 'PUBLIC') not in data.environment:
data.environment.append(ndb.Key('Environment', 'PUBLIC'))
else:
logging.error('dataset already in a public environment')
elif self.request.get('permission') == 'PRIVATE':
data.environment.remove(ndb.Key('Environment', 'PUBLIC'))
else:
logging.error('unknown permission value')
logging.error(self.request.get('permission'))
self.error(400)
return
data.put()
else:
self.response.write('Dataset only allowed')
self.error(400)
return
else:
self.response.write('Dataset not found')
self.error(404)
return
else:
self.response.write('Access Denied')
self.error(403)
else:
logging.error('Missing Parameters')
self.response.write('Missing parameters')
self.error(400)
def update_api_data(data_id=None, items=None, user=None, content_type=None):
data = APIData.get_by_id(normalize_id(data_id))
if not data:
return
if content_type == "application/json":
tags = []
try:
for key, value in items.items():
if key.startswith('unindexed_'): # unindexed_
ad_key = key.replace("unindexed_", "")
data.additional_data[ad_key] = value.strip()
if key.startswith('indexed_'):
ad_key = key.replace("indexed_", "")
data.additional_data[ad_key] = value
for d in data.indexed_data:
ad_key = key.replace("indexed_", "")
if d.startswith(ad_key.upper()):
try:
data.indexed_data.remove(d)
except Exception as e:
logging.exception(e)
logging.info("Cannot remove from list")
data.indexed_data.append(
create_indexed_tag(
key, value))
try:
tags += create_tags(value)
except Exception as e:
logging.exception("Cannot create tag from: ")
if user:
data.username = user.name
data.user = user.key
logging.info(tags)
data.indexed_data = uniquify(data.indexed_data)
data.tags = uniquify(tags)
data.put()
except Exception as e:
logging.exception(e)
else:
tags = []
try:
for arg in self.request.arguments():
for d in data.indexed_data:
ad_key = arg.replace("indexed_", "")
if d.startswith(ad_key.upper()):
try:
data.indexed_data.remove(d)
except Exception as e:
logging.exception(e)
if arg.startswith('unindexed_'): # unindexed_
ad_key = arg.replace("unindexed_", "")
ad_value = self.request.POST.get(arg)
data.additional_data[ad_key] = ad_value.strip()
if arg.startswith('indexed_'):
ad_key = arg.replace("indexed_", "")
ad_value = self.request.POST.get(arg)
data.additional_data[ad_key] = ad_value
try:
tags += create_tags(ad_value)
except Exception as e:
logging.exception("Cannot create tag from: ")
data.indexed_data.append(
create_indexed_tag(
arg, self.request.POST.get(arg)))
if arg.startswith('file_'):
filename = BUCKET_NAME
filename += random_string(20) + "/"
ad_key = arg.replace("file_", "")
data.additional_data[ad_key] = {}
try:
file_name = items.get(arg).filename
filename += file_name
gcs_options = {'x-goog-acl': 'public-read'}
gcs_file = gcs.open(filename, 'w',
options=gcs_options)
gcs_file.write(self.request.get(arg))
gcs_file.close()
full_url = "https://storage.googleapis.com" \
+ filename
data.file_url = full_url
data.additional_data[ad_key]["file_url"] = full_url
try:
blob_key = blobstore.create_gs_key("/gs" + filename)
data.serving_url = images.get_serving_url(blob_key)
data.additional_data[ad_key]["serving_url"] = data.serving_url
data.gcs_key = blobstore.BlobKey(blob_key)
except Exception as e:
logging.exception(e)
data.additional_data[ad_key]["serving_url"] = full_url
except AttributeError, e:
logging.exception(e)
if self.user:
data.username = self.user.name
data.user = self.user.key
data.indexed_data = uniquify(data.indexed_data)
data.tags = uniquify(tags)
data.put()
except Exception as e:
logging.exception(e)
def post(self, data_id=None):
response = {}
response["success"] = True
logging.info(self.request.headers)
content_type = self.request.headers["Content_Type"]
if not self.user:
if content_type == "application/json":
if "Authorization" not in self.request.headers:
logging.info("No Authorization in headers")
desc = "You must be logged in to use the API."
response["success"] = False
response["response"] = "AuthorizationError"
response["description"] = desc
response["code"] = 400
wrap_response(self, response)
return
if self.request.headers["Authorization"] == API_KEY:
if not self.request.headers["From"]:
logging.info("No email defined")
desc = "Cannot find user."
response["success"] = False
response["response"] = "InvalidUserError"
response["description"] = desc
response["code"] = 400
wrap_response(self, response)
return
user_email = self.request.headers["From"].lower()
query = User.query()
owner = query.filter(
User.current_email == user_email).get()
if not owner:
logging.info("Cannot find user")
desc = "Cannot find user."
response["success"] = False
response["response"] = "InvalidUserError"
response["description"] = desc
response["code"] = 400
wrap_response(self, response)
return
else:
token = Token.get_by_id(
self.request.headers["Authorization"])
if not token:
logging.info(
"Cannot find token: " +
str(self.request.headers["Authorization"]))
desc = "The token you provided is invalid."
response["success"] = False
response["response"] = "InvalidTokenError"
response["description"] = desc
response["code"] = 400
wrap_response(self, response)
return
logging.info(token)
session = token.session.get()
if not session:
logging.info("Cannot find session")
desc = "The token has already expired."
response["error"] = False
response["response"] = "InvalidTokenError"
response["description"] = desc
response["code"] = 400
wrap_response(self, response)
return
logging.info(session)
if session.expires < datetime.datetime.now(
) or session.status is False:
logging.info("token has expired or not active")
desc = "The token has already expired."
response["success"] = False
response["response"] = "InvalidTokenError"
response["description"] = desc
response["code"] = 400
wrap_response(self, response)
return
else:
desc = "You must be logged in to use the API."
if self.POST("r"):
url = urllib.unquote(str(self.POST("r")))
else:
url = self.request.referer
if url:
if "?" in url:
url = url.split("?")[0]
url += "?error=" + urllib.quote(desc)
self.redirect(url)
if not data_id:
desc = "ID is missing from the request."
if content_type == "application/json":
response["success"] = False
response["response"] = "MissingParametersError"
response["description"] = desc
response["code"] = 400
wrap_response(self, response)
else:
if self.POST("r"):
url = urllib.unquote(str(self.POST("r")))
else:
url = self.request.referer
if url:
if "?" in url:
url = url.split("?")[0]
url += "?error=" + urllib.quote(desc)
self.redirect(url)
return
data = APIData.get_by_id(normalize_id(data_id))
if not data:
desc = "Cannot find the package."
if content_type == "application/json":
response["success"] = False
response["response"] = "InvalidIDError"
response["description"] = desc
response["code"] = 400
wrap_response(self, response)
else:
if self.POST("r"):
url = urllib.unquote(str(self.POST("r")))
else:
url = self.request.referer
if url:
if "?" in url:
url = url.split("?")[0]
url += "?error=" + urllib.quote(desc)
self.redirect(url)
return
if data.archived:
desc = "Cannot find the package."
if content_type == "application/json":
response["success"] = False
response["response"] = "InvalidIDError"
response["description"] = desc
response["code"] = 400
wrap_response(self, response)
else:
if self.POST("r"):
url = urllib.unquote(str(self.POST("r")))
else:
url = self.request.referer
if url:
if "?" in url:
url = url.split("?")[0]
url += "?error=" + urllib.quote(desc)
self.redirect(url)
return
desc = "There are missing parameters in your request."
if content_type == "application/json":
if not self.request.body:
response["success"] = False
response["response"] = "MissingParametersError"
response["description"] = desc
response["code"] = 400
wrap_response(self, response)
return
try:
body = json.loads(self.request.body)
except Exception as e:
logging.info(e)
desc = "Invalid JSON format."
response["success"] = False
response["response"] = "InvalidJSONError"
response["description"] = desc
response["code"] = 400
wrap_response(self, response)
return
tags = []
try:
for key, value in body.items():
try:
tags += create_tags(value)
except Exception as e:
logging.info("Cannot create tag from: ")
logging.info(e)
if key.startswith('unindexed_'): # unindexed_
ad_key = key.replace("unindexed_", "")
data.additional_data[ad_key] = value.strip()
if key.startswith('indexed_'):
ad_key = key.replace("indexed_", "")
data.additional_data[ad_key] = value
for d in data.indexed_data:
ad_key = key.replace("indexed_", "")
if d.startswith(ad_key.upper()):
try:
data.indexed_data.remove(d)
except Exception as e:
logging.exception(e)
logging.info("Cannot remove from list")
data.indexed_data.append(create_indexed_tag(
key, value))
if self.user:
data.username = self.user.name
data.user = self.user.key
data.indexed_data = uniquify(data.indexed_data)
data.tags = uniquify(tags)
data.put()
desc = "Data has been saved."
response["success"] = True
response["response"] = "Success"
response["description"] = desc
response["code"] = 200
response["data"] = data.to_api_object()
wrap_response(self, response)
except Exception as e:
logging.exception(e)
desc = "A server error occured. Please try again later."
response["success"] = False
response["response"] = "ServerError"
response["description"] = desc
response["code"] = 500
wrap_response(self, response)
else:
if not self.request.arguments():
if self.POST("r"):
url = urllib.unquote(str(self.POST("r")))
else:
url = self.request.referer
if url:
if "?" in url:
url = url.split("?")[0]
url += "?error=" + urllib.quote(desc)
self.redirect(url)
return
tags = []
try:
for arg in self.request.arguments():
for d in data.indexed_data:
ad_key = arg.replace("indexed_", "")
if d.startswith(ad_key.upper()):
try:
data.indexed_data.remove(d)
except Exception as e:
logging.exception(e)
logging.info("Cannot remove from list")
if arg.startswith('unindexed_'):
ad_key = arg.replace("unindexed_", "")
ad_value = self.request.POST.get(arg)
data.additional_data[ad_key] = ad_value.strip()
try:
tags += create_tags(ad_value)
except Exception as e:
logging.info("Cannot create tag from: ")
logging.info(e)
if arg.startswith('indexed_'):
ad_key = arg.replace("indexed_", "")
ad_value = self.request.POST.get(arg)
data.additional_data[ad_key] = ad_value
try:
tags += create_tags(ad_value)
except Exception as e:
logging.info("Cannot create tag from: ")
logging.info(e)
data.indexed_data.append(
create_indexed_tag(arg,
self.request.POST.get(arg)))
if arg.startswith('file_'):
filename = BUCKET_NAME
filename += random_string(20) + "/"
ad_key = arg.replace("file_", "")
data.additional_data[ad_key] = {}
try:
# try:
file_name = self.request.POST.get(arg).filename
filename += file_name
gcs_file = gcs.open(
filename,
'w',
options={'x-goog-acl': 'public-read'})
gcs_file.write(self.request.get(arg))
gcs_file.close()
full_url = "https://storage.googleapis.com" + filename
# data.additional_data["file"]["file_url"] = full_url
data.file_url = full_url
data.additional_data[ad_key]["file_url"] = full_url
try:
blob_key = blobstore.create_gs_key("/gs" +
filename)
data.serving_url = images.get_serving_url(
blob_key)
data.additional_data[ad_key][
"serving_url"] = data.serving_url
data.gcs_key = blobstore.BlobKey(blob_key)
except Exception as e:
logging.exception(e)
logging.error("not an image??")
data.additional_data[ad_key][
"serving_url"] = full_url
except AttributeError, e:
logging.exception(e)
logging.exception("NO FILE ATTACHED")
if self.user:
data.username = self.user.name
data.user = self.user.key
data.indexed_data = uniquify(data.indexed_data)
data.tags = uniquify(tags)
data.put()
desc = "Data has been updated."
if self.POST("r"):
url = urllib.unquote(str(self.POST("r")))
else:
url = self.request.referer
if url:
if "?" in url:
url = url.split("?")[0]
url += "?success=" + urllib.quote(desc)
self.redirect(url)
else:
response["success"] = True
response["response"] = "Success"
response["description"] = desc
response["code"] = 200
response["data"] = data.to_api_object()
wrap_response(self, response)
except Exception as e:
logging.exception(e)
desc = "A server error occured. Please try again later."
if self.POST("r"):
url = urllib.unquote(str(self.POST("r")))
else:
url = self.request.referer
if url:
if "?" in url:
url = url.split("?")[0]
url += "?error=" + urllib.quote(desc)
self.redirect(url)
else:
response["success"] = False
response["response"] = "ServerError"
response["description"] = desc
response["code"] = 500
wrap_response(self, response)