def get(self):
"""
Handles the /api/v1/data endpoint.
Returns list of datasets.
"""
response = {
"code": 200,
"type": "List of geostore saved data.",
"method": "GET",
"response": "OK",
"data": []
}
# Default number of entities to be retrieved is 50.
n = 50
if self.GET("n"):
n = int(self.GET("n"))
# if the number of entities to be retrieved given is
# greater than 100. Switch back to default which is 100
if n > 100:
n = 100
query = APIData.query()
query = query.filter(APIData.archived == False)
if not self.user:
if "Authorization" in self.request.headers:
token = Token.get_by_id(self.request.headers["Authorization"])
if not token:
logging.info("Cannot find token: " +
str(self.request.headers["Authorization"]))
desc = "The token you provided is invalid."
response["success"] = False
response["response"] = "InvalidTokenError"
response["description"] = desc
response["code"] = 400
wrap_response(self, response)
return
logging.info(token)
session = token.session.get()
if not session:
logging.info("Cannot find session")
desc = "The token has already expired."
response["error"] = False
response["response"] = "InvalidTokenError"
response["description"] = desc
response["code"] = 400
wrap_response(self, response)
return
logging.info(session)
if session.expires < datetime.datetime.now(
) or session.status is False:
logging.info("token has expired or not active")
desc = "The token has already expired."
response["success"] = False
response["response"] = "InvalidTokenError"
response["description"] = desc
response["code"] = 400
wrap_response(self, response)
return
owner = session.owner.get()
if not owner:
logging.info("Cannot find user")
desc = "Cannot find user."
response["success"] = False
response["response"] = "InvalidUserError"
response["description"] = desc
response["code"] = 400
wrap_response(self, response)
return
self.user = owner
if self.user and self.GET('workspace'):
environment = ndb.Key(urlsafe=self.GET('workspace')).get()
if environment:
if self.user.key in environment.users:
environment_key = environment.key
query = query.filter(
APIData.environment == environment_key)
elif self.user and not self.GET('workspace') and (
not self.GET('type')
or not self.GET('type') in ['PROJECT', 'SUBPROJECT']):
environments_user = Environment.query(
Environment.users == self.user.key).fetch(keys_only=True)
environments_user.append(ndb.Key('Environment', 'PUBLIC'))
query = query.filter(
APIData.environment.IN(environments_user)).order(APIData._key)
else:
query = query.filter(
APIData.environment == ndb.Key('Environment', 'PUBLIC'))
if self.GET("_search_"):
response["type"] = "Search geostore saved data."
response["query"] = self.GET("_search_")
search = self.GET("_search_").strip().upper()
query = query.filter(APIData.tags >= search)
query = query.order(APIData.tags)
else:
for arg in self.request.arguments():
if arg.lower() == "callback" \
or arg.lower() == "_" \
or arg.lower() == "order" \
or arg.lower() == "cursor" \
or arg.lower() == "n" \
or arg.lower() == "_search_" \
or arg.lower() == "show_image_dates" \
or arg.lower() == "start_updated_from" \
or arg.lower() == "start_created_from":
continue
ad_value = self.GET(arg)
tag = create_indexed_tag(arg, ad_value)
query = query.filter(APIData.indexed_data == tag)
if self.GET("order"):
if self.GET("order").lower() in ["asc", "ascending"]:
query = query.order(APIData.updated_time)
elif self.GET("order").lower() in ["desc", "descending"]:
query = query.order(-APIData.updated_time)
elif self.GET("order").lower() == "created_asc":
query = query.order(APIData.created_time)
elif self.GET("order").lower() == "created_desc":
query = query.order(-APIData.created_time)
elif self.GET("order").lower() == "modified":
query = query.order(APIData.updated_time)
else:
query = query.order(-APIData.created_time)
if self.GET("start_updated_from"):
logging.debug(self.GET("start_updated_from"))
if self.GET("order").lower() in ["desc", "descending"]:
query = query.filter(
APIData.updated_time <= datetime.datetime.
fromtimestamp(int(self.GET("start_updated_from"))))
else:
query = query.filter(
APIData.updated_time >= datetime.datetime.
fromtimestamp(int(self.GET("start_updated_from"))))
elif self.GET('start_created_from'):
logging.debug(self.GET("start_created_from"))
if self.GET("order").lower() in ["desc", "descending"]:
query = query.filter(
APIData.created_time <= datetime.datetime.
fromtimestamp(int(self.GET("start_created_from"))))
else:
query = query.filter(
APIData.created_time >= datetime.datetime.
fromtimestamp(int(self.GET("start_created_from"))))
logging.info(query)
if self.GET("cursor"):
curs = Cursor(urlsafe=self.GET("cursor"))
data, cursor, more = query.fetch_page(n, start_cursor=curs)
else:
data, cursor, more = query.fetch_page(n)
if data:
response["cursor"] = ""
for d in data:
try:
response["data"].append(d.to_api_object())
except Exception as e:
logging.exception(e)
if more:
response["cursor"] = cursor.urlsafe()
if self.GET('show_image_dates'):
if self.GET('type'):
if self.GET('type').upper() == 'CLASSIFICATION':
if 'data' in response:
image_ids = []
for classification in response['data']:
image_ids.append(
ndb.Key('APIData',
int(classification['image_id'])))
images = ndb.get_multi(image_ids)
for image in images:
date = ''
image_latlng = image.additional_data['latlng']
if 'date' in image.additional_data:
date = image.additional_data['date']
else:
date = image.created_time.strftime(
'%Y:%m:%d %H:%M:%S')
for i in range(0, len(response['data'])):
if response['data'][i]['image_id'] == str(
image.key.id()):
response['data'][i]['image_date'] = date
response['data'][i][
'image_latlng'] = image_latlng
if self.GET("callback"):
callback = self.GET("callback")
d = json.dumps(response)
self.response.out.write(callback + "(" + d + ");")
else:
wrap_response(self, response)
def post(self):
"""
Handles the /api/v1/data endpoint.
Creates a dataset.
"""
owner = None
response = {}
response["success"] = True
content_type = self.request.headers[
"Content_Type"] or self.request.headers["Content-Type"]
if not self.user:
if content_type == "application/json":
if "Authorization" not in self.request.headers:
logging.info("No Authorization in headers")
desc = "You must be logged in to use the API."
response["success"] = False
response["response"] = "AuthorizationError"
response["description"] = desc
response["code"] = 400
wrap_response(self, response)
return
if self.request.headers["Authorization"] == API_KEY:
if not self.request.headers["From"]:
logging.info("No email defined")
desc = "Cannot find user."
response["success"] = False
response["response"] = "InvalidUserError"
response["description"] = desc
response["code"] = 400
wrap_response(self, response)
return
user_email = self.request.headers["From"].lower()
query = User.query()
owner = query.filter(
User.current_email == user_email).get()
if not owner:
logging.info("Cannot find user")
desc = "Cannot find user."
response["success"] = False
response["response"] = "InvalidUserError"
response["description"] = desc
response["code"] = 400
wrap_response(self, response)
return
else:
token = Token.get_by_id(
self.request.headers["Authorization"])
if not token:
logging.info(
"Cannot find token: " +
str(self.request.headers["Authorization"]))
desc = "The token you provided is invalid."
response["success"] = False
response["response"] = "InvalidTokenError"
response["description"] = desc
response["code"] = 400
wrap_response(self, response)
return
logging.info(token)
session = token.session.get()
if not session:
logging.info("Cannot find session")
desc = "The token has already expired."
response["error"] = False
response["response"] = "InvalidTokenError"
response["description"] = desc
response["code"] = 400
wrap_response(self, response)
return
logging.info(session)
if session.expires < datetime.datetime.now(
) or session.status is False:
logging.info("token has expired or not active")
desc = "The token has already expired."
response["success"] = False
response["response"] = "InvalidTokenError"
response["description"] = desc
response["code"] = 400
wrap_response(self, response)
return
owner = session.owner.get()
if not owner:
logging.info("Cannot find user")
desc = "Cannot find user."
response["success"] = False
response["response"] = "InvalidUserError"
response["description"] = desc
response["code"] = 400
wrap_response(self, response)
return
else:
if "Authorization" in self.request.headers:
token = Token.get_by_id(
self.request.headers["Authorization"])
if not token:
logging.info(
"Cannot find token: " +
str(self.request.headers["Authorization"]))
desc = "The token you provided is invalid."
response["success"] = False
response["response"] = "InvalidTokenError"
response["description"] = desc
response["code"] = 400
wrap_response(self, response)
return
logging.info(token)
session = token.session.get()
if not session:
logging.info("Cannot find session")
desc = "The token has already expired."
response["error"] = False
response["response"] = "InvalidTokenError"
response["description"] = desc
response["code"] = 400
wrap_response(self, response)
return
logging.info(session)
if session.expires < datetime.datetime.now(
) or session.status is False:
logging.info("token has expired or not active")
desc = "The token has already expired."
response["success"] = False
response["response"] = "InvalidTokenError"
response["description"] = desc
response["code"] = 400
wrap_response(self, response)
return
owner = session.owner.get()
if not owner:
logging.info("Cannot find user")
desc = "Cannot find user."
response["success"] = False
response["response"] = "InvalidUserError"
response["description"] = desc
response["code"] = 400
wrap_response(self, response)
return
else:
desc = "You must be logged in to use the API."
if self.POST("r"):
url = urllib.unquote(str(self.POST("r")))
if "?" in url:
url = url.split("?")[0]
url += "?error=" + urllib.quote(desc)
self.redirect(url)
else:
logging.info("User is not logged in")
desc = "You must be logged in to use the api."
response["success"] = False
response["response"] = "InvalidRequestError"
response["description"] = desc
response["code"] = 400
wrap_response(self, response)
return
else:
owner = self.user
logging.debug(content_type)
if content_type == 'application/json':
# d = write_to_api(json.loads(self.request.body), owner, content_type)
d = write_to_api(json.loads(self.request.body), owner)
else:
d = write_to_api_params(self.request.arguments(), owner,
content_type, False, self)
wrap_response(self, d.to_api_object())
def post(self, data_id=None):
response = {}
response["success"] = True
logging.info(self.request.headers)
content_type = self.request.headers["Content_Type"]
if not self.user:
if content_type == "application/json":
if "Authorization" not in self.request.headers:
logging.info("No Authorization in headers")
desc = "You must be logged in to use the API."
response["success"] = False
response["response"] = "AuthorizationError"
response["description"] = desc
response["code"] = 400
wrap_response(self, response)
return
if self.request.headers["Authorization"] == API_KEY:
if not self.request.headers["From"]:
logging.info("No email defined")
desc = "Cannot find user."
response["success"] = False
response["response"] = "InvalidUserError"
response["description"] = desc
response["code"] = 400
wrap_response(self, response)
return
user_email = self.request.headers["From"].lower()
query = User.query()
owner = query.filter(
User.current_email == user_email).get()
if not owner:
logging.info("Cannot find user")
desc = "Cannot find user."
response["success"] = False
response["response"] = "InvalidUserError"
response["description"] = desc
response["code"] = 400
wrap_response(self, response)
return
else:
token = Token.get_by_id(
self.request.headers["Authorization"])
if not token:
logging.info(
"Cannot find token: " +
str(self.request.headers["Authorization"]))
desc = "The token you provided is invalid."
response["success"] = False
response["response"] = "InvalidTokenError"
response["description"] = desc
response["code"] = 400
wrap_response(self, response)
return
logging.info(token)
session = token.session.get()
if not session:
logging.info("Cannot find session")
desc = "The token has already expired."
response["error"] = False
response["response"] = "InvalidTokenError"
response["description"] = desc
response["code"] = 400
wrap_response(self, response)
return
logging.info(session)
if session.expires < datetime.datetime.now(
) or session.status is False:
logging.info("token has expired or not active")
desc = "The token has already expired."
response["success"] = False
response["response"] = "InvalidTokenError"
response["description"] = desc
response["code"] = 400
wrap_response(self, response)
return
else:
desc = "You must be logged in to use the API."
if self.POST("r"):
url = urllib.unquote(str(self.POST("r")))
else:
url = self.request.referer
if url:
if "?" in url:
url = url.split("?")[0]
url += "?error=" + urllib.quote(desc)
self.redirect(url)
if not data_id:
desc = "ID is missing from the request."
if content_type == "application/json":
response["success"] = False
response["response"] = "MissingParametersError"
response["description"] = desc
response["code"] = 400
wrap_response(self, response)
else:
if self.POST("r"):
url = urllib.unquote(str(self.POST("r")))
else:
url = self.request.referer
if url:
if "?" in url:
url = url.split("?")[0]
url += "?error=" + urllib.quote(desc)
self.redirect(url)
return
data = APIData.get_by_id(normalize_id(data_id))
if not data:
desc = "Cannot find the package."
if content_type == "application/json":
response["success"] = False
response["response"] = "InvalidIDError"
response["description"] = desc
response["code"] = 400
wrap_response(self, response)
else:
if self.POST("r"):
url = urllib.unquote(str(self.POST("r")))
else:
url = self.request.referer
if url:
if "?" in url:
url = url.split("?")[0]
url += "?error=" + urllib.quote(desc)
self.redirect(url)
return
if data.archived:
desc = "Cannot find the package."
if content_type == "application/json":
response["success"] = False
response["response"] = "InvalidIDError"
response["description"] = desc
response["code"] = 400
wrap_response(self, response)
else:
if self.POST("r"):
url = urllib.unquote(str(self.POST("r")))
else:
url = self.request.referer
if url:
if "?" in url:
url = url.split("?")[0]
url += "?error=" + urllib.quote(desc)
self.redirect(url)
return
desc = "There are missing parameters in your request."
if content_type == "application/json":
if not self.request.body:
response["success"] = False
response["response"] = "MissingParametersError"
response["description"] = desc
response["code"] = 400
wrap_response(self, response)
return
try:
body = json.loads(self.request.body)
except Exception as e:
logging.info(e)
desc = "Invalid JSON format."
response["success"] = False
response["response"] = "InvalidJSONError"
response["description"] = desc
response["code"] = 400
wrap_response(self, response)
return
tags = []
try:
for key, value in body.items():
try:
tags += create_tags(value)
except Exception as e:
logging.info("Cannot create tag from: ")
logging.info(e)
if key.startswith('unindexed_'): # unindexed_
ad_key = key.replace("unindexed_", "")
data.additional_data[ad_key] = value.strip()
if key.startswith('indexed_'):
ad_key = key.replace("indexed_", "")
data.additional_data[ad_key] = value
for d in data.indexed_data:
ad_key = key.replace("indexed_", "")
if d.startswith(ad_key.upper()):
try:
data.indexed_data.remove(d)
except Exception as e:
logging.exception(e)
logging.info("Cannot remove from list")
data.indexed_data.append(create_indexed_tag(
key, value))
if self.user:
data.username = self.user.name
data.user = self.user.key
data.indexed_data = uniquify(data.indexed_data)
data.tags = uniquify(tags)
data.put()
desc = "Data has been saved."
response["success"] = True
response["response"] = "Success"
response["description"] = desc
response["code"] = 200
response["data"] = data.to_api_object()
wrap_response(self, response)
except Exception as e:
logging.exception(e)
desc = "A server error occured. Please try again later."
response["success"] = False
response["response"] = "ServerError"
response["description"] = desc
response["code"] = 500
wrap_response(self, response)
else:
if not self.request.arguments():
if self.POST("r"):
url = urllib.unquote(str(self.POST("r")))
else:
url = self.request.referer
if url:
if "?" in url:
url = url.split("?")[0]
url += "?error=" + urllib.quote(desc)
self.redirect(url)
return
tags = []
try:
for arg in self.request.arguments():
for d in data.indexed_data:
ad_key = arg.replace("indexed_", "")
if d.startswith(ad_key.upper()):
try:
data.indexed_data.remove(d)
except Exception as e:
logging.exception(e)
logging.info("Cannot remove from list")
if arg.startswith('unindexed_'):
ad_key = arg.replace("unindexed_", "")
ad_value = self.request.POST.get(arg)
data.additional_data[ad_key] = ad_value.strip()
try:
tags += create_tags(ad_value)
except Exception as e:
logging.info("Cannot create tag from: ")
logging.info(e)
if arg.startswith('indexed_'):
ad_key = arg.replace("indexed_", "")
ad_value = self.request.POST.get(arg)
data.additional_data[ad_key] = ad_value
try:
tags += create_tags(ad_value)
except Exception as e:
logging.info("Cannot create tag from: ")
logging.info(e)
data.indexed_data.append(
create_indexed_tag(arg,
self.request.POST.get(arg)))
if arg.startswith('file_'):
filename = BUCKET_NAME
filename += random_string(20) + "/"
ad_key = arg.replace("file_", "")
data.additional_data[ad_key] = {}
try:
# try:
file_name = self.request.POST.get(arg).filename
filename += file_name
gcs_file = gcs.open(
filename,
'w',
options={'x-goog-acl': 'public-read'})
gcs_file.write(self.request.get(arg))
gcs_file.close()
full_url = "https://storage.googleapis.com" + filename
# data.additional_data["file"]["file_url"] = full_url
data.file_url = full_url
data.additional_data[ad_key]["file_url"] = full_url
try:
blob_key = blobstore.create_gs_key("/gs" +
filename)
data.serving_url = images.get_serving_url(
blob_key)
data.additional_data[ad_key][
"serving_url"] = data.serving_url
data.gcs_key = blobstore.BlobKey(blob_key)
except Exception as e:
logging.exception(e)
logging.error("not an image??")
data.additional_data[ad_key][
"serving_url"] = full_url
except AttributeError, e:
logging.exception(e)
logging.exception("NO FILE ATTACHED")
if self.user:
data.username = self.user.name
data.user = self.user.key
data.indexed_data = uniquify(data.indexed_data)
data.tags = uniquify(tags)
data.put()
desc = "Data has been updated."
if self.POST("r"):
url = urllib.unquote(str(self.POST("r")))
else:
url = self.request.referer
if url:
if "?" in url:
url = url.split("?")[0]
url += "?success=" + urllib.quote(desc)
self.redirect(url)
else:
response["success"] = True
response["response"] = "Success"
response["description"] = desc
response["code"] = 200
response["data"] = data.to_api_object()
wrap_response(self, response)
except Exception as e:
logging.exception(e)
desc = "A server error occured. Please try again later."
if self.POST("r"):
url = urllib.unquote(str(self.POST("r")))
else:
url = self.request.referer
if url:
if "?" in url:
url = url.split("?")[0]
url += "?error=" + urllib.quote(desc)
self.redirect(url)
else:
response["success"] = False
response["response"] = "ServerError"
response["description"] = desc
response["code"] = 500
wrap_response(self, response)
