def get(self): """ Handles the /api/v1/data endpoint. Returns list of datasets. """ response = { "code": 200, "type": "List of geostore saved data.", "method": "GET", "response": "OK", "data": [] } # Default number of entities to be retrieved is 50. n = 50 if self.GET("n"): n = int(self.GET("n")) # if the number of entities to be retrieved given is # greater than 100. Switch back to default which is 100 if n > 100: n = 100 query = APIData.query() query = query.filter(APIData.archived == False) if not self.user: if "Authorization" in self.request.headers: token = Token.get_by_id(self.request.headers["Authorization"]) if not token: logging.info("Cannot find token: " + str(self.request.headers["Authorization"])) desc = "The token you provided is invalid." response["success"] = False response["response"] = "InvalidTokenError" response["description"] = desc response["code"] = 400 wrap_response(self, response) return logging.info(token) session = token.session.get() if not session: logging.info("Cannot find session") desc = "The token has already expired." response["error"] = False response["response"] = "InvalidTokenError" response["description"] = desc response["code"] = 400 wrap_response(self, response) return logging.info(session) if session.expires < datetime.datetime.now( ) or session.status is False: logging.info("token has expired or not active") desc = "The token has already expired." response["success"] = False response["response"] = "InvalidTokenError" response["description"] = desc response["code"] = 400 wrap_response(self, response) return owner = session.owner.get() if not owner: logging.info("Cannot find user") desc = "Cannot find user." response["success"] = False response["response"] = "InvalidUserError" response["description"] = desc response["code"] = 400 wrap_response(self, response) return self.user = owner if self.user and self.GET('workspace'): environment = ndb.Key(urlsafe=self.GET('workspace')).get() if environment: if self.user.key in environment.users: environment_key = environment.key query = query.filter( APIData.environment == environment_key) elif self.user and not self.GET('workspace') and ( not self.GET('type') or not self.GET('type') in ['PROJECT', 'SUBPROJECT']): environments_user = Environment.query( Environment.users == self.user.key).fetch(keys_only=True) environments_user.append(ndb.Key('Environment', 'PUBLIC')) query = query.filter( APIData.environment.IN(environments_user)).order(APIData._key) else: query = query.filter( APIData.environment == ndb.Key('Environment', 'PUBLIC')) if self.GET("_search_"): response["type"] = "Search geostore saved data." response["query"] = self.GET("_search_") search = self.GET("_search_").strip().upper() query = query.filter(APIData.tags >= search) query = query.order(APIData.tags) else: for arg in self.request.arguments(): if arg.lower() == "callback" \ or arg.lower() == "_" \ or arg.lower() == "order" \ or arg.lower() == "cursor" \ or arg.lower() == "n" \ or arg.lower() == "_search_" \ or arg.lower() == "show_image_dates" \ or arg.lower() == "start_updated_from" \ or arg.lower() == "start_created_from": continue ad_value = self.GET(arg) tag = create_indexed_tag(arg, ad_value) query = query.filter(APIData.indexed_data == tag) if self.GET("order"): if self.GET("order").lower() in ["asc", "ascending"]: query = query.order(APIData.updated_time) elif self.GET("order").lower() in ["desc", "descending"]: query = query.order(-APIData.updated_time) elif self.GET("order").lower() == "created_asc": query = query.order(APIData.created_time) elif self.GET("order").lower() == "created_desc": query = query.order(-APIData.created_time) elif self.GET("order").lower() == "modified": query = query.order(APIData.updated_time) else: query = query.order(-APIData.created_time) if self.GET("start_updated_from"): logging.debug(self.GET("start_updated_from")) if self.GET("order").lower() in ["desc", "descending"]: query = query.filter( APIData.updated_time <= datetime.datetime. fromtimestamp(int(self.GET("start_updated_from")))) else: query = query.filter( APIData.updated_time >= datetime.datetime. fromtimestamp(int(self.GET("start_updated_from")))) elif self.GET('start_created_from'): logging.debug(self.GET("start_created_from")) if self.GET("order").lower() in ["desc", "descending"]: query = query.filter( APIData.created_time <= datetime.datetime. fromtimestamp(int(self.GET("start_created_from")))) else: query = query.filter( APIData.created_time >= datetime.datetime. fromtimestamp(int(self.GET("start_created_from")))) logging.info(query) if self.GET("cursor"): curs = Cursor(urlsafe=self.GET("cursor")) data, cursor, more = query.fetch_page(n, start_cursor=curs) else: data, cursor, more = query.fetch_page(n) if data: response["cursor"] = "" for d in data: try: response["data"].append(d.to_api_object()) except Exception as e: logging.exception(e) if more: response["cursor"] = cursor.urlsafe() if self.GET('show_image_dates'): if self.GET('type'): if self.GET('type').upper() == 'CLASSIFICATION': if 'data' in response: image_ids = [] for classification in response['data']: image_ids.append( ndb.Key('APIData', int(classification['image_id']))) images = ndb.get_multi(image_ids) for image in images: date = '' image_latlng = image.additional_data['latlng'] if 'date' in image.additional_data: date = image.additional_data['date'] else: date = image.created_time.strftime( '%Y:%m:%d %H:%M:%S') for i in range(0, len(response['data'])): if response['data'][i]['image_id'] == str( image.key.id()): response['data'][i]['image_date'] = date response['data'][i][ 'image_latlng'] = image_latlng if self.GET("callback"): callback = self.GET("callback") d = json.dumps(response) self.response.out.write(callback + "(" + d + ");") else: wrap_response(self, response)
def post(self): """ Handles the /api/v1/data endpoint. Creates a dataset. """ owner = None response = {} response["success"] = True content_type = self.request.headers[ "Content_Type"] or self.request.headers["Content-Type"] if not self.user: if content_type == "application/json": if "Authorization" not in self.request.headers: logging.info("No Authorization in headers") desc = "You must be logged in to use the API." response["success"] = False response["response"] = "AuthorizationError" response["description"] = desc response["code"] = 400 wrap_response(self, response) return if self.request.headers["Authorization"] == API_KEY: if not self.request.headers["From"]: logging.info("No email defined") desc = "Cannot find user." response["success"] = False response["response"] = "InvalidUserError" response["description"] = desc response["code"] = 400 wrap_response(self, response) return user_email = self.request.headers["From"].lower() query = User.query() owner = query.filter( User.current_email == user_email).get() if not owner: logging.info("Cannot find user") desc = "Cannot find user." response["success"] = False response["response"] = "InvalidUserError" response["description"] = desc response["code"] = 400 wrap_response(self, response) return else: token = Token.get_by_id( self.request.headers["Authorization"]) if not token: logging.info( "Cannot find token: " + str(self.request.headers["Authorization"])) desc = "The token you provided is invalid." response["success"] = False response["response"] = "InvalidTokenError" response["description"] = desc response["code"] = 400 wrap_response(self, response) return logging.info(token) session = token.session.get() if not session: logging.info("Cannot find session") desc = "The token has already expired." response["error"] = False response["response"] = "InvalidTokenError" response["description"] = desc response["code"] = 400 wrap_response(self, response) return logging.info(session) if session.expires < datetime.datetime.now( ) or session.status is False: logging.info("token has expired or not active") desc = "The token has already expired." response["success"] = False response["response"] = "InvalidTokenError" response["description"] = desc response["code"] = 400 wrap_response(self, response) return owner = session.owner.get() if not owner: logging.info("Cannot find user") desc = "Cannot find user." response["success"] = False response["response"] = "InvalidUserError" response["description"] = desc response["code"] = 400 wrap_response(self, response) return else: if "Authorization" in self.request.headers: token = Token.get_by_id( self.request.headers["Authorization"]) if not token: logging.info( "Cannot find token: " + str(self.request.headers["Authorization"])) desc = "The token you provided is invalid." response["success"] = False response["response"] = "InvalidTokenError" response["description"] = desc response["code"] = 400 wrap_response(self, response) return logging.info(token) session = token.session.get() if not session: logging.info("Cannot find session") desc = "The token has already expired." response["error"] = False response["response"] = "InvalidTokenError" response["description"] = desc response["code"] = 400 wrap_response(self, response) return logging.info(session) if session.expires < datetime.datetime.now( ) or session.status is False: logging.info("token has expired or not active") desc = "The token has already expired." response["success"] = False response["response"] = "InvalidTokenError" response["description"] = desc response["code"] = 400 wrap_response(self, response) return owner = session.owner.get() if not owner: logging.info("Cannot find user") desc = "Cannot find user." response["success"] = False response["response"] = "InvalidUserError" response["description"] = desc response["code"] = 400 wrap_response(self, response) return else: desc = "You must be logged in to use the API." if self.POST("r"): url = urllib.unquote(str(self.POST("r"))) if "?" in url: url = url.split("?")[0] url += "?error=" + urllib.quote(desc) self.redirect(url) else: logging.info("User is not logged in") desc = "You must be logged in to use the api." response["success"] = False response["response"] = "InvalidRequestError" response["description"] = desc response["code"] = 400 wrap_response(self, response) return else: owner = self.user logging.debug(content_type) if content_type == 'application/json': # d = write_to_api(json.loads(self.request.body), owner, content_type) d = write_to_api(json.loads(self.request.body), owner) else: d = write_to_api_params(self.request.arguments(), owner, content_type, False, self) wrap_response(self, d.to_api_object())
def post(self, data_id=None): response = {} response["success"] = True logging.info(self.request.headers) content_type = self.request.headers["Content_Type"] if not self.user: if content_type == "application/json": if "Authorization" not in self.request.headers: logging.info("No Authorization in headers") desc = "You must be logged in to use the API." response["success"] = False response["response"] = "AuthorizationError" response["description"] = desc response["code"] = 400 wrap_response(self, response) return if self.request.headers["Authorization"] == API_KEY: if not self.request.headers["From"]: logging.info("No email defined") desc = "Cannot find user." response["success"] = False response["response"] = "InvalidUserError" response["description"] = desc response["code"] = 400 wrap_response(self, response) return user_email = self.request.headers["From"].lower() query = User.query() owner = query.filter( User.current_email == user_email).get() if not owner: logging.info("Cannot find user") desc = "Cannot find user." response["success"] = False response["response"] = "InvalidUserError" response["description"] = desc response["code"] = 400 wrap_response(self, response) return else: token = Token.get_by_id( self.request.headers["Authorization"]) if not token: logging.info( "Cannot find token: " + str(self.request.headers["Authorization"])) desc = "The token you provided is invalid." response["success"] = False response["response"] = "InvalidTokenError" response["description"] = desc response["code"] = 400 wrap_response(self, response) return logging.info(token) session = token.session.get() if not session: logging.info("Cannot find session") desc = "The token has already expired." response["error"] = False response["response"] = "InvalidTokenError" response["description"] = desc response["code"] = 400 wrap_response(self, response) return logging.info(session) if session.expires < datetime.datetime.now( ) or session.status is False: logging.info("token has expired or not active") desc = "The token has already expired." response["success"] = False response["response"] = "InvalidTokenError" response["description"] = desc response["code"] = 400 wrap_response(self, response) return else: desc = "You must be logged in to use the API." if self.POST("r"): url = urllib.unquote(str(self.POST("r"))) else: url = self.request.referer if url: if "?" in url: url = url.split("?")[0] url += "?error=" + urllib.quote(desc) self.redirect(url) if not data_id: desc = "ID is missing from the request." if content_type == "application/json": response["success"] = False response["response"] = "MissingParametersError" response["description"] = desc response["code"] = 400 wrap_response(self, response) else: if self.POST("r"): url = urllib.unquote(str(self.POST("r"))) else: url = self.request.referer if url: if "?" in url: url = url.split("?")[0] url += "?error=" + urllib.quote(desc) self.redirect(url) return data = APIData.get_by_id(normalize_id(data_id)) if not data: desc = "Cannot find the package." if content_type == "application/json": response["success"] = False response["response"] = "InvalidIDError" response["description"] = desc response["code"] = 400 wrap_response(self, response) else: if self.POST("r"): url = urllib.unquote(str(self.POST("r"))) else: url = self.request.referer if url: if "?" in url: url = url.split("?")[0] url += "?error=" + urllib.quote(desc) self.redirect(url) return if data.archived: desc = "Cannot find the package." if content_type == "application/json": response["success"] = False response["response"] = "InvalidIDError" response["description"] = desc response["code"] = 400 wrap_response(self, response) else: if self.POST("r"): url = urllib.unquote(str(self.POST("r"))) else: url = self.request.referer if url: if "?" in url: url = url.split("?")[0] url += "?error=" + urllib.quote(desc) self.redirect(url) return desc = "There are missing parameters in your request." if content_type == "application/json": if not self.request.body: response["success"] = False response["response"] = "MissingParametersError" response["description"] = desc response["code"] = 400 wrap_response(self, response) return try: body = json.loads(self.request.body) except Exception as e: logging.info(e) desc = "Invalid JSON format." response["success"] = False response["response"] = "InvalidJSONError" response["description"] = desc response["code"] = 400 wrap_response(self, response) return tags = [] try: for key, value in body.items(): try: tags += create_tags(value) except Exception as e: logging.info("Cannot create tag from: ") logging.info(e) if key.startswith('unindexed_'): # unindexed_ ad_key = key.replace("unindexed_", "") data.additional_data[ad_key] = value.strip() if key.startswith('indexed_'): ad_key = key.replace("indexed_", "") data.additional_data[ad_key] = value for d in data.indexed_data: ad_key = key.replace("indexed_", "") if d.startswith(ad_key.upper()): try: data.indexed_data.remove(d) except Exception as e: logging.exception(e) logging.info("Cannot remove from list") data.indexed_data.append(create_indexed_tag( key, value)) if self.user: data.username = self.user.name data.user = self.user.key data.indexed_data = uniquify(data.indexed_data) data.tags = uniquify(tags) data.put() desc = "Data has been saved." response["success"] = True response["response"] = "Success" response["description"] = desc response["code"] = 200 response["data"] = data.to_api_object() wrap_response(self, response) except Exception as e: logging.exception(e) desc = "A server error occured. Please try again later." response["success"] = False response["response"] = "ServerError" response["description"] = desc response["code"] = 500 wrap_response(self, response) else: if not self.request.arguments(): if self.POST("r"): url = urllib.unquote(str(self.POST("r"))) else: url = self.request.referer if url: if "?" in url: url = url.split("?")[0] url += "?error=" + urllib.quote(desc) self.redirect(url) return tags = [] try: for arg in self.request.arguments(): for d in data.indexed_data: ad_key = arg.replace("indexed_", "") if d.startswith(ad_key.upper()): try: data.indexed_data.remove(d) except Exception as e: logging.exception(e) logging.info("Cannot remove from list") if arg.startswith('unindexed_'): ad_key = arg.replace("unindexed_", "") ad_value = self.request.POST.get(arg) data.additional_data[ad_key] = ad_value.strip() try: tags += create_tags(ad_value) except Exception as e: logging.info("Cannot create tag from: ") logging.info(e) if arg.startswith('indexed_'): ad_key = arg.replace("indexed_", "") ad_value = self.request.POST.get(arg) data.additional_data[ad_key] = ad_value try: tags += create_tags(ad_value) except Exception as e: logging.info("Cannot create tag from: ") logging.info(e) data.indexed_data.append( create_indexed_tag(arg, self.request.POST.get(arg))) if arg.startswith('file_'): filename = BUCKET_NAME filename += random_string(20) + "/" ad_key = arg.replace("file_", "") data.additional_data[ad_key] = {} try: # try: file_name = self.request.POST.get(arg).filename filename += file_name gcs_file = gcs.open( filename, 'w', options={'x-goog-acl': 'public-read'}) gcs_file.write(self.request.get(arg)) gcs_file.close() full_url = "https://storage.googleapis.com" + filename # data.additional_data["file"]["file_url"] = full_url data.file_url = full_url data.additional_data[ad_key]["file_url"] = full_url try: blob_key = blobstore.create_gs_key("/gs" + filename) data.serving_url = images.get_serving_url( blob_key) data.additional_data[ad_key][ "serving_url"] = data.serving_url data.gcs_key = blobstore.BlobKey(blob_key) except Exception as e: logging.exception(e) logging.error("not an image??") data.additional_data[ad_key][ "serving_url"] = full_url except AttributeError, e: logging.exception(e) logging.exception("NO FILE ATTACHED") if self.user: data.username = self.user.name data.user = self.user.key data.indexed_data = uniquify(data.indexed_data) data.tags = uniquify(tags) data.put() desc = "Data has been updated." if self.POST("r"): url = urllib.unquote(str(self.POST("r"))) else: url = self.request.referer if url: if "?" in url: url = url.split("?")[0] url += "?success=" + urllib.quote(desc) self.redirect(url) else: response["success"] = True response["response"] = "Success" response["description"] = desc response["code"] = 200 response["data"] = data.to_api_object() wrap_response(self, response) except Exception as e: logging.exception(e) desc = "A server error occured. Please try again later." if self.POST("r"): url = urllib.unquote(str(self.POST("r"))) else: url = self.request.referer if url: if "?" in url: url = url.split("?")[0] url += "?error=" + urllib.quote(desc) self.redirect(url) else: response["success"] = False response["response"] = "ServerError" response["description"] = desc response["code"] = 500 wrap_response(self, response)