def put(self, entity: BE) -> int:
"""Save an entity in the datastore, or update an existing one
:param entity: the entity to save
:type entity: a class that inherits BaseEntity
:returns: the id of the entity
:rtype: int
:raises: ValueError if entity is not a subclass of BaseEntity
"""
# Just to be sure the given class is a subclass of BaseEntity
if not isinstance(entity, BaseEntity):
raise ValueError("Param entity has to be subclass of BaseEntity")
kind = entity.get_entity_name()
if BaseEntity.NO_ID == entity.id:
self._logger.debug("Saving a new item of kind {}".format(kind))
key = self._client.key(kind)
else:
self._logger.debug(
"Updating an existing item of kind {} with id {}".format(
kind, entity.id))
key = self._client.key(kind, entity.id)
with self._client.transaction():
datastore_entity = datastore.Entity(key=key)
# Updates all the fields of the entity
datastore_entity.update(entity.to_dict())
self._client.put(datastore_entity)
if BaseEntity.NO_ID == entity.id:
entity.id = datastore_entity.key.id
return entity.id
def test_already_completed_registration(self):
registration_id = "registration_id_mock"
request_data = {"code": "code", "registration_id": registration_id}
key = datastore_client.key(DATA_STORE_REGISTRATION_KIND,
registration_id)
registration = datastore.Entity(key=key)
registration.update({
"code": "code",
"msisdn": "msisdn",
"date": datetime.now(tz=pytz.utc),
"registration_id": registration_id,
"sms_send": False,
"ip": "ip",
"status": REGISTRATION_STATUS_COMPLETED,
})
datastore_client.put(registration)
self.registrations_entities_ids_to_delete.append(registration_id)
response = requests.post(f"{BASE_URL}{CONFIRM_REGISTRATION_ENDPOINT}",
json=request_data)
assert response.status_code == 422
assert response.json()["message"] == "Invalid data"
def namespace_run_query(client):
# Create an entity in another namespace.
task = datastore.Entity(
client.key("Task", "sample-task", namespace="google"))
client.put(task)
# [START datastore_namespace_run_query]
# All namespaces
query = client.query(kind="__namespace__")
query.keys_only()
all_namespaces = [entity.key.id_or_name for entity in query.fetch()]
# Filtered namespaces
start_namespace = client.key("__namespace__", "g")
end_namespace = client.key("__namespace__", "h")
query = client.query(kind="__namespace__")
query.key_filter(start_namespace, ">=")
query.key_filter(end_namespace, "<")
filtered_namespaces = [entity.key.id_or_name for entity in query.fetch()]
# [END datastore_namespace_run_query]
return all_namespaces, filtered_namespaces
def update_infra(
self,
project: str,
tables_to_delete: Sequence[Union[FeatureTable, FeatureView]],
tables_to_keep: Sequence[Union[FeatureTable, FeatureView]],
partial: bool,
):
from google.cloud import datastore
client = self._initialize_client()
for table in tables_to_keep:
key = client.key("Project", project, "Table", table.name)
entity = datastore.Entity(key=key)
entity.update({"created_ts": datetime.utcnow()})
client.put(entity)
for table in tables_to_delete:
_delete_all_values(
client, client.key("Project", project, "Table", table.name))
# Delete the table metadata datastore entity
key = client.key("Project", project, "Table", table.name)
client.delete(key)
def to_datastore(ds: datastore.Client, kind: TipoEntidade, entidade: dict):
"""Converte os dados a serem gravados do Google Datastore para o formato padronizado de cada tipo de campo.
Retorno:
Entidade do Google Datastore.
"""
if not entidade:
return None
elif type(entidade) != dict:
entidade = entidade.__dict__
# Varre os atributos da entidade atualizando os padrões de tipos de dados
for atributo in entidade.keys():
if atributo == 'id':
key = ds.key(kind.value, entidade['id'])
# Tratamento de campos data e data/hora
# elif atributo.startswith(('dt_','dth_')):
# # Converte o campo datatime para string no padrão ISO
# entidade[atributo] = entidade[atributo].isoformat()
elif type(entidade[atributo]).__name__ == 'date':
entidade[atributo] = entidade[atributo].isoformat()
elif type(entidade[atributo]) == datetime:
entidade[atributo] = entidade[atributo].isoformat()
# Se entidade não possuia campo 'id' gera chave automática do datastore
if not key:
key = ds.key(kind.value)
# entity = datastore.Entity(
# key=key) ,
# exclude_from_indexes=['description'])
# Cria uma entidade com a chave obtida
entity = datastore.Entity(key=key)
# Atualiza a entidade com os atributos recebidos no dictionary
entity.update(entidade)
return entity
def add_timestamp_keys(client=None):
if client is None:
# Get a client that uses the test dataset.
client = datastore.Client()
num_batches = 2
batch_size = 500
timestamp_micros = set()
for batch_num in range(num_batches):
with client.batch() as batch:
for seq_no in range(batch_size):
print(
"time_time: batch: {}, sequence: {}".format(
batch_num, seq_no))
now_micros = int(time.time() * 1e6)
while now_micros in timestamp_micros:
now_micros = int(time.time() * 1e6)
timestamp_micros.add(now_micros)
key = client.key('timestamp_key', now_micros)
entity = datastore.Entity(key=key)
entity['batch_num'] = batch_num
entity['seq_no'] = seq_no
batch.put(entity)
def test_failure_with_contention(self):
contention_prop_name = 'baz'
local_client = clone_client(Config.CLIENT)
# Insert an entity which will be retrieved in a transaction
# and updated outside it with a contentious value.
key = local_client.key('BreakTxn', 1234)
orig_entity = datastore.Entity(key=key)
orig_entity['foo'] = u'bar'
local_client.put(orig_entity)
self.case_entities_to_delete.append(orig_entity)
with self.assertRaises(Conflict):
with local_client.transaction() as txn:
entity_in_txn = local_client.get(key)
# Update the original entity outside the transaction.
orig_entity[contention_prop_name] = u'outside'
Config.CLIENT.put(orig_entity)
# Try to update the entity which we already updated outside the
# transaction.
entity_in_txn[contention_prop_name] = u'inside'
txn.put(entity_in_txn)
def upload_text():
text = request.form["text"]
# Analyse sentiment using Sentiment API call
sentiment = analyze_text_sentiment(text)[0].get('sentiment score')
# Create a Cloud Datastore client.
datastore_client = datastore.Client()
# Fetch the current date / time.
current_datetime = datetime.now()
# The kind for the new entity. This is so all 'Sentences' can be queried.
kind = "Sentences"
# Create the Cloud Datastore key for the new entity.
key = datastore_client.key(kind, 'sample_task')
# Alternative to above, the following would store a history of all previous requests as no key
# identifier is specified, only a 'kind'. Datastore automatically provisions numeric ids.
# key = datastore_client.key(kind)
# Construct the new entity using the key. Set dictionary values for entity
entity = datastore.Entity(key)
entity["text"] = text
entity["timestamp"] = current_datetime
entity["sentiment"] = sentiment
# Save the new entity to Datastore.
datastore_client.put(entity)
# test save results to a csv
save_txt_sentiment_to_csv(text, sentiment)
# Redirect to the home page.
return redirect("/")
def analyseEntity():
text = request.form["text"]
# Analyse sentiment using Sentiment API call
output = gcp_analyze_entities(text, 0)
# Create a Cloud Datastore client.
datastore_client = datastore.Client()
# Fetch the current date / time.
current_datetime = datetime.now()
# The kind for the new entity. This is so all 'Sentences' can be queried.
kind = "Entities"
# Create the Cloud Datastore key for the new entity.
key = datastore_client.key(kind, 'sample_task')
print(json.dumps(output, indent=4))
# Alternative to above, the following would store a history of all previous requests as no key
# identifier is specified, only a 'kind'. Datastore automatically provisions numeric ids.
# key = datastore_client.key(kind)
# Construct the new entity using the key. Set dictionary values for entity
entity = datastore.Entity(key)
entity["text"] = text
entity["timestamp"] = current_datetime
entity["entities"] = output
# Save the new entity to Datastore.
datastore_client.put(entity)
# Redirect to the home page.
return redirect("/")
def start_time(project_id):
url = clockify_api('/workspaces/{workspaceId}/timeEntries/'.format(
workspaceId=WORKSPACE_ID
))
current_task = {
"start": datetime.utcnow().isoformat() + 'Z',
"billable": True,
"description": "My text 2",
"projectId": project_id,
"taskId": None,
"tagIds": []
}
response = requests.post(url, headers=headers, json=current_task)
last_response = response.json()
current_task['taskId'] = last_response['id']
client = datastore.Client()
key = client.key('current_task', 'last')
entity = datastore.Entity(key=key)
entity.update(current_task)
client.put(entity)
def AddNewRoom(index, room_number, users_list=[], class_list=[]):
# AddNewRoom() will add to the "Room" entity
# first argument is the index in database
#
# second argument should be the room number
# that has a lock on it i.e. BE301, BE340A, E2-599
#
# third argument is array of Group that
# have acces to the room i.e. CMPE123A, CMPE121, CMPE167L
ds = get_client()
entity = 'Rooms'
ID = 'index{}'.format(room_number)
new_key = ds.key(entity, ID)
# Prepares the new entity
new_room_entity = datastore.Entity(key=new_key)
new_room_entity.update({
'Group': class_list,
'Room_Number': room_number,
'Users': users_list,
'Capacity': 0,
'Last_Login_time': None
})
ds.put(new_room_entity)
def Trasact(self, senderID, recieverID):
transactionUID = sha256(str(
datetime.now()).encode("UTF-8")).hexdigest()
client = datastore.Client()
key = client.key('Mempool', transactionUID)
entity = datastore.Entity(key=key)
entity.update({
"UID":
transactionUID,
"senderID":
senderID,
"recieverID":
recieverID,
"amount":
1,
"fee":
.02,
"data":
str(transactionUID) + "," + str(senderID) + "," + str(recieverID) +
"," + str(1) + "," + str(.02)
})
client.put(entity)
self.getMempool()
def postdiscourse():
"""TODO: Fill in Documentation
.. :quickref: UNDOCUMENTED;
"""
received_form_response = json.loads(request.data.decode('utf-8'))
current_date = datetime.utcnow()
user_uuid = received_form_response.get("user_uuid", "Error")
post_id = received_form_response.get("post_id", "Error")
# Add the user to the users kind of entity
key = datastore_client.key('ForumQuestions')
# Indexes every other column except the description
device_reg_task = datastore.Entity(key, exclude_from_indexes=[])
device_reg_task.update({
'user_uuid': user_uuid,
'post_id': post_id,
'current_date': current_date
})
datastore_client.put(device_reg_task)
return success_response(message="success")
def get_car_location(self, car_location):
loc_key = self.client.key("CarLocation", car_location["license_hash"])
entity = self.client.get(loc_key)
if entity is None:
entity = datastore.Entity(key=loc_key)
if "when" not in entity or entity["when"] < car_location["when"]:
entity.update(
{
"geometry": car_location["geometry"],
"when": car_location["when"],
"status": car_location.get("what", None),
"license": car_location.get("license", None),
}
)
return entity
logging.debug("Populate location {} - {}".format(entity.key, entity))
else:
logging.debug(
"Skipping {} - late notification {}/{}".format(
car_location["license_hash"], car_location["when"], entity["when"]
)
)
return None
def namespace_run_query(client):
# Create an entity in another namespace.
task = datastore.Entity(
client.key('Task', 'sample-task', namespace='google'))
client.put(task)
# [START namespace_run_query]
# All namespaces
query = client.query(kind='__namespace__')
query.keys_only()
all_namespaces = [entity.key.id_or_name for entity in query.fetch()]
# Filtered namespaces
start_namespace = client.key('__namespace__', 'g')
end_namespace = client.key('__namespace__', 'h')
query = client.query(kind='__namespace__')
query.key_filter(start_namespace, '>=')
query.key_filter(end_namespace, '<')
filtered_namespaces = [entity.key.id_or_name for entity in query.fetch()]
# [END namespace_run_query]
return all_namespaces, filtered_namespaces
def add_task(client, n, category, shard_length, gcs_output_path):
key = client.key(task_kind)
task = datastore.Entity(
key, exclude_from_indexes=[])
task.update({
'task_number': n,
'status': 'Queued',
'created': time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()),
'category' : category,
'shard_length': shard_length,
'shard_index': n,
'gcs_output_path': gcs_output_path,
'completed_time' : None,
'elapsed_time_s' : None,
'shard_length_tiles' : None,
'tfrecord_size_MBi' : None,
'tissue':tissue
})
client.put(task)
return task.key
def put_readings(name, plantType, temp, humid, light, moist, imgurl):
t = str(int(time.time()))
user_key = datastore_client.key(kind, name + "-" + t)
user = datastore.Entity(key=user_key)
user['name'] = name
user['type'] = plantType
user['temp'] = temp
user['humid'] = humid
user['light'] = light
user['moist'] = moist
user['lastupdate'] = t
user['pic'] = imgurl
datastore_client.put(user)
print('Saved to google cloud -- {}: {} {}'.format(user.key.name,
user['name'],
user['pic']))
payload = {}
payload["name"] = name
payload['type'] = plantType
payload['temp'] = temp
payload['humid'] = humid
payload['light'] = light
payload['moist'] = moist
payload['lastupdate'] = t
payload['pic'] = imgurl
print("payload ready")
print(payload)
result = db.readings.insert_one(payload)
print(result)
def add_user(email, password):
logging.info("add_user [{0}], [{1}]".format(email, password))
password_hash = sha256_as_hex(password)
try:
op_datetime = datetime.utcnow()
logging.info("op_datetime [{0}]".format(op_datetime))
with db.transaction():
logging.info("Getting key")
key = get_new_entity_key()
logging.info("Key is [{0}]".format(key))
# parent_key = db.key('zxsh', 'message_list')
# key = db.key('Task', 'sample_task', parent=parent_key)
ent = db.get(key)
logging.info("Entity is [{0}]".format(ent))
if not ent:
logging.info("Retrieving entity")
ent = datastore.Entity(key)
logging.info("Updating entity")
ent.update({
'body': 'sample',
'email': email,
'password_hash': password,
'cre_dt': op_datetime,
'verify': False,
'status': UserStatus.UnvettedAccount
})
logging.info("Putting entity")
db.put(ent)
logging.info("All ok returning key")
return key
#print("Create loan request: {0}".format(data['nric']))
except Exception as e:
logging.error(str(dir(e)))
logging.error(e)
def create_lot(space_count):
i = 0
while i < space_count:
i += 1
ID = 'Space-{}'.format(i)
new_key = datastore_client.key(parking_lot, ID)
new_spot = datastore.Entity(key=new_key)
new_spot['Sub-Structure'] = subStructure
new_spot['Code'] = 0
new_spot['License Plate'] = u'Empty'
new_spot['Permission'] = permission
new_spot['Occupied'] = False
new_spot['Authorized'] = False
new_spot['Request Pic'] = False
new_spot['Timeframe'] = datetime.now()
new_spot['Image'] = 'image'
new_spot['Day'] = 0
new_spot['Hour'] = 0
# Saves the entity
datastore_client.put(new_spot)
print('Saved {} {} {}: {}'.format(new_spot['Sub-Structure'],
new_spot['Code'], new_spot.key.name,
new_spot['Occupied']))
def manual_upload_category():
# NB BOTH cat_id and "value" as strings!
articles_querified = [["cat_id", "value"], ["cat_id2", "value2"]]
articles_querified = []
task_list = [] # Used for bulk upload to datastore
# Loop over all articles and insert one by one
for article in articles_querified:
id = article[0]
# Only happens if we have not seen this item before. Then we add it to
# unmapped items with an id of -1.
task_key = datastore_client.key(DATASTORE_KIND_CATEGORIES, id)
task = datastore.Entity(key=task_key)
task["cat_id"] = article[1]
task_list.append(task)
print('Added {}: {}'.format(task.key.name, task['cat_name']))
# End for loop
# Saves multiple entities at once:
datastore_client.put_multi(task_list)
def test_registration_expired(self):
registration_id = "registration_id_mock"
request_data = {
"code": "code",
"registration_id": registration_id,
"lang": LANG
}
key = datastore_client.key(DATA_STORE_REGISTRATION_KIND,
registration_id)
registration = datastore.Entity(key=key)
registration.update({
"code":
"code",
"msisdn":
"msisdn",
"date":
datetime.now(tz=pytz.utc) - timedelta(minutes=11),
"registration_id":
registration_id,
"sms_send":
False,
"ip":
"ip",
"status":
REGISTRATION_STATUS_PENDING,
})
datastore_client.put(registration)
self.registrations_entities_ids_to_delete.append(registration_id)
response = requests.post(f"{BASE_URL}{CONFIRM_REGISTRATION_ENDPOINT}",
json=request_data)
assert response.status_code == 422
assert response.json()["message"] == _get_message(
MESSAGE_REGISTRATION_EXPIRED, LANG)
def create_user(ds, username, profile):
key = path_to_key(ds, '{0}.user'.format(username))
entity = datastore.Entity(key)
entity.update(profile)
ds.put(entity)
def store_search():
if 'user' not in session:
return redirect('/')
user = firebase_auth.get_account_info(session['user'])
my_files = get_files()
if request.method == 'POST':
if request.files:
file_to_upload = request.files["file"]
try:
extension = os.path.splitext(file_to_upload.filename)[1]
name = os.path.splitext(file_to_upload.filename)[0]
if extension not in ['.txt', '.mp3']:
return render_template(
'file_storage.html',
len=len(my_files),
names=my_files,
upload_error="You have to choose a txt or mp3 file!",
user_email=user['users'][0]['email'])
if name in my_files:
return render_template(
'file_storage.html',
len=len(my_files),
names=my_files,
upload_error="You have already a file with this name!",
user_email=user['users'][0]['email'])
uid = user['users'][0]['localId']
new_file_path = 'user/' + uid + '/textFiles/' + file_to_upload.filename
file_url = firebase_storage.child(new_file_path).put(
file_to_upload, session['user'])
entity1 = datastore.Entity(key=data_store.key('file-token'))
entity1.update({
"file_name": new_file_path,
"access_token": file_url['downloadTokens']
})
data_store.put(entity1)
extension = os.path.splitext(file_to_upload.filename)
if extension[1] == '.mp3':
translation = record_sound(new_file_path)
new_file_path = 'user/' + uid + '/textFiles/' + extension[
0] + ".txt"
file_url = firebase_storage.child(new_file_path).put(
translation.encode('utf-8'), session['user'])
entity2 = datastore.Entity(
key=data_store.key('file-token'))
entity2.update({
"file_name": new_file_path,
"access_token": file_url['downloadTokens']
})
data_store.put(entity2)
else:
query = data_store.query(kind="file-token")
query.add_filter("file_name", "=", new_file_path)
token = ""
for result in query.fetch(limit=1):
token = result['access_token']
print(token)
url = firebase_storage.child(new_file_path).get_url(token)
response = req.urlopen(url).read()
text = response.decode("UTF-8")
translation = create_voice_file(text)
new_file_path = 'user/' + uid + '/textFiles/' + extension[
0] + ".mp3"
file_url = firebase_storage.child(new_file_path).put(
translation, session['user'])
entity2 = datastore.Entity(
key=data_store.key('file-token'))
entity2.update({
"file_name": new_file_path,
"access_token": file_url['downloadTokens']
})
data_store.put(entity2)
return redirect(request.url)
except Exception as e:
print(e)
delete_file(file_to_upload.filename)
return render_template('file_storage.html',
len=len(my_files),
names=my_files,
upload_error="The file is too big.",
user_email=user['users'][0]['email'])
return render_template('file_storage.html',
user_email=user['users'][0]['email'],
len=len(my_files),
names=my_files)
from google.cloud import datastore
client = datastore.Client()
kind = 'Task'
key = client.key(kind)
# key = client.key(kind, 1)
task = datastore.Entity(key)
task.update({
'category': 'Personal',
'done': False,
'priority': 4,
'description': 'Learn Cloud Datastore'
})
client.put(task)
# client.put_multi([task1, task2])
def save_question(question):
key = datastore_client.key('Question')
q_entity = datastore.Entity(key=key)
for q_prop, q_val in question.items():
q_entity[q_prop] = q_val
datastore_client.put(q_entity)
def put(self, key: str, value: bytes):
entity = datastore.Entity(self.client.key(key))
entity.update({'data': value})
self.client.put(entity)
def register_launch():
ds = datastore.Client()
new_launch = datastore.Entity(key=ds.key('launches'))
new_launch.update({"data": request.get_json()["data"]})
ds.put(new_launch)
return json.dumps({"success": True})
def main(request):
import google.auth
from google.cloud import datastore, storage, vision
import os
from datetime import datetime
# Set the project ID
PROJECT_ID = os.environ['GCP_PROJECT']
FUNCTION_REGION = os.environ['FUNCTION_REGION']
NONCE = os.environ.get('NONCE', 'Specified environment variable is not set.')
RESOURCE_PREFIX = os.environ.get('RESOURCE_PREFIX', 'Specified environment variable is not set.')
LEVEL_NAME = os.environ.get('LEVEL_NAME', 'Specified environment variable is not set.')
CLOUD_STORAGE_BUCKET = f'{RESOURCE_PREFIX}-bucket-{NONCE}'
KIND = f'{RESOURCE_PREFIX}-{NONCE}-{PROJECT_ID}'
# Get credential of cloud function account
credentials, project_id = google.auth.default()
#score function url
surl = f'https://{FUNCTION_REGION}-{PROJECT_ID}.cloudfunctions.net/scores-f-{NONCE}'
#check function url
url=f'https://{FUNCTION_REGION}-{PROJECT_ID}.cloudfunctions.net/{RESOURCE_PREFIX}-f-check-{NONCE}'
#upload url
up_url = f'/{RESOURCE_PREFIX}-f-access-{NONCE}'
#err_build=request.args['err_build'] if request.args and 'err_build' in request.args else ''
err_build = ''
err_query=''
image_entities = []
if request.files and 'file' in request.files:
try:
photo = request.files['file']
# Create a Cloud Storage client.
storage_client = storage.Client(credentials=credentials)
# Get the bucket that the file will be uploaded to.
bucket = storage_client.get_bucket(CLOUD_STORAGE_BUCKET)
# Create a new blob and upload the file's content.
blob = bucket.blob(photo.filename)
blob.upload_from_string(photo.read(), content_type=photo.content_type)
# Make the blob publicly viewable.
blob.make_public()
# Create a Cloud Vision client.
vision_client = vision.ImageAnnotatorClient()
# Use the Cloud Vision client to detect a face for our image.
source_uri = 'gs://{}/{}'.format(CLOUD_STORAGE_BUCKET, blob.name)
image = vision.Image(source=vision.ImageSource(gcs_image_uri=source_uri))
faces = vision_client.face_detection(image).face_annotations
# If a face is detected, save to Datastore the likelihood that the face
# displays 'joy,' as determined by Google's Machine Learning algorithm.
if len(faces) > 0:
face = faces[0]
# Convert the likelihood string.
likelihoods = [
'Unknown', 'Very Unlikely', 'Unlikely', 'Possible', 'Likely',
'Very Likely']
face_joy = likelihoods[face.joy_likelihood]
else:
face_joy = 'Unknown'
# Create a Cloud Datastore client.
datastore_client = datastore.Client(credentials=credentials)
# Fetch the current date / time.
current_datetime = datetime.now()
# The kind for the new entity.
kind = KIND
# The name/ID for the new entity.
name = blob.name
# Create the Cloud Datastore key for the new entity.
key = datastore_client.key(kind, name)
# Construct the new entity using the key. Set dictionary values for entity
# keys blob_name, storage_public_url, timestamp, and joy.
entity = datastore.Entity(key)
entity['blob_name'] = blob.name
entity['image_public_url'] = blob.public_url
entity['timestamp'] = current_datetime
entity['joy'] = face_joy
# Save the new entity to Datastore.
datastore_client.put(entity)
except Exception as e:
err_build = str(e)
if err_build == '':
return redirect(up_url)
try:
#Build datastore REST API python object
client = datastore.Client(credentials=credentials )
# Use the Cloud Datastore client to fetch information from Datastore about each photo.
query = client.query(kind=KIND)
image_entities = list(query.fetch())
except Exception as e:
err_query=str(e)
return render_template(f'{RESOURCE_PREFIX}-access.html', url=url,
err_build=err_build,err_query=err_query,prefix=RESOURCE_PREFIX, level_name=LEVEL_NAME,
nonce=NONCE,surl=surl,image_entities=image_entities,up_url=up_url)
def execute(request):
# Check Payload
if not request.form or not 'external_ref' in request.form:
return ('Bad request: External Ref is required', 400)
if not request.form or not 'external_key' in request.form:
return ('Bad request: External Key is required', 400)
if not request.files or not 'file' in request.files:
return ('Bad request: File is required', 400)
# Get external ref
external_ref = request.form.get('external_ref')
# Get external key
external_key = request.form.get('external_key')
# External ref uuid name
external_ref_uuid_name = 'extref=%s%s' % (slugify.slugify(external_ref), external_key)
# Generate UUID external ref
external_ref_uuid = uuid.uuid5(uuid.NAMESPACE_X500, external_ref_uuid_name)
# Get external ref uuid str
external_ref_uuid_str = str(uuid.uuid5(uuid.NAMESPACE_X500, external_ref_uuid_name))
# Get unique key external ref
external_ref_key = hashlib.md5(external_ref_uuid.bytes).hexdigest()
try: # Get/Create bucket
client_storage = storage.Client()
bucket = client_storage.create_bucket(BUCKET_NAME)
except google.api_core.exceptions.Conflict: # Bucket already exists
bucket = client_storage.get_bucket(BUCKET_NAME)
except Exception as e: # Any exception
return (u'Error: %s' % e, 500)
# Bucket file uuid name
encryption_key_name = 'extrefkey=%s' % external_ref_key
# Generate UUID Bucket file
encryption_key_uuid = uuid.uuid5(uuid.NAMESPACE_X500, encryption_key_name)
# Generate encryption key Bucket file
encryption_key = hashlib.md5(encryption_key_uuid.bytes).hexdigest()
try: # Get file
file = request.files.get('file')
# Generate path file
path_file = "%s/%s/%s" % (str(external_ref_uuid), str(encryption_key_uuid), file.filename)
# Create encripty Blob
blob = Blob(name=path_file, bucket=bucket, encryption_key=encryption_key)
except Exception as e:
return (u'Error: %s' % e, 500)
try: # Read and upload file into bucket
buff = file.read()
blob.upload_from_string(buff, content_type=file.content_type)
except Exception as e: # Any exception
return (e, 500)
# Generate File hash
file_hash = hashlib.md5(buff).hexdigest()
try: # Create entity
client_datastore = datastore.Client()
# Generate Datastore Key
item_key = client_datastore.key(DS_KIND, "%s-%s" % (external_ref_key, file_hash))
# Entity
item = datastore.Entity(key=item_key,) # Insert user key
item['external_ref_key'] = external_ref_key
item['file_hash'] = file_hash
item['file_path'] = path_file
item['file_content_type'] = file.content_type
client_datastore.put(item)
except Exception as e: # Any exception
return (u'Error: %s' % e, 500)
# Data return
data = json.dumps({'filename': file.filename,'hash' : file_hash})
# Response
return Response(data, mimetype='application/json')
def increment_latest_id(self):
cur = self._get_latest_id()
entity = datastore.Entity(key=self.key)
entity.update({'active': cur + 1})
self.client.put(entity)
return cur