def store(event, context):
"""store file"""
# check the store on load configuration
if runtime_context.STORE:
return {
'statusCode':
400,
'body':
json.dumps({
'message':
'Configured to store files at the time of upload.'
})
}
# get the list of files from the request
file_ids = json.loads(event.get('body'))
file_ids = file_ids[:runtime_context.
QUERY_LIMIT] # limit the number of files to store
stored_file_ids = []
dt = datetime.utcnow()
for file_id in file_ids:
FileModel.update({
'id': file_id,
'stored_at': dt,
})
LOGGER.debug(
'Files item updated (stored). service=ddb method=update_item id={}'
.format(file_id))
stored_file_ids.append(file_id)
return {"statusCode": 200, "body": json.dumps(stored_file_ids)}
def post(cls):
'''Upload user submission'''
try:
user_mail = get_jwt_identity()
# check current submission counts
daily_counts = FileModel.get_interval_upload_count_by_mail(
email=user_mail, AOEtime=get_AOE_today(to_str=False))
weekly_counts = FileModel.get_interval_upload_count_by_mail(
email=user_mail, AOEtime=get_AOE_week(to_str=False))
if (daily_counts >= configs["DAILY_SUBMIT_LIMIT"]) or (weekly_counts >= configs["WEEKLY_SUBMIT_LIMIT"]):
return {"message": f"Exceed Submission Limit: You have submitted {daily_counts} times today and {weekly_counts} times this week."}, HTTPStatus.FORBIDDEN
# file validation
file = request.files['file']
if file.filename == "":
return {"message": "No file selected."}, HTTPStatus.FORBIDDEN
if not file_upload.zipfile_check(file):
return {"message": "Wrong file format."}, HTTPStatus.FORBIDDEN
# load form data
formData = publicFormSchema.load(request.form)
# get file path
upload_count = FileModel.get_upload_count_by_mail(
email=user_mail) + 1
folder = file_upload.create_folder(user_mail, str(upload_count))
file_path = file_upload.get_full_path(folder, file.filename)
# add column for db
formData.update({"email": user_mail, "filePath": file_path})
fileObj = FileModel(**formData)
scoreObj = ScoreModel()
fileObj.scores.append(scoreObj)
fileObj.save_to_db()
try:
file.save(file_path)
# start processing
thread = Thread(target=metric_calculate_pipeline, kwargs={"file_path": file_path,
"submitUUID": formData["submitUUID"]})
thread.start()
return {"message": "Upload successfully!"}, HTTPStatus.OK
except Exception as e:
fileObj.delete_from_db() # Rollback
return {"message": "Internal Server Error!"}, HTTPStatus.INTERNAL_SERVER_ERROR
except ValidationError as e:
return {"message": "There's something worng with your input!"}, HTTPStatus.BAD_REQUEST
except Exception as e:
print(e)
return {"message": "Internal Server Error!"}, HTTPStatus.INTERNAL_SERVER_ERROR
def info(event, context):
"""get the info for a list of files"""
query_string_parameters = event.get('queryStringParameters') or {}
include_deleted = query_string_parameters.get('deleted') == 'yes'
include_nonstored = query_string_parameters.get('nonstored') == 'yes'
LOGGER.debug('File info includes deleted={} includes non-stored={}'.format(
include_deleted, include_nonstored))
file_ids = json.loads(event.get('body'))
file_ids = file_ids[:runtime_context.
QUERY_LIMIT] # limit the number of files queried
files = FileModel.get_by_ids(file_ids)
for file in files:
if not file.get('deleted_at') and file.get(
'stored_at'): # only return not deleted and stored
file['url'] = get_presigned_url_for_download(file)
else:
if (file.get('deleted_at')
and include_deleted) or (not file.get('stored_at')
and include_nonstored):
pass
else: # remove deleted or non-stored
files.remove(file)
return {
"statusCode": 200,
"body": json.dumps(files, cls=utils.DateTimeEncoder)
}
def get(cls):
current_user_id = get_jwt_identity()
paths = [
file.path
for file in FileModel.get_unique_filepaths(current_user_id)
]
return {"file_paths": paths}, 200
def file_path(self):
"""
图片文件路径
:return:
"""
from models.file import FileModel
file_model = FileModel().dao_get(self.file_id) # type: FileModel
return file_model.file_path if is_not_empty(file_model) else None
def delete(event, context):
"""delete files"""
file_ids = json.loads(event.get('body'))
deleted_file_ids = []
for file_id in file_ids:
# NOTE: there is no check if file has already been deleted
FileModel.update({'id': file_id, 'deleted_at': datetime.utcnow()})
LOGGER.debug(
'Files item updated (deleted). service=ddb method=update_item id={}'
.format(file_id))
S3_CLIENT.delete_object(Bucket=runtime_context.BUCKET_NAME,
Key=file_id)
LOGGER.debug(
'S3 object deleted. service=s3 method=delete_object id={}'.format(
file_id))
deleted_file_ids.append(file_id)
return {"statusCode": 200, "body": json.dumps(deleted_file_ids)}
def file_md5(self):
"""
图片文件md5值
:return:
"""
from models.file import FileModel
file_model = FileModel().dao_get(self.file_id) # type: FileModel
return file_model.md5_id if is_not_empty(file_model) else None
def file_data(self):
"""
图片文件的base64字符
:return:
"""
from models.file import FileModel
from utils.encodes import file_to_base64
file_model = FileModel().dao_get(self.file_id) # type: FileModel
file_path = file_model.file_path
return file_to_base64(file_path) if os.path.isfile(file_path) else None
def get(cls):
'''Get user all submission info'''
try:
user_mail = get_jwt_identity()
submission_records = FileModel.find_by_email(email=user_mail).all()
submission_info = submission_records_parser(
submission_records, configs, mode="individual")
return make_response(jsonify({"submission_info": submission_info}), HTTPStatus.OK)
except Exception as e:
return {"message": "Internal Server Error!"}, HTTPStatus.INTERNAL_SERVER_ERROR
def expire(event, context):
"""remove files that are uploaded, not stored, and older than the expiration time
scheduled event
"""
# scan the database for expired files
expiry_at = datetime.utcnow() - runtime_context.NONSTORED_TIMEOUT
files = FileModel.list_expired(expiry_at)
# remove all files and all items one-by-one
for file in files:
file_id = file['id']['S']
FileModel.update({'id': file_id, 'deleted_at': datetime.utcnow()})
LOGGER.debug(
'Files item updated (expired). service=ddb method=update_item id={}'
.format(file_id))
S3_CLIENT.delete_object(Bucket=runtime_context.BUCKET_NAME,
Key=file_id)
LOGGER.debug(
'S3 object deleted. service=s3 method=delete_object id={}'.format(
file_id))
def dao_get_source_files(self, id):
"""
查询流水源文件
:param id:
:return:
"""
return FileModel().query.\
join(ImgDetailModel, ImgDetailModel.parent_file_id == FileModel.id).\
join(ImgDataModel, ImgDataModel.id == ImgDetailModel.img_data_id).\
filter(ImgDataModel.id == id).\
distinct().all()
def uploaded(event, context):
"""S3 event triggers when file is uploaded
event: https://docs.aws.amazon.com/AmazonS3/latest/dev/notification-content-structure.html
"""
dt = datetime.utcnow()
# NOTE: the event might include multiple records
for r in event['Records']:
file_id = r['s3']['object']['key']
file = {
'id': file_id,
'size': r['s3']['object']['size'],
'type': get_s3_file_type(file_id),
'uploaded_at': dt,
}
if runtime_context.STORE:
file['stored_at'] = dt
FileModel.update(file)
LOGGER.debug(
'Files item updated (uploaded). service=ddb method=update_item id={}'
.format(file_id))
return {"statusCode": 200}
def patch(cls, submitID):
'''Change user submission show on leaderboard or not by uuid'''
try:
user_mail = get_jwt_identity()
submission_record = FileModel.find_by_submitID(submitUUID=submitID)
assert submission_record.email == user_mail
task_id = submission_record.task.value # == mapping[task]
if submission_record.showOnLeaderboard == Show.YES:
FileModel.unset_show_attribute_by_submitID(submitUUID=submitID)
return {"message": "Remove from the leaderboard!", "submitID": submitID}, HTTPStatus.OK
else:
FileModel.set_show_attribute_by_submitID(submitUUID=submitID)
return {"message": "Shown on the leaderboard!", "submitID": submitID}, HTTPStatus.OK
except Exception as e:
print(e)
return {"message": "Internal Server Error!"}, HTTPStatus.INTERNAL_SERVER_ERROR
def get(cls, submitID):
'''Get submission by submit uuid'''
try:
user_mail = get_jwt_identity()
submission_record = FileModel.find_by_submitID(submitUUID=submitID)
assert submission_record.email == user_mail
download_path = submission_record.filePath
return send_file(download_path, as_attachment=True)
except Exception as e:
print(e)
return {"message": "Internal Server Error!"}, HTTPStatus.INTERNAL_SERVER_ERROR
def preprocess(event, context):
"""Sample pure Lambda function
event: https://docs.aws.amazon.com/apigateway/latest/developerguide/set-up-lambda-proxy-integrations.html#api-gateway-simple-proxy-for-lambda-input-format
context: https://docs.aws.amazon.com/lambda/latest/dg/python-context-object.html
Returns: https://docs.aws.amazon.com/apigateway/latest/developerguide/set-up-lambda-proxy-integrations.html
"""
# create file in DDB
file_id = utils.generate_id()
file_request = json.loads(event.get('body'))
FileModel.create({'id': file_id, 'name': file_request.get('name')})
LOGGER.debug(
'Files item created. service=ddb method=put_item id={}'.format(
file_id))
# generate signed URL for posting file
url = S3_CLIENT.generate_presigned_url(
ClientMethod='put_object',
Params={
'Bucket': runtime_context.BUCKET_NAME,
'Key': file_id
},
ExpiresIn=runtime_context.EXPIRATION)
LOGGER.debug(
'Presigned URL generated. service=s3 method=put_object id={}'.format(
file_id))
# send back the signed url
return {
"statusCode": 200,
"body": json.dumps({
'id': file_id,
'url': url
}),
# CORS header
"headers": {
"Access-Control-Allow-Origin": "*"
}
}
def get(cls):
'''Get user personal info (userName, public dialy count, weekly counts, daily left, weekly left, and hidden ones)'''
try:
user_mail = get_jwt_identity()
user = UserModel.find_by_email(email=user_mail)
daily_counts = FileModel.get_interval_upload_count_by_mail(
email=user_mail, AOEtime=get_AOE_today(to_str=False))
weekly_counts = FileModel.get_interval_upload_count_by_mail(
email=user_mail, AOEtime=get_AOE_week(to_str=False))
hidden_daily_counts = HiddenFileModel.get_interval_upload_count_by_mail(
email=user_mail, AOEtime=get_AOE_today(to_str=False))
hidden_weekly_counts = HiddenFileModel.get_interval_upload_count_by_mail(
email=user_mail, AOEtime=get_AOE_week(to_str=False))
daily_left = configs["DAILY_SUBMIT_LIMIT"] - daily_counts
weekly_left = configs["WEEKLY_SUBMIT_LIMIT"] - weekly_counts
hidden_daily_left = configs["HIDDEN_DAILY_SUBMIT_LIMIT"] - daily_counts
hidden_weekly_left = configs["HIDDEN_WEEKLY_SUBMIT_LIMIT"] - weekly_counts
return {"username": user.name, "daily_counts": daily_counts, "weekly_counts": weekly_counts, "daily_left": daily_left, "weekly_left": weekly_left,
"hidden_daily_counts":hidden_daily_counts, "hidden_weekly_counts":hidden_weekly_counts, "hidden_daily_left":hidden_daily_left, "hidden_weekly_left": hidden_weekly_left}, HTTPStatus.OK
except Exception as e:
print(e)
return {"message": "Internal Server Error!"}, HTTPStatus.INTERNAL_SERVER_ERROR
def post(cls):
data = _datafile_parser.parse_args()
current_user_id = get_jwt_identity()
user = UserModel.find_by_id(current_user_id)
links = list()
total_size = 0
# "files in data[] contains a list/array of files object/dict which contains file name and size
for file_obj in json.loads(data["files"]):
file = FileModel.find_by_name(name=file_obj["name"],
user_id=current_user_id)
if file:
return {"msg": "A file with that name already exists!"}, 403
if user.files_size + file_obj["size"] >= configs.allowed_size[
user.account_type]:
return {"msg": "Not enough space to upload!"}, 403
# links is a list of upload link, so it is appended with link for each file
links.append(
s3_storage.get_upload_url(file_obj["name"], current_user_id,
data["file_path"]))
file = FileModel(file_obj["name"], current_user_id,
data["file_path"], file_obj["size"])
file.save_to_db()
total_size += file_obj["size"]
# updating the total size of files the user currently has
user.increment_files_size(total_size)
return {"links": links}, 200
def delete(cls):
data = _datafile_parser.parse_args()
current_user_id = get_jwt_identity()
file = FileModel.find_by_name(name=data["file_name"],
user_id=current_user_id)
if not file:
return {"msg": "The file was not found."}, 404
file_path = file.path
s3_storage.delete_file(data["file_name"], current_user_id, file_path)
user = UserModel.find_by_id(current_user_id)
user.decrement_files_size(file.size)
file.delete_from_db()
return {"msg": "The file was deleted successfully."}, 200
def get(cls):
'''Get leaderboard data'''
try:
leaderboard_default_data = get_leaderboard_default()
leaderboard_user_data = FileModel.find_show_on_leaderboard()
submission_names = []
for user_data in leaderboard_user_data:
submission_names.append(
UserModel.find_by_email(email=user_data.email).name)
submission_info = submission_records_parser(
leaderboard_user_data, configs, mode="leaderboard")
for single_info, name in zip(submission_info, submission_names):
single_info.update({"name": name})
leaderboard_default_data += submission_info
return {"leaderboard": leaderboard_default_data}, HTTPStatus.OK
except Exception as e:
print(e)
return {"message": "Something went wrong!"}, HTTPStatus.INTERNAL_SERVER_ERROR
def post(cls):
data = _databooks_parser.parse_args()
current_user_id = get_jwt_identity()
databook = DatabookModel.find_by_name(data["databook_name"], current_user_id)
if not databook:
return {"msg": "No databook with that name found."}, 404
# below line gets the required data file from s3 if already not present locally
file = FileModel.find_by_name(databook.file_name, current_user_id)
if not file:
return {
"msg": "The data-file with the name - {}, for this databook is not found.".format(
databook.file_name)
}, 404
user = UserModel.find_by_id(current_user_id)
allowed_size = (configs.allowed_size[user.account_type]) // 30
s3_storage.get_file_to_local(databook.file_name, current_user_id, file.path, file.size, allowed_size)
# think of unpickling logic
if databook.json()["obj_created"]:
ret_data = databook_action_handler.unpickle_databook(user_id=current_user_id,
databook_name=data["databook_name"])
else:
ret_data = databook_action_handler.load_databook(user_id=current_user_id,
file_path="{}/data/{}/{}/{}".format(volume,
current_user_id,
file.path,
databook.file_name),
databook_name=data["databook_name"],
datastore_name=data["store_name"])
return ret_data, 200
def dao_add_info(self, loan_dir, subtransactions=False, nested=False):
"""
信息入库
:param nested:
:param subtransactions:
:param loan_dir: 资料目录
:return:
"""
with db.auto_commit_db(subtransactions,
nested,
error_call=FileUtil.del_dir,
dir_path=loan_dir) as s:
s.add(self)
# 写入磁盘
file_models = []
for file in self.file_data:
file_path = base64_to_file(file.file_base64, loan_dir,
file.file_name, file.file_format)
# 文件入库
file_model = FileModel()
file_model.dao_init_file(file_path, nested=True)
file_models.append(file_model)
# 图片处理明细信息
details = []
for file_model in file_models:
if file_model.file_format.strip().upper(
) == FileFormat.PDF.value:
# 文件为pdf, 分割 pdf, 更新资料信息
img_path = FileUtil.path_join(
loan_dir, current_app.config.get('DATA_DIR_IMG'),
file_model.id)
page_num, img_num, fail_num, detail_info = PDFUtil.pdf_to_pic(
file_model.file_path, img_path)
images = detail_info.get('images', None)
if img_num > 0 and is_not_empty(images):
for image in images:
if is_empty(image.get('error_msg', None)):
# 图片文件入库
img_model = FileModel()
img_model.dao_init_file(image.get(
'img_path', ''),
nested=True)
# 图片明细入库
img_detail = ImgDetailModel()
img_detail.dao_add(self.id,
file_model.id,
img_model.id,
nested=True)
self.page_num += page_num
self.success_num += img_num
self.fail_num += fail_num
details.append(
dict(id=file_model.id,
md5=file_model.md5_id,
page_num=page_num,
success_num=img_num,
fail_num=fail_num))
else:
# 文件备份， 迁移处理文件
new_img_path = FileUtil.copy_file(
file_model.file_path,
FileUtil.path_join(
loan_dir, current_app.config.get('DATA_DIR_IMG')))
# 图片文件入库
img_model = FileModel()
img_model.dao_init_file(new_img_path, nested=True)
# 文件为图片, 图片明细入库
img_detail = ImgDetailModel()
img_detail.dao_add(self.id,
file_model.id,
img_model.id,
nested=True)
self.page_num += 1
self.success_num += 1
self.fail_num += 0
details.append(
dict(id=file_model.id,
md5=file_model.md5_id,
page_num=1,
success_num=1,
fail_num=0))
return dict(id=self.id, details=details)