def get(self, dataset_name):
filename = request.args.get('filename')
save_dir = './data'
s3_file_path = f'{dataset_name}/{filename}'
local_file_path = f'{save_dir}/{filename}'
try:
shutil.rmtree(save_dir)
except:
print("Removing of the directory %s failed" % save_dir)
try:
os.mkdir(save_dir)
except OSError:
print("Creation of the directory %s failed" % save_dir)
aws_object = AWS_S3()
status = aws_object.download_file(local_file_path=local_file_path,
s3_file_path=s3_file_path,
bucket_name=get_etcd_config(
'/data-storage/bucket_name',
'BUCKET_NAME'))
if status == True:
try:
return send_from_directory(os.getcwd() + '/data/',
filename=filename,
as_attachment=True)
except FileNotFoundError:
return {'message': 'File Not Found'}, 400
else:
return {'message': status}, 400
def save_file_to_s3(file, filename, dataset_name):
files = {'file': (filename, file.read(), dataset_name)}
data_storage_url = get_etcd_config('/data-upload/storage-url',
'DATA_STORAGE_URL') + '/v1/files'
try:
r_data_storage = requests.post(data_storage_url, files=files)
return (r_data_storage.text, r_data_storage.status_code)
except:
return ('data_storage_service not aviable', 400)
def demo_healthcheck():
health_status = get_etcd_config('/data-upload/health-demo-status',
'HEALTH_DEMO_STATUS')
if health_status == 'True':
health_status = True
message = "data-upload ok"
else:
health_status = False
message = "data-upload error"
return health_status, message
def delete(self, dataset_name):
filename = request.args.get('filename')
s3_file_path = f'{dataset_name}/{filename}'
aws_object = AWS_S3()
status = aws_object.delete_file(s3_file_path=s3_file_path,
bucket_name=get_etcd_config(
'/data-storage/bucket_name',
'BUCKET_NAME'))
if status == True:
return {'message': 'File delete succesfully.'}, 200
else:
return {'message': status}, 400
def update_data_catalog_info(dataset_name, file_name, file_size, num_lines):
# catalog microservice
catalog_post_data = {
"dataset_lenght": num_lines,
"dataset_size": file_size,
"dataset_name": dataset_name,
"file_name": file_name,
}
catalog_adderss = get_etcd_config('/data-upload/catalog-url',
'DATA_CATALOG_URL')
catalog_url = catalog_adderss + '/v1/datasets'
headers = {'Content-type': 'application/json'}
try:
r_catalog = requests.post(catalog_url,
data=json.dumps(catalog_post_data),
headers=headers)
return r_catalog.status_code
except:
return 400
def post(self):
file = request.files['file']
filename = secure_filename(file.filename)
dataset_name = file.content_type
s3_file_path = f'{dataset_name}/{filename}'
file.save(f'./{filename}')
aws_object = AWS_S3()
status = aws_object.upload_file(local_file_path=f'./{filename}',
s3_file_path=s3_file_path,
bucket_name=get_etcd_config(
'/data-storage/bucket_name',
'BUCKET_NAME'))
try:
os.remove(f'./{filename}')
except:
print('Error deleting file.') # add to log
if status == True:
return {'message': 'File save succesfully to S3'}, 201
else:
return {'message': status}, 400
def post(self):
info_log(app, 'UploadFile post', 'ENTRY ', 'method call')
# check if the post request has the file part
if 'file' not in request.files:
error_log(app, 'UploadFile post', 'ERROR ',
'No file part in the request')
return {'message': 'No file part in the request'}, 400
file = request.files['file']
# Ensuring the file has a name
if file.filename == '':
error_log(app, 'UploadFile post', 'ERROR ',
'No file selected for uploading')
return {'message': 'No file selected for uploading'}, 400
# Ensuring the file type is allowed
# Ensuring the filename is allowed
# Ensuring the filesize is allowed
if allowed_file(file):
filename = secure_filename(file.filename)
dataset_name = file.content_type
# preveri ali ima dataset name presledke
if " " in dataset_name:
error_log(app, 'UploadFile post', 'ERROR ',
'Dataset name has spaces.')
return {'message': 'Error: Dataset name has spaces.'}, 400
# preveri ali datasename že obstaja
try:
catalog_adderss = get_etcd_config('/data-upload/catalog-url',
'DATA_CATALOG_URL')
catalog_url = f'{catalog_adderss}/v1/datasets/{dataset_name}'
dataset_status = requests.get(catalog_url)
except:
error_log(
app, 'UploadFile post', 'ERROR ',
'Error saving fail. (error connecting to data-catalog)')
return {
'message':
'Error saving fail. (error connecting to data-catalog)'
}, 400
if dataset_status.status_code == 404:
s3_message, s3_status_code = save_file_to_s3(
file, filename, dataset_name)
if s3_status_code == 201:
file.seek(0) # postavimo file nazaj na začetek
file_path = os.path.join(app.config['UPLOAD_FOLDER'],
filename)
file.save(file_path)
file_size = os.path.getsize(file_path) / (1024 * 1024)
num_lines = sum(1 for line in open(file_path))
catalog_update_status = update_data_catalog_info(
dataset_name, filename, file_size, num_lines)
try:
os.remove(file_path)
except:
error_log(app, 'UploadFile post', 'ERROR ',
'Error deleting file.')
if catalog_update_status == 201:
info_log(app, 'UploadFile post', 'EXIT ',
'method call')
return {'message': 'File successfully uploaded'}, 201
else:
error_log(app, 'UploadFile post', 'ERROR ',
'Error adding file info to database.')
return {
'message': 'Error adding file info to database.'
}, 400
else:
error_log(app, 'UploadFile post', 'ERROR ', s3_message)
return {'message': s3_message}, 400
else:
error_log(app, 'UploadFile post', 'ERROR ',
'Dataset name already used.')
return {'message': 'Dataset name already used.'}, 400
else:
error_log(app, 'UploadFile post', 'ERROR ',
'Allowed file type is .csv')
return {'message': 'Allowed file type is .csv'}, 400
def get(self):
message = get_etcd_config('/data-catalog/test-message', 'MESSAGE')
return message