def get_cwd(auth_token, app):
with app.app_context():
resp = decode_auth_token(auth_token)
if isinstance(resp, str):
# Auth token invalid
print(resp)
# Is a user_id
row = MLStatus.query.filter_by(user_id=resp).first()
if not row:
# Cannot find use in the status DB. No bueno
print('Cannot find user in the status DB')
# Return the location of the original file
return row.working_directory
def get_values(auth_token, app):
with app.app_context():
resp = decode_auth_token(auth_token)
if isinstance(resp, str):
# Auth token invalid
print(resp, flush=True)
# Is a user_id
row = MLStatus.query.filter_by(user_id=resp).first()
if not row:
# Cannot find use in the status DB. No bueno
print('Cannot find user in the status DB')
# Return the selected files from the user
return row.selected_files
def insert_cwd(app, auth_token, cwd):
with app.app_context():
resp = decode_auth_token(auth_token)
if isinstance(resp, str):
# auth_token isn't valid anymore - that's not good
print(resp)
# If its not a string, its a user_id
status = MLStatus.query.filter_by(user_id=resp).first()
if not status:
# Can't find user in status DB - that's not good
print("Cannot find user in status DB")
status.working_directory = cwd
db.session.commit()
def set_status_error(app, auth_token, error):
with app.app_context():
resp = decode_auth_token(auth_token)
if isinstance(resp, str):
# auth_token isn't valid anymore - that's not good
print(resp)
status = MLStatus.query.filter_by(user_id=resp).first()
if not status:
# Can't find user in status DB - that's not good
print("Cannot find user in status DB")
status.status = "Failed to process files."
status.error_msg = error
db.session.commit()
return
def get_authentication(auth_header):
responseObject = {
'status': 'fail',
'message': '',
'status_code': 401,
'user_id': -1
}
if auth_header:
try:
auth_token = auth_header.split(" ")[1]
except IndexError:
responseObject['message'] = 'Bearer token malformed.'
return responseObject
else:
auth_token = ''
if auth_token:
resp = decode_auth_token(auth_token)
if not isinstance(resp, str):
user_id = resp
responseObject['user_id'] = user_id
else:
responseObject = {
'status': 'fail',
'message': "Issue with auth_token: " + resp
}
return responseObject
else:
responseObject = {
'status': 'fail',
'message': 'Provide a valid auth token.'
}
return responseObject
if isinstance(user_id, int):
user_id = str(user_id)
if user_id == None or user_id == '':
responseObject['message'] = "No user_id found"
responseObject['status_code'] = 400
return responseObject
responseObject['user_id'] = user_id
responseObject['status_code'] = 200
return responseObject
def update_status(auth_token, app, j_frame):
with app.app_context():
resp = decode_auth_token(auth_token)
if isinstance(resp, str):
# Auth token invalid
print(resp)
# Is a user_id
row = MLStatus.query.filter_by(user_id=resp).first()
if not row:
# Cannot find use in the status DB. No bueno
print('Cannot find user in the status DB')
# update the user status
row.status = 'Completed.'
# Update the status field
row.classified_json = j_frame
# send to db
db.session.commit()
def get(self):
auth_header = request.headers.get('Authorization')
if auth_header:
try:
auth_token = auth_header.split(" ")[1]
except IndexError:
responseObject = {
'status': 'fail',
'message': 'Bearer token malformed.'
}
return make_response(jsonify(responseObject)), 401
else:
auth_token = ''
if auth_token:
resp = decode_auth_token(auth_token)
if not isinstance(resp, str):
files = S3Files.query.filter_by(user_id=resp).all()
classified_names = list()
classified_links = list()
for row in files:
if row.classified_filename and row.classified_url:
classified_names.append(row.classified_filename)
classified_links.append(row.classified_url)
file_json = [{
"classified_names": n,
"classified_links": l
} for n, l in zip(classified_names, classified_links)]
responseObject = {
'status': 'success',
'message':
'Listing ' + str(len(file_json)) + ' classified files.',
'data': file_json
}
return make_response(jsonify(responseObject)), 200
responseObject = {'status': 'fail', 'message': resp}
return make_response(jsonify(responseObject)), 401
else:
responseObject = {
'status': 'fail',
'message': 'Provide a valid auth token.'
}
return make_response(jsonify(responseObject)), 401
def post(self):
auth_header = request.headers.get('Authorization')
responseObject = {'status': 'fail', 'message': ''}
if auth_header:
try:
auth_token = auth_header.split(" ")[1]
except IndexError:
responseObject['message'] = 'Bearer token malformed.'
return make_response(jsonify(responseObject)), 401
else:
auth_token = ''
if auth_token:
resp = decode_auth_token(auth_token)
if not isinstance(resp, str):
user_id = resp
responseObject['user_id'] = user_id
else:
responseObject = {
'status': 'fail',
'message': "Issue with auth_token: " + resp
}
return make_response(jsonify(responseObject)), 401
else:
responseObject = {
'status': 'fail',
'message': 'Provide a valid auth token.'
}
return make_response(jsonify(responseObject)), 401
if isinstance(user_id, int):
user_id = str(user_id)
if user_id == None or user_id == '':
responseObject['message'] = "No user_id found"
return jsonify(responseObject), 400
# FIXME: Temp keys for dev
ACCESS_ID = 'AKIAJCYLNW4GRKVGAIGA'
ACCESS_KEY = 'EQWk5FAv8pTLmEoL81VKYWnqJAcaFEJtp1bfOsTI'
client = boto3.client('s3',
aws_access_key_id=ACCESS_ID,
aws_secret_access_key=ACCESS_KEY)
if (request.values.get('bucket_name') == 'uploads'):
bucket_name = os.getenv('S3_UPLOAD')
file_type = "uploads"
elif (request.values.get('bucket_name') == 'classified'):
bucket_name = os.getenv('S3_CLASSIFIED')
file_type = "classified"
else:
responseObject['message'] = "Incorrect bucket name passed"
return make_response(jsonify(responseObject)), 401
responseObject['bucket'] = bucket_name
responseObject['user_id'] = user_id
for files in request.files:
cur_file = request.files[files]
cur_name = request.files[files].filename
responseObject[files] = cur_name
if (cur_name == ''):
print('no file selected')
return jsonify(responseObject), 400
key_name = cur_name + '.' + user_id + '.' + file_type # this is the name of the file that will be stored in the bucket
client.upload_fileobj(
Bucket=bucket_name, # name of the bucket
Fileobj=
cur_file, # this is the name of the file that is being uploaded
Key=key_name)
responseObject[files] = key_name
# TODO: Check to see if user and file exist in DB
# TODO: if yes, update with URL. else create new item
responseObject['status'] = 'success'
responseObject['message'] = 'Everything is okay :)'
response_object = {'status': 'success', 'data': responseObject}
return make_response(jsonify(response_object)), 200
def get(self):
auth_header = request.headers.get('Authorization')
responseObject = {'status': 'fail', 'message': ''}
if auth_header:
try:
auth_token = auth_header.split(" ")[1]
except IndexError:
responseObject['message'] = 'Bearer token malformed.'
return make_response(jsonify(responseObject)), 401
else:
auth_token = ''
if auth_token:
resp = decode_auth_token(auth_token)
if not isinstance(resp, str):
user_id = resp
responseObject['user_id'] = user_id
else:
responseObject = {
'status': 'fail',
'message': "Issue with auth_token: " + resp
}
return make_response(jsonify(responseObject)), 401
else:
responseObject = {
'status': 'fail',
'message': 'Provide a valid auth token.'
}
return make_response(jsonify(responseObject)), 401
if isinstance(user_id, int):
user_id = str(user_id)
if user_id == None or user_id == '':
responseObject['message'] = "No user_id found"
return jsonify(responseObject), 400
# Get Key_name
if 'key_name' not in request.values:
responseObject['message'] = "No key name given"
return jsonify(responseObject), 400
elif request.values.get('key_name') == '':
responseObject['message'] = "No key name given"
return jsonify(responseObject), 400
else:
key_name = request.values.get('key_name')
# Get file_name
if 'file_name' not in request.values:
responseObject['message'] = "No file name to save as"
return jsonify(responseObject), 400
elif request.values.get('file_name') == '':
responseObject['message'] = "No file name to save as"
return jsonify(responseObject), 400
else:
file_name = request.values.get('file_name')
# Get bucket_name
if (request.values.get('bucket_name') == 'uploads'):
bucket_name = os.getenv('S3_UPLOAD')
file_type = "uploads"
elif (request.values.get('bucket_name') == 'classified'):
bucket_name = os.getenv('S3_CLASSIFIED')
file_type = "classified"
else:
responseObject['message'] = "passed: " + request.values.get(
'bucket_name')
return make_response(jsonify(responseObject)), 401
responseObject['key_name'] = key_name
responseObject['file_name'] = file_name
responseObject['bucket'] = bucket_name
responseObject['user_id'] = user_id
# FIXME: Temp keys for dev
ACCESS_ID = 'AKIAJCYLNW4GRKVGAIGA'
ACCESS_KEY = 'EQWk5FAv8pTLmEoL81VKYWnqJAcaFEJtp1bfOsTI'
client = boto3.client('s3',
aws_access_key_id=ACCESS_ID,
aws_secret_access_key=ACCESS_KEY)
with open(file_name, 'wb') as data:
client.download_fileobj(
Bucket=bucket_name, # name of the bucket
Fileobj=
data, # this is the name you want the downloaded file saved as
Key=key_name)
responseObject['data'] = data
responseObject['new_file'] = file_name
responseObject['message'] = 'Everything is okay :)'
responseObject['status'] = 'success'
response_object = {'status': 'success', 'data': responseObject}
return make_response(jsonify(response_object)), 200
def test_decode_auth_token(self):
"""Ensure auth tokens are decoded correctly."""
auth_token = encode_auth_token(1)
self.assertTrue(isinstance(auth_token, bytes))
self.assertTrue(decode_auth_token(auth_token.decode("utf-8")) == 1)
def post(self):
# get the auth token
auth_header = request.headers.get('Authorization')
if auth_header:
try:
auth_token = auth_header.split(" ")[1]
except IndexError:
responseObject = {
'status': 'fail',
'message': 'Bearer token malformed.'
}
return make_response(jsonify(responseObject)), 401
else:
auth_token = ''
if auth_token:
resp = decode_auth_token(auth_token)
# Check if decode_auth_token returned a string (which means it failed)
# If it succeeded, resp now holds the user_id value
if not isinstance(resp, str):
# Get the filenames from the post
post_data = request.get_json()
if post_data:
files = post_data.get('files')
if len(files) <= 0:
responseObject = {
'status': 'fail',
'message': 'No files provided.'
}
return make_response(jsonify(responseObject)), 400
# Get the current user status
status = MLStatus.query.filter_by(user_id=resp).first()
if not status:
# User not in status table - add them now
try:
user = MLStatus(user_id=resp,
status="Waiting for files.")
# insert the user
db.session.add(user)
db.session.commit()
# update status object to reference newly created user
status = MLStatus.query.filter_by(
user_id=resp).first()
if not status:
responseObject = {
'status':
'fail',
'message':
'User could not be added to status database.'
}
return make_response(
jsonify(responseObject)), 401
except Exception as e:
responseObject = {
'status': 'fail',
'message':
'Some error occurred. Please try again.'
}
return make_response(jsonify(responseObject)), 401
# Check the user's status - make sure its "Waiting for files."
if status.status == 'Waiting for files.' or status.status == 'Completed.' or status.status == "Failed to process files.":
status.selected_files = files
status.status = 'Processing.'
db.session.commit()
# TODO: @D3lta - Trigger your method here
print("Creating Parse Thread\n")
parseThread = Thread(
target=extract_columns,
args=(current_app._get_current_object(),
auth_token, files))
parseThread.start()
parseThread.join()
else:
# User is not ready for files - still processing?
responseObject = {
'status': 'fail',
'message':
'Already processing files for this user.'
}
return make_response(jsonify(responseObject)), 401
responseObject = {
'status':
'success',
'message':
'Successfully started ML on ' + str(len(files)) +
' files.'
}
return make_response(jsonify(responseObject)), 200
else:
responseObject = {
'status': 'fail',
'message': 'No files provided.'
}
return make_response(jsonify(responseObject)), 400
else:
responseObject = {'status': 'fail', 'message': resp}
return make_response(jsonify(responseObject)), 401
else:
responseObject = {
'status': 'fail',
'message': 'Provide a valid auth token.'
}
return make_response(jsonify(responseObject)), 401
def post(self):
# get the auth token
auth_header = request.headers.get('Authorization')
if auth_header:
try:
auth_token = auth_header.split(" ")[1]
except IndexError:
responseObject = {
'status': 'fail',
'message': 'Bearer token malformed.'
}
return make_response(jsonify(responseObject)), 401
else:
auth_token = ''
if auth_token:
resp = decode_auth_token(auth_token)
# Check if decode_auth_token returned a string (which means it failed)
# If it succeeded, resp now holds the user_id value
if not isinstance(resp, str):
# Get the filename from the post
post_data = request.get_json()
if post_data:
file_name = post_data.get('file_name')
if file_name:
download_url = ""
# check if config is TESTING
if current_app.config.get('TESTING'):
if file_name == 'bad_download_code':
# Simulate a bad download code
download_url = "https://raw.githubusercontent.com/geraldomacias/MarkLogic/non_existant_branch/"
else:
# Set download_url to a test json file on repo
download_url = "https://raw.githubusercontent.com/geraldomacias/MarkLogic/spencer-dev/services/machine_learning/project/tests/test.json"
else:
# Get download_url from file_system endpoint
file_system_url = "http://file_system:5000/s3/downloadClassified"
file_system_headers = {
"Authorization": 'Bearer ' + auth_token
}
file_system_params = {"classified": file_name}
url_response = requests.get(
url=file_system_url,
headers=file_system_headers,
params=file_system_params)
if url_response.status_code == 200:
url_key = file_name + '_download_url'
download_url = (url_response.json(
))['data']['classified_response'][url_key]
else:
responseObject = {
'status': 'fail',
'message':
'Error connecting to file_system.'
}
return make_response(
jsonify(responseObject)), 500
download_response = requests.get(download_url)
if download_response.status_code == 200:
responseObject = {
'status': 'success',
'message': 'Returning classified information.',
'data': download_response.json()
}
return make_response(jsonify(responseObject)), 200
else:
responseObject = {
'status':
'fail',
'message':
'Bad response from download url. Please try downloading again, or classify your csv again.'
}
return make_response(jsonify(responseObject)), 404
else:
responseObject = {
'status': 'fail',
'message': 'File name not provided.'
}
return make_response(jsonify(responseObject)), 400
else:
responseObject = {
'status': 'fail',
'message': 'File name not provided.'
}
return make_response(jsonify(responseObject)), 400
else:
responseObject = {'status': 'fail', 'message': resp}
return make_response(jsonify(responseObject)), 401
else:
responseObject = {
'status': 'fail',
'message': 'Provide a valid auth token.'
}
return make_response(jsonify(responseObject)), 401
def get(self):
# get the auth token
auth_header = request.headers.get('Authorization')
if auth_header:
try:
auth_token = auth_header.split(" ")[1]
except IndexError:
responseObject = {
'status': 'fail',
'message': 'Bearer token malformed.'
}
return make_response(jsonify(responseObject)), 401
else:
auth_token = ''
if auth_token:
resp = decode_auth_token(auth_token)
# Check if decode_auth_token returned a string (which means it failed)
# If it succeeded, resp now holds the user_id value
if not isinstance(resp, str):
# Get a status row for the user
status = MLStatus.query.filter_by(user_id=resp).first()
if not status:
# User not in status table - error
responseObject = {
'status': 'fail',
'message': 'User has not classified any data.'
}
return make_response(jsonify(responseObject)), 404
else:
if status.status == "Completed.":
# make sure row has JSON value
resp_json = status.classified_json
if resp_json:
responseObject = {
'status': 'success',
'message': 'Returning classified information.',
'data': resp_json
}
return make_response(jsonify(responseObject)), 200
else:
# No json object in db for user
responseObject = {
'status':
'fail',
'message':
'No classified data found for given user.'
}
return make_response(jsonify(responseObject)), 404
else:
# ML has not yet completed for user
responseObject = {
'status':
'fail',
'message':
'Classification not yet completed for given user. Current status: '
+ status.status
}
return make_response(jsonify(responseObject)), 401
else:
responseObject = {'status': 'fail', 'message': resp}
return make_response(jsonify(responseObject)), 401
else:
responseObject = {
'status': 'fail',
'message': 'Provide a valid auth token.'
}
return make_response(jsonify(responseObject)), 401
def get(self):
# get the auth token
auth_header = request.headers.get('Authorization')
if auth_header:
try:
auth_token = auth_header.split(" ")[1]
except IndexError:
responseObject = {
'status': 'fail',
'message': 'Bearer token malformed.'
}
return make_response(jsonify(responseObject)), 401
else:
auth_token = ''
if auth_token:
resp = decode_auth_token(auth_token)
# Check if decode_auth_token returnned a string (which means it failed)
# If it succeeded, resp now holds the user_id value
if not isinstance(resp, str):
# Check the ml status for the user
status = MLStatus.query.filter_by(user_id=resp).first()
if not status:
# User not in status table - add them now
try:
user = MLStatus(user_id=resp,
status="Waiting for files.")
# insert the user
db.session.add(user)
db.session.commit()
# update status object to reference newly created user
status = MLStatus.query.filter_by(user_id=resp).first()
if not status:
responseObject = {
'status':
'fail',
'message':
'User could not be added to status database.'
}
return make_response(jsonify(responseObject)), 401
except Exception as e:
responseObject = {
'status': 'fail',
'message': 'Some error occurred. Please try again.'
}
return make_response(jsonify(responseObject)), 401
if status == 'Failed to process files.':
responseObject = {
'status': 'success',
'message': status.status,
'error_msg': status.error_msg
}
return make_response(jsonify(responseObject)), 200
responseObject = {
'status': 'success',
'message': status.status
}
return make_response(jsonify(responseObject)), 200
else:
responseObject = {'status': 'fail', 'message': resp}
return make_response(jsonify(responseObject)), 401
else:
responseObject = {
'status': 'fail',
'message': 'Provide a valid auth token.'
}
return make_response(jsonify(responseObject)), 401
