def get_cwd(auth_token, app): with app.app_context(): resp = decode_auth_token(auth_token) if isinstance(resp, str): # Auth token invalid print(resp) # Is a user_id row = MLStatus.query.filter_by(user_id=resp).first() if not row: # Cannot find use in the status DB. No bueno print('Cannot find user in the status DB') # Return the location of the original file return row.working_directory
def get_values(auth_token, app): with app.app_context(): resp = decode_auth_token(auth_token) if isinstance(resp, str): # Auth token invalid print(resp, flush=True) # Is a user_id row = MLStatus.query.filter_by(user_id=resp).first() if not row: # Cannot find use in the status DB. No bueno print('Cannot find user in the status DB') # Return the selected files from the user return row.selected_files
def insert_cwd(app, auth_token, cwd): with app.app_context(): resp = decode_auth_token(auth_token) if isinstance(resp, str): # auth_token isn't valid anymore - that's not good print(resp) # If its not a string, its a user_id status = MLStatus.query.filter_by(user_id=resp).first() if not status: # Can't find user in status DB - that's not good print("Cannot find user in status DB") status.working_directory = cwd db.session.commit()
def set_status_error(app, auth_token, error): with app.app_context(): resp = decode_auth_token(auth_token) if isinstance(resp, str): # auth_token isn't valid anymore - that's not good print(resp) status = MLStatus.query.filter_by(user_id=resp).first() if not status: # Can't find user in status DB - that's not good print("Cannot find user in status DB") status.status = "Failed to process files." status.error_msg = error db.session.commit() return
def get_authentication(auth_header): responseObject = { 'status': 'fail', 'message': '', 'status_code': 401, 'user_id': -1 } if auth_header: try: auth_token = auth_header.split(" ")[1] except IndexError: responseObject['message'] = 'Bearer token malformed.' return responseObject else: auth_token = '' if auth_token: resp = decode_auth_token(auth_token) if not isinstance(resp, str): user_id = resp responseObject['user_id'] = user_id else: responseObject = { 'status': 'fail', 'message': "Issue with auth_token: " + resp } return responseObject else: responseObject = { 'status': 'fail', 'message': 'Provide a valid auth token.' } return responseObject if isinstance(user_id, int): user_id = str(user_id) if user_id == None or user_id == '': responseObject['message'] = "No user_id found" responseObject['status_code'] = 400 return responseObject responseObject['user_id'] = user_id responseObject['status_code'] = 200 return responseObject
def update_status(auth_token, app, j_frame): with app.app_context(): resp = decode_auth_token(auth_token) if isinstance(resp, str): # Auth token invalid print(resp) # Is a user_id row = MLStatus.query.filter_by(user_id=resp).first() if not row: # Cannot find use in the status DB. No bueno print('Cannot find user in the status DB') # update the user status row.status = 'Completed.' # Update the status field row.classified_json = j_frame # send to db db.session.commit()
def get(self): auth_header = request.headers.get('Authorization') if auth_header: try: auth_token = auth_header.split(" ")[1] except IndexError: responseObject = { 'status': 'fail', 'message': 'Bearer token malformed.' } return make_response(jsonify(responseObject)), 401 else: auth_token = '' if auth_token: resp = decode_auth_token(auth_token) if not isinstance(resp, str): files = S3Files.query.filter_by(user_id=resp).all() classified_names = list() classified_links = list() for row in files: if row.classified_filename and row.classified_url: classified_names.append(row.classified_filename) classified_links.append(row.classified_url) file_json = [{ "classified_names": n, "classified_links": l } for n, l in zip(classified_names, classified_links)] responseObject = { 'status': 'success', 'message': 'Listing ' + str(len(file_json)) + ' classified files.', 'data': file_json } return make_response(jsonify(responseObject)), 200 responseObject = {'status': 'fail', 'message': resp} return make_response(jsonify(responseObject)), 401 else: responseObject = { 'status': 'fail', 'message': 'Provide a valid auth token.' } return make_response(jsonify(responseObject)), 401
def post(self): auth_header = request.headers.get('Authorization') responseObject = {'status': 'fail', 'message': ''} if auth_header: try: auth_token = auth_header.split(" ")[1] except IndexError: responseObject['message'] = 'Bearer token malformed.' return make_response(jsonify(responseObject)), 401 else: auth_token = '' if auth_token: resp = decode_auth_token(auth_token) if not isinstance(resp, str): user_id = resp responseObject['user_id'] = user_id else: responseObject = { 'status': 'fail', 'message': "Issue with auth_token: " + resp } return make_response(jsonify(responseObject)), 401 else: responseObject = { 'status': 'fail', 'message': 'Provide a valid auth token.' } return make_response(jsonify(responseObject)), 401 if isinstance(user_id, int): user_id = str(user_id) if user_id == None or user_id == '': responseObject['message'] = "No user_id found" return jsonify(responseObject), 400 # FIXME: Temp keys for dev ACCESS_ID = 'AKIAJCYLNW4GRKVGAIGA' ACCESS_KEY = 'EQWk5FAv8pTLmEoL81VKYWnqJAcaFEJtp1bfOsTI' client = boto3.client('s3', aws_access_key_id=ACCESS_ID, aws_secret_access_key=ACCESS_KEY) if (request.values.get('bucket_name') == 'uploads'): bucket_name = os.getenv('S3_UPLOAD') file_type = "uploads" elif (request.values.get('bucket_name') == 'classified'): bucket_name = os.getenv('S3_CLASSIFIED') file_type = "classified" else: responseObject['message'] = "Incorrect bucket name passed" return make_response(jsonify(responseObject)), 401 responseObject['bucket'] = bucket_name responseObject['user_id'] = user_id for files in request.files: cur_file = request.files[files] cur_name = request.files[files].filename responseObject[files] = cur_name if (cur_name == ''): print('no file selected') return jsonify(responseObject), 400 key_name = cur_name + '.' + user_id + '.' + file_type # this is the name of the file that will be stored in the bucket client.upload_fileobj( Bucket=bucket_name, # name of the bucket Fileobj= cur_file, # this is the name of the file that is being uploaded Key=key_name) responseObject[files] = key_name # TODO: Check to see if user and file exist in DB # TODO: if yes, update with URL. else create new item responseObject['status'] = 'success' responseObject['message'] = 'Everything is okay :)' response_object = {'status': 'success', 'data': responseObject} return make_response(jsonify(response_object)), 200
def get(self): auth_header = request.headers.get('Authorization') responseObject = {'status': 'fail', 'message': ''} if auth_header: try: auth_token = auth_header.split(" ")[1] except IndexError: responseObject['message'] = 'Bearer token malformed.' return make_response(jsonify(responseObject)), 401 else: auth_token = '' if auth_token: resp = decode_auth_token(auth_token) if not isinstance(resp, str): user_id = resp responseObject['user_id'] = user_id else: responseObject = { 'status': 'fail', 'message': "Issue with auth_token: " + resp } return make_response(jsonify(responseObject)), 401 else: responseObject = { 'status': 'fail', 'message': 'Provide a valid auth token.' } return make_response(jsonify(responseObject)), 401 if isinstance(user_id, int): user_id = str(user_id) if user_id == None or user_id == '': responseObject['message'] = "No user_id found" return jsonify(responseObject), 400 # Get Key_name if 'key_name' not in request.values: responseObject['message'] = "No key name given" return jsonify(responseObject), 400 elif request.values.get('key_name') == '': responseObject['message'] = "No key name given" return jsonify(responseObject), 400 else: key_name = request.values.get('key_name') # Get file_name if 'file_name' not in request.values: responseObject['message'] = "No file name to save as" return jsonify(responseObject), 400 elif request.values.get('file_name') == '': responseObject['message'] = "No file name to save as" return jsonify(responseObject), 400 else: file_name = request.values.get('file_name') # Get bucket_name if (request.values.get('bucket_name') == 'uploads'): bucket_name = os.getenv('S3_UPLOAD') file_type = "uploads" elif (request.values.get('bucket_name') == 'classified'): bucket_name = os.getenv('S3_CLASSIFIED') file_type = "classified" else: responseObject['message'] = "passed: " + request.values.get( 'bucket_name') return make_response(jsonify(responseObject)), 401 responseObject['key_name'] = key_name responseObject['file_name'] = file_name responseObject['bucket'] = bucket_name responseObject['user_id'] = user_id # FIXME: Temp keys for dev ACCESS_ID = 'AKIAJCYLNW4GRKVGAIGA' ACCESS_KEY = 'EQWk5FAv8pTLmEoL81VKYWnqJAcaFEJtp1bfOsTI' client = boto3.client('s3', aws_access_key_id=ACCESS_ID, aws_secret_access_key=ACCESS_KEY) with open(file_name, 'wb') as data: client.download_fileobj( Bucket=bucket_name, # name of the bucket Fileobj= data, # this is the name you want the downloaded file saved as Key=key_name) responseObject['data'] = data responseObject['new_file'] = file_name responseObject['message'] = 'Everything is okay :)' responseObject['status'] = 'success' response_object = {'status': 'success', 'data': responseObject} return make_response(jsonify(response_object)), 200
def test_decode_auth_token(self): """Ensure auth tokens are decoded correctly.""" auth_token = encode_auth_token(1) self.assertTrue(isinstance(auth_token, bytes)) self.assertTrue(decode_auth_token(auth_token.decode("utf-8")) == 1)
def post(self): # get the auth token auth_header = request.headers.get('Authorization') if auth_header: try: auth_token = auth_header.split(" ")[1] except IndexError: responseObject = { 'status': 'fail', 'message': 'Bearer token malformed.' } return make_response(jsonify(responseObject)), 401 else: auth_token = '' if auth_token: resp = decode_auth_token(auth_token) # Check if decode_auth_token returned a string (which means it failed) # If it succeeded, resp now holds the user_id value if not isinstance(resp, str): # Get the filenames from the post post_data = request.get_json() if post_data: files = post_data.get('files') if len(files) <= 0: responseObject = { 'status': 'fail', 'message': 'No files provided.' } return make_response(jsonify(responseObject)), 400 # Get the current user status status = MLStatus.query.filter_by(user_id=resp).first() if not status: # User not in status table - add them now try: user = MLStatus(user_id=resp, status="Waiting for files.") # insert the user db.session.add(user) db.session.commit() # update status object to reference newly created user status = MLStatus.query.filter_by( user_id=resp).first() if not status: responseObject = { 'status': 'fail', 'message': 'User could not be added to status database.' } return make_response( jsonify(responseObject)), 401 except Exception as e: responseObject = { 'status': 'fail', 'message': 'Some error occurred. Please try again.' } return make_response(jsonify(responseObject)), 401 # Check the user's status - make sure its "Waiting for files." if status.status == 'Waiting for files.' or status.status == 'Completed.' or status.status == "Failed to process files.": status.selected_files = files status.status = 'Processing.' db.session.commit() # TODO: @D3lta - Trigger your method here print("Creating Parse Thread\n") parseThread = Thread( target=extract_columns, args=(current_app._get_current_object(), auth_token, files)) parseThread.start() parseThread.join() else: # User is not ready for files - still processing? responseObject = { 'status': 'fail', 'message': 'Already processing files for this user.' } return make_response(jsonify(responseObject)), 401 responseObject = { 'status': 'success', 'message': 'Successfully started ML on ' + str(len(files)) + ' files.' } return make_response(jsonify(responseObject)), 200 else: responseObject = { 'status': 'fail', 'message': 'No files provided.' } return make_response(jsonify(responseObject)), 400 else: responseObject = {'status': 'fail', 'message': resp} return make_response(jsonify(responseObject)), 401 else: responseObject = { 'status': 'fail', 'message': 'Provide a valid auth token.' } return make_response(jsonify(responseObject)), 401
def post(self): # get the auth token auth_header = request.headers.get('Authorization') if auth_header: try: auth_token = auth_header.split(" ")[1] except IndexError: responseObject = { 'status': 'fail', 'message': 'Bearer token malformed.' } return make_response(jsonify(responseObject)), 401 else: auth_token = '' if auth_token: resp = decode_auth_token(auth_token) # Check if decode_auth_token returned a string (which means it failed) # If it succeeded, resp now holds the user_id value if not isinstance(resp, str): # Get the filename from the post post_data = request.get_json() if post_data: file_name = post_data.get('file_name') if file_name: download_url = "" # check if config is TESTING if current_app.config.get('TESTING'): if file_name == 'bad_download_code': # Simulate a bad download code download_url = "https://raw.githubusercontent.com/geraldomacias/MarkLogic/non_existant_branch/" else: # Set download_url to a test json file on repo download_url = "https://raw.githubusercontent.com/geraldomacias/MarkLogic/spencer-dev/services/machine_learning/project/tests/test.json" else: # Get download_url from file_system endpoint file_system_url = "http://file_system:5000/s3/downloadClassified" file_system_headers = { "Authorization": 'Bearer ' + auth_token } file_system_params = {"classified": file_name} url_response = requests.get( url=file_system_url, headers=file_system_headers, params=file_system_params) if url_response.status_code == 200: url_key = file_name + '_download_url' download_url = (url_response.json( ))['data']['classified_response'][url_key] else: responseObject = { 'status': 'fail', 'message': 'Error connecting to file_system.' } return make_response( jsonify(responseObject)), 500 download_response = requests.get(download_url) if download_response.status_code == 200: responseObject = { 'status': 'success', 'message': 'Returning classified information.', 'data': download_response.json() } return make_response(jsonify(responseObject)), 200 else: responseObject = { 'status': 'fail', 'message': 'Bad response from download url. Please try downloading again, or classify your csv again.' } return make_response(jsonify(responseObject)), 404 else: responseObject = { 'status': 'fail', 'message': 'File name not provided.' } return make_response(jsonify(responseObject)), 400 else: responseObject = { 'status': 'fail', 'message': 'File name not provided.' } return make_response(jsonify(responseObject)), 400 else: responseObject = {'status': 'fail', 'message': resp} return make_response(jsonify(responseObject)), 401 else: responseObject = { 'status': 'fail', 'message': 'Provide a valid auth token.' } return make_response(jsonify(responseObject)), 401
def get(self): # get the auth token auth_header = request.headers.get('Authorization') if auth_header: try: auth_token = auth_header.split(" ")[1] except IndexError: responseObject = { 'status': 'fail', 'message': 'Bearer token malformed.' } return make_response(jsonify(responseObject)), 401 else: auth_token = '' if auth_token: resp = decode_auth_token(auth_token) # Check if decode_auth_token returned a string (which means it failed) # If it succeeded, resp now holds the user_id value if not isinstance(resp, str): # Get a status row for the user status = MLStatus.query.filter_by(user_id=resp).first() if not status: # User not in status table - error responseObject = { 'status': 'fail', 'message': 'User has not classified any data.' } return make_response(jsonify(responseObject)), 404 else: if status.status == "Completed.": # make sure row has JSON value resp_json = status.classified_json if resp_json: responseObject = { 'status': 'success', 'message': 'Returning classified information.', 'data': resp_json } return make_response(jsonify(responseObject)), 200 else: # No json object in db for user responseObject = { 'status': 'fail', 'message': 'No classified data found for given user.' } return make_response(jsonify(responseObject)), 404 else: # ML has not yet completed for user responseObject = { 'status': 'fail', 'message': 'Classification not yet completed for given user. Current status: ' + status.status } return make_response(jsonify(responseObject)), 401 else: responseObject = {'status': 'fail', 'message': resp} return make_response(jsonify(responseObject)), 401 else: responseObject = { 'status': 'fail', 'message': 'Provide a valid auth token.' } return make_response(jsonify(responseObject)), 401
def get(self): # get the auth token auth_header = request.headers.get('Authorization') if auth_header: try: auth_token = auth_header.split(" ")[1] except IndexError: responseObject = { 'status': 'fail', 'message': 'Bearer token malformed.' } return make_response(jsonify(responseObject)), 401 else: auth_token = '' if auth_token: resp = decode_auth_token(auth_token) # Check if decode_auth_token returnned a string (which means it failed) # If it succeeded, resp now holds the user_id value if not isinstance(resp, str): # Check the ml status for the user status = MLStatus.query.filter_by(user_id=resp).first() if not status: # User not in status table - add them now try: user = MLStatus(user_id=resp, status="Waiting for files.") # insert the user db.session.add(user) db.session.commit() # update status object to reference newly created user status = MLStatus.query.filter_by(user_id=resp).first() if not status: responseObject = { 'status': 'fail', 'message': 'User could not be added to status database.' } return make_response(jsonify(responseObject)), 401 except Exception as e: responseObject = { 'status': 'fail', 'message': 'Some error occurred. Please try again.' } return make_response(jsonify(responseObject)), 401 if status == 'Failed to process files.': responseObject = { 'status': 'success', 'message': status.status, 'error_msg': status.error_msg } return make_response(jsonify(responseObject)), 200 responseObject = { 'status': 'success', 'message': status.status } return make_response(jsonify(responseObject)), 200 else: responseObject = {'status': 'fail', 'message': resp} return make_response(jsonify(responseObject)), 401 else: responseObject = { 'status': 'fail', 'message': 'Provide a valid auth token.' } return make_response(jsonify(responseObject)), 401