def getErrorMetrics(self) : """ Returns an Http response object containing error information for every validation job in specified submission """ responseDict = {} returnDict = {} try: safeDictionary = RequestDictionary(self.request) submission_id = safeDictionary.getValue("submission_id") # Check if user has permission to specified submission self.checkSubmissionPermission(self.jobManager.getSubmissionById(submission_id)) jobIds = self.jobManager.getJobsBySubmission(submission_id) for currentId in jobIds : if(self.jobManager.getJobType(currentId) == "csv_record_validation"): fileName = self.jobManager.getFileType(currentId) dataList = self.interfaces.errorDb.getErrorMetricsByJobId(currentId) returnDict[fileName] = dataList return JsonResponse.create(StatusCode.OK,returnDict) except ( ValueError , TypeError ) as e: return JsonResponse.error(e,StatusCode.CLIENT_ERROR) except ResponseException as e: return JsonResponse.error(e,e.status) except Exception as e: # Unexpected exception, this is a 500 server error return JsonResponse.error(e,StatusCode.INTERNAL_ERROR)
def uploadFile(self): """saves a file and returns the saved path""" try: if(self.isLocal): uploadedFile = request.files['file'] if(uploadedFile): seconds = int((datetime.utcnow()-datetime(1970,1,1)).total_seconds()) filename = "".join([str(seconds),"_", secure_filename(uploadedFile.filename)]) path = os.path.join(self.serverPath, filename) uploadedFile.save(path) returnDict = {"path":path} return JsonResponse.create(StatusCode.OK,returnDict) else: exc = ResponseException("Failure to read file", StatusCode.CLIENT_ERROR) return JsonResponse.error(exc,exc.status) else : exc = ResponseException("Route Only Valid For Local Installs", StatusCode.CLIENT_ERROR) return JsonResponse.error(exc,exc.status) except ( ValueError , TypeError ) as e: return JsonResponse.error(e,StatusCode.CLIENT_ERROR) except ResponseException as e: return JsonResponse.error(e,e.status) except Exception as e: # Unexpected exception, this is a 500 server error return JsonResponse.error(e,StatusCode.INTERNAL_ERROR)
def getErrorReportURLsForSubmission(self): """ Gets the Signed URLs for download based on the submissionId """ try : self.s3manager = s3UrlHandler() safeDictionary = RequestDictionary(self.request) submissionId = safeDictionary.getValue("submission_id") responseDict ={} for jobId in self.jobManager.getJobsBySubmission(submissionId): if(self.jobManager.getJobType(jobId) == "csv_record_validation"): if(not self.isLocal): responseDict["job_"+str(jobId)+"_error_url"] = self.s3manager.getSignedUrl("errors",self.jobManager.getReportPath(jobId),"GET") else: path = os.path.join(self.serverPath, self.jobManager.getReportPath(jobId)) responseDict["job_"+str(jobId)+"_error_url"] = path if(not self.isLocal): crossFileReport = self.s3manager.getSignedUrl("errors",self.jobManager.getCrossFileReportPath(submissionId),"GET") else: crossFileReport = os.path.join(self.serverPath, self.jobManager.getCrossFileReportPath(submissionId)) responseDict["cross_file_error_url"] = crossFileReport return JsonResponse.create(StatusCode.OK,responseDict) except ResponseException as e: return JsonResponse.error(e,StatusCode.CLIENT_ERROR) except Exception as e: # Unexpected exception, this is a 500 server error return JsonResponse.error(e,StatusCode.INTERNAL_ERROR)
def get_submission_data(submission, file_type=''): """ Get data for the submission specified Args: submission: submission to retrieve metadata for file_type: the type of job to retrieve metadata for Returns: JsonResponse containing the error information or the object containing metadata for all relevant file types """ sess = GlobalDB.db().session file_type = file_type.lower() # Make sure the file type provided is valid if file_type and file_type not in FILE_TYPE_DICT and file_type != 'cross': return JsonResponse.error(ValueError(file_type + ' is not a valid file type'), StatusCode.CLIENT_ERROR) # Make sure the file type provided is valid for the submission type is_fabs = submission.d2_submission if file_type and (is_fabs and file_type != 'fabs') or (not is_fabs and file_type == 'fabs'): return JsonResponse.error(ValueError(file_type + ' is not a valid file type for this submission'), StatusCode.CLIENT_ERROR) job_query = sess.query(Job).filter(Job.submission_id == submission.submission_id) if not file_type: relevant_job_types = (JOB_TYPE_DICT['csv_record_validation'], JOB_TYPE_DICT['validation']) job_query = job_query.filter(Job.job_type_id.in_(relevant_job_types)) elif file_type == 'cross': job_query = job_query.filter(Job.job_type_id == JOB_TYPE_DICT['validation']) else: job_query = job_query.filter(Job.file_type_id == FILE_TYPE_DICT[file_type]) job_dict = {'jobs': [job_to_dict(job) for job in job_query]} return JsonResponse.create(StatusCode.OK, job_dict)
def finalize(self): """ Set upload job in job tracker database to finished, allowing dependent jobs to be started Flask request should include key "upload_id", which holds the job_id for the file_upload job Returns: A flask response object, if successful just contains key "success" with value True, otherwise value is False """ responseDict = {} try: inputDictionary = RequestDictionary(self.request) jobId = inputDictionary.getValue("upload_id") # Compare user ID with user who submitted job, if no match return 400 job = self.jobManager.getJobById(jobId) submission = self.jobManager.getSubmissionForJob(job) if(submission.user_id != LoginSession.getName(session)): # This user cannot finalize this job raise ResponseException("Cannot finalize a job created by a different user", StatusCode.CLIENT_ERROR) # Change job status to finished if(self.jobManager.checkUploadType(jobId)): self.jobManager.changeToFinished(jobId) responseDict["success"] = True return JsonResponse.create(StatusCode.OK,responseDict) else: raise ResponseException("Wrong job type for finalize route",StatusCode.CLIENT_ERROR) except ( ValueError , TypeError ) as e: return JsonResponse.error(e,StatusCode.CLIENT_ERROR) except ResponseException as e: return JsonResponse.error(e,e.status) except Exception as e: # Unexpected exception, this is a 500 server error return JsonResponse.error(e,StatusCode.INTERNAL_ERROR)
def resetPassword(self,system_email,session): """ Remove old password and email user a token to set a new password. Request should have key "email" arguments: system_email -- (string) email used to send messages session -- (Session) object from flask """ requestDict = RequestDictionary(self.request) if(not (requestDict.exists("email"))): # Don't have the keys we need in request exc = ResponseException("Reset password route requires key 'email'",StatusCode.CLIENT_ERROR) return JsonResponse.error(exc,exc.status) # Get user object try: user = self.interfaces.userDb.getUserByEmail(requestDict.getValue("email")) except Exception as e: exc = ResponseException("Unknown Error",StatusCode.CLIENT_ERROR,ValueError) return JsonResponse.error(exc,exc.status) email = requestDict.getValue("email") LoginSession.logout(session) self.sendResetPasswordEmail(user, system_email, email) # Return success message return JsonResponse.create(StatusCode.OK,{"message":"Password reset"})
def resetPassword(self,system_email,session): """ Remove old password and email user a token to set a new password. Request should have key "email" arguments: system_email -- (string) email used to send messages session -- (Session) object from flask """ requestDict = RequestDictionary(self.request) if(not (requestDict.exists("email"))): # Don't have the keys we need in request exc = ResponseException("Reset password route requires key 'email'",StatusCode.CLIENT_ERROR) return JsonResponse.error(exc,exc.status) # Get user object try: user = self.interfaces.userDb.getUserByEmail(requestDict.getValue("email")) except Exception as e: exc = ResponseException("Unknown Error",StatusCode.CLIENT_ERROR,ValueError) return JsonResponse.error(exc,exc.status) LoginSession.logout(session) self.interfaces.userDb.session.commit() email = requestDict.getValue("email") # Send email with token emailToken = sesEmail.createToken(email,self.interfaces.userDb,"password_reset") link= "".join([ AccountHandler.FRONT_END,'#/forgotpassword/',emailToken]) emailTemplate = { '[URL]':link} newEmail = sesEmail(user.email, system_email,templateType="reset_password",parameters=emailTemplate,database=self.interfaces.userDb) newEmail.send() # Return success message return JsonResponse.create(StatusCode.OK,{"message":"Password reset"})
def delete_submission(submission): """ Deletes all data associated with the specified submission NOTE: THERE IS NO WAY TO UNDO THIS """ if submission.publish_status_id != PUBLISH_STATUS_DICT['unpublished']: return JsonResponse.error(ValueError("Submissions that have been certified cannot be deleted"), StatusCode.CLIENT_ERROR) sess = GlobalDB.db().session # Check if the submission has any jobs that are currently running, if so, do not allow deletion jobs = sess.query(Job).filter(Job.submission_id == submission.submission_id, Job.job_status_id == JOB_STATUS_DICT['running']).all() if jobs: return JsonResponse.error(ValueError("Submissions with running jobs cannot be deleted"), StatusCode.CLIENT_ERROR) sess.query(SubmissionSubTierAffiliation).filter( SubmissionSubTierAffiliation.submission_id == submission.submission_id).delete( synchronize_session=False) sess.query(Submission).filter(Submission.submission_id == submission.submission_id).delete( synchronize_session=False) sess.expire_all() return JsonResponse.create(StatusCode.OK, {"message": "Success"})
def createEmailConfirmation(self,system_email,session): """ Creates user record and email arguments: system_email -- (string) email used to send messages session -- (Session) object from flask """ requestFields = RequestDictionary(self.request) if(not requestFields.exists("email")): exc = ResponseException("Request body must include email", StatusCode.CLIENT_ERROR) return JsonResponse.error(exc,exc.status) email = requestFields.getValue("email") if( not re.match("[^@]+@[^@]+\.[^@]+",email)) : return JsonResponse.error(ValueError("Invalid Email Format"),StatusCode.CLIENT_ERROR) try : user = self.interfaces.userDb.getUserByEmail(requestFields.getValue("email")) except ResponseException as e: self.interfaces.userDb.addUnconfirmedEmail(email) else: if(not (user.user_status_id == self.interfaces.userDb.getUserStatusId("awaiting_confirmation") or user.user_status_id == self.interfaces.userDb.getUserStatusId("email_confirmed"))): exc = ResponseException("User already registered", StatusCode.CLIENT_ERROR) return JsonResponse.error(exc,exc.status) emailToken = sesEmail.createToken(email,self.interfaces.userDb,"validate_email") link= "".join([AccountHandler.FRONT_END,'#/registration/',emailToken]) emailTemplate = {'[USER]': email, '[URL]':link} newEmail = sesEmail(email, system_email,templateType="validate_email",parameters=emailTemplate,database=self.interfaces.userDb) newEmail.send() return JsonResponse.create(StatusCode.OK,{"message":"Email Sent"})
def setSkipGuide(self, session): """ Set current user's skip guide parameter """ uid = session["name"] userDb = self.interfaces.userDb user = userDb.getUserByUID(uid) requestDict = RequestDictionary(self.request) if not requestDict.exists("skip_guide"): exc = ResponseException("Must include skip_guide parameter", StatusCode.CLIENT_ERROR) return JsonResponse.error(exc, exc.status) skipGuide = requestDict.getValue("skip_guide") if type(skipGuide) == type(True): # param is a bool user.skip_guide = skipGuide elif type(skipGuide) == type("string"): # param is a string, allow "true" or "false" if skipGuide.lower() == "true": user.skip_guide = True elif skipGuide.lower() == "false": user.skip_guide = False else: exc = ResponseException("skip_guide must be true or false", StatusCode.CLIENT_ERROR) return JsonResponse.error(exc, exc.status) else: exc = ResponseException("skip_guide must be a boolean", StatusCode.CLIENT_ERROR) return JsonResponse.error(exc, exc.status) userDb.session.commit() return JsonResponse.create(StatusCode.OK,{"message":"skip_guide set successfully","skip_guide":skipGuide})
def run_instance_function( accountManager, accountFunction, getSystemEmail=False, getSession=False, getUser=False, getCredentials=False ): """ Standard error handling around each route """ interfaces = InterfaceHolder() try: accountManager.addInterfaces(interfaces) if getSystemEmail and getSession: return accountFunction(RouteUtils.SYSTEM_EMAIL, session) elif getSystemEmail: return accountFunction(RouteUtils.SYSTEM_EMAIL) elif getSession: return accountFunction(session) elif getUser: if getCredentials: return accountFunction(LoginSession.getName(session), RouteUtils.CREATE_CREDENTIALS) else: # Currently no functions with user but not credentials flag raise ValueError("Invalid combination of flags to run_instance_function") else: return accountFunction() except ResponseException as e: return JsonResponse.error(e, e.status) except Exception as e: exc = ResponseException(str(e), StatusCode.INTERNAL_ERROR, type(e)) return JsonResponse.error(exc, exc.status) finally: interfaces.close()
def listUsersWithStatus(self): """ List all users with the specified status. Associated request body must have key 'status' """ requestDict = RequestDictionary(self.request) if(not (requestDict.exists("status"))): # Missing a required field, return 400 exc = ResponseException("Request body must include status", StatusCode.CLIENT_ERROR) return JsonResponse.error(exc,exc.status) current_user = self.interfaces.userDb.getUserByUID(flaskSession["name"]) try: if self.interfaces.userDb.hasPermission(current_user, "agency_admin"): users = self.interfaces.userDb.getUsersByStatus(requestDict.getValue("status"), current_user.cgac_code) else: users = self.interfaces.userDb.getUsersByStatus(requestDict.getValue("status")) except ValueError as e: # Client provided a bad status exc = ResponseException(str(e),StatusCode.CLIENT_ERROR,ValueError) return JsonResponse.error(exc,exc.status) userInfo = [] for user in users: agency_name = self.interfaces.validationDb.getAgencyName(user.cgac_code) thisInfo = {"name":user.name, "title":user.title, "agency_name":agency_name, "cgac_code":user.cgac_code, "email":user.email, "id":user.user_id } userInfo.append(thisInfo) return JsonResponse.create(StatusCode.OK,{"users":userInfo})
def login(self,session): """ Logs a user in if their password matches arguments: session -- (Session) object from flask return the reponse object """ try: safeDictionary = RequestDictionary(self.request) username = safeDictionary.getValue('username') password = safeDictionary.getValue('password') try: user = self.interfaces.userDb.getUserByEmail(username) except Exception as e: raise ValueError("user name and or password invalid") if(not self.interfaces.userDb.checkStatus(user,"approved")): raise ValueError("user name and or password invalid") # Only check if user is active after they've logged in for the first time if user.last_login_date is not None and not self.isUserActive(user): raise ValueError("Your account has expired. Please contact an administrator.") try: if(self.interfaces.userDb.checkPassword(user,password,self.bcrypt)): # We have a valid login LoginSession.login(session,user.user_id) permissionList = [] for permission in self.interfaces.userDb.getPermssionList(): if(self.interfaces.userDb.hasPermission(user, permission.name)): permissionList.append(permission.permission_type_id) self.interfaces.userDb.updateLastLogin(user) return JsonResponse.create(StatusCode.OK,{"message":"Login successful","user_id": int(user.user_id),"name":user.name,"title":user.title ,"agency":user.agency, "permissions" : permissionList}) else : raise ValueError("user name and or password invalid") except Exception as e: LoginSession.logout(session) raise ValueError("user name and or password invalid") except (TypeError, KeyError, NotImplementedError) as e: # Return a 400 with appropriate message return JsonResponse.error(e,StatusCode.CLIENT_ERROR) except ValueError as e: # Return a 401 for login denied return JsonResponse.error(e,StatusCode.LOGIN_REQUIRED) except Exception as e: # Return 500 return JsonResponse.error(e,StatusCode.INTERNAL_ERROR) return self.response
def submission_list_certifications(submission): if submission.d2_submission: return JsonResponse.error(ValueError("FABS submissions do not have a certification history"), StatusCode.CLIENT_ERROR) sess = GlobalDB.db().session certify_history = sess.query(CertifyHistory).filter_by(submission_id=submission.submission_id) if certify_history.count() == 0: return JsonResponse.error(ValueError("This submission has no certification history"), StatusCode.CLIENT_ERROR) return list_certifications(submission)
def login(self, session): """ Logs a user in if their password matches arguments: session -- (Session) object from flask return the response object """ try: sess = GlobalDB.db().session safe_dictionary = RequestDictionary(self.request) username = safe_dictionary.get_value('username') password = safe_dictionary.get_value('password') try: user = sess.query(User).filter(func.lower(User.email) == func.lower(username)).one() except Exception: raise ValueError("Invalid username and/or password") try: if check_correct_password(user, password, self.bcrypt): # We have a valid login return self.create_session_and_response(session, user) else: raise ValueError("Invalid username and/or password") except ValueError as ve: LoginSession.logout(session) raise ve except Exception as e: LoginSession.logout(session) raise e except (TypeError, KeyError, NotImplementedError) as e: # Return a 400 with appropriate message return JsonResponse.error(e, StatusCode.CLIENT_ERROR) except ValueError as e: # Return a 401 for login denied return JsonResponse.error(e, StatusCode.LOGIN_REQUIRED) except Exception as e: # Return 500 return JsonResponse.error(e, StatusCode.INTERNAL_ERROR)
def checkPasswordToken(self,session): """ Checks the password token if its valid arguments: session -- (Session) object from flask return the reponse object with a error code and a message """ requestFields = RequestDictionary(self.request) if(not requestFields.exists("token")): exc = ResponseException("Request body must include token", StatusCode.CLIENT_ERROR) return JsonResponse.error(exc,exc.status) token = requestFields.getValue("token") success,message,errorCode = sesEmail.checkToken(token,self.interfaces.userDb,"password_reset") if(success): #mark session that password can be filled out LoginSession.resetPassword(session) return JsonResponse.create(StatusCode.OK,{"email":message,"errorCode":errorCode,"message":"success"}) else: #failure but alert UI of issue return JsonResponse.create(StatusCode.OK,{"errorCode":errorCode,"message":message})
def set_skip_guide(self): """ Set current user's skip guide parameter Returns: JsonResponse object containing results of setting the skip guide or details of the error that occurred. Possible errors include the request not containing a skip_guide parameter or it not being a boolean value """ sess = GlobalDB.db().session request_dict = RequestDictionary.derive(self.request) try: if 'skip_guide' not in request_dict: raise ResponseException( "Must include skip_guide parameter", StatusCode.CLIENT_ERROR ) skip_guide = str(request_dict['skip_guide']).lower() if skip_guide not in ("true", "false"): raise ResponseException( "skip_guide must be true or false", StatusCode.CLIENT_ERROR ) g.user.skip_guide = skip_guide == "true" except ResponseException as exc: return JsonResponse.error(exc, exc.status) sess.commit() return JsonResponse.create(StatusCode.OK, {"message": "skip_guide set successfully", "skip_guide": skip_guide})
def checkEmailConfirmationToken(self,session): """ Creates user record and email arguments: session -- (Session) object from flask return the reponse object with a error code and a message """ requestFields = RequestDictionary(self.request) if(not requestFields.exists("token")): exc = ResponseException("Request body must include token", StatusCode.CLIENT_ERROR) return JsonResponse.error(exc,exc.status) token = requestFields.getValue("token") success,message,errorCode = sesEmail.checkToken(token,self.interfaces.userDb,"validate_email") if(success): #mark session that email can be filled out LoginSession.register(session) #remove token so it cant be used again # The following line is commented out for issues with registration email links bouncing users back # to the original email input page instead of the registration page #self.interfaces.userDb.deleteToken(token) #set the status only if current status is awaiting confirmation user = self.interfaces.userDb.getUserByEmail(message) if self.interfaces.userDb.checkStatus(user,"awaiting_confirmation"): self.interfaces.userDb.changeStatus(user,"email_confirmed") return JsonResponse.create(StatusCode.OK,{"email":message,"errorCode":errorCode,"message":"success"}) else: #failure but alert UI of issue return JsonResponse.create(StatusCode.OK,{"errorCode":errorCode,"message":message})
def upload_fabs_file(): if "multipart/form-data" not in request.headers['Content-Type']: return JsonResponse.error(ValueError("Request must be a multipart/form-data type"), StatusCode.CLIENT_ERROR) params = RequestDictionary.derive(request) fabs = params.get('_files', {}).get('fabs', None) file_manager = FileHandler(request, is_local=is_local, server_path=server_path) return file_manager.upload_fabs_file(fabs)
def check_year_and_quarter(cgac_code, frec_code, reporting_fiscal_year, reporting_fiscal_period): """ Check if cgac (or frec) code, year, and quarter already has a published submission """ if not cgac_code and not frec_code: return JsonResponse.error(ValueError("CGAC or FR Entity Code required"), StatusCode.CLIENT_ERROR) sess = GlobalDB.db().session return find_existing_submissions_in_period(sess, cgac_code, frec_code, reporting_fiscal_year, reporting_fiscal_period)
def login(self, session): """ Logs a user in if their password matches using local data Args: session: the Session object from flask Returns: A JsonResponse containing the user information or details on which error occurred, such as whether a type was wrong, something wasn't implemented, invalid keys were provided, login was denied, or a different, unexpected error occurred. """ try: sess = GlobalDB.db().session safe_dictionary = RequestDictionary(self.request) username = safe_dictionary.get_value('username') password = safe_dictionary.get_value('password') try: user = sess.query(User).filter(func.lower(User.email) == func.lower(username)).one() except Exception: raise ValueError("Invalid username and/or password") try: if check_correct_password(user, password, self.bcrypt): # We have a valid login return self.create_session_and_response(session, user) else: raise ValueError("Invalid username and/or password") except ValueError as ve: LoginSession.logout(session) raise ve except Exception as e: LoginSession.logout(session) raise e # Catch any specifically raised errors or any other errors that may have happened and return them cleanly except (TypeError, KeyError, NotImplementedError) as e: # Return a 400 with appropriate message return JsonResponse.error(e, StatusCode.CLIENT_ERROR) except ValueError as e: # Return a 401 for login denied return JsonResponse.error(e, StatusCode.LOGIN_REQUIRED) except Exception as e: # Return 500 return JsonResponse.error(e, StatusCode.INTERNAL_ERROR)
def generate_detached_file(file_type, cgac_code, frec_code, start, end): """ Generate a file from external API, independent from a submission """ if not cgac_code and not frec_code: return JsonResponse.error(ValueError("Detached file generation requires CGAC or FR Entity Code"), StatusCode.CLIENT_ERROR) file_manager = FileHandler(request, is_local=is_local, server_path=server_path) return file_manager.generate_detached_file(file_type, cgac_code, frec_code, start, end)
def certify_submission(submission): if not submission.publishable: return JsonResponse.error(ValueError("Submission cannot be certified due to critical errors"), StatusCode.CLIENT_ERROR) if not submission.is_quarter_format: return JsonResponse.error(ValueError("Monthly submissions cannot be certified"), StatusCode.CLIENT_ERROR) if submission.publish_status_id == PUBLISH_STATUS_DICT['published']: return JsonResponse.error(ValueError("Submission has already been certified"), StatusCode.CLIENT_ERROR) windows = get_window() for window in windows: if window.block_certification: return JsonResponse.error(ValueError(window.message), StatusCode.CLIENT_ERROR) sess = GlobalDB.db().session response = find_existing_submissions_in_period(sess, submission.cgac_code, submission.frec_code, submission.reporting_fiscal_year, submission.reporting_fiscal_period, submission.submission_id) if response.status_code == StatusCode.OK: sess = GlobalDB.db().session # create the certify_history entry certify_history = CertifyHistory(created_at=datetime.utcnow(), user_id=g.user.user_id, submission_id=submission.submission_id) sess.add(certify_history) sess.commit() # get the certify_history entry including the PK certify_history = sess.query(CertifyHistory).filter_by(submission_id=submission.submission_id).\ order_by(CertifyHistory.created_at.desc()).first() # move files (locally we don't move but we still need to populate the certified_files_history table file_manager = FileHandler(request, is_local=is_local, server_path=server_path) file_manager.move_certified_files(submission, certify_history, is_local) # set submission contents submission.certifying_user_id = g.user.user_id submission.publish_status_id = PUBLISH_STATUS_DICT['published'] sess.commit() return response
def validate_threaded(): """Start the validation process on a new thread.""" @copy_current_request_context def ThreadedFunction(arg): """The new thread.""" threadedManager = ValidationManager(local, error_report_path) threadedManager.threadedValidateJob(arg) try: interfaces = InterfaceHolder() jobTracker = interfaces.jobDb except ResponseException as e: open("errorLog","a").write(str(e) + "\n") return JsonResponse.error(e,e.status,table = "cannot connect to job database") except Exception as e: open("errorLog","a").write(str(e) + "\n") exc = ResponseException(str(e),StatusCode.INTERNAL_ERROR,type(e)) return JsonResponse.error(exc,exc.status,table= "cannot connect to job database") jobId = None manager = ValidationManager(local, error_report_path) try: jobId = manager.getJobID(request) except ResponseException as e: manager.markJob(jobId,jobTracker,"invalid",interfaces.errorDb,manager.filename) CloudLogger.logError(str(e),e,traceback.extract_tb(sys.exc_info()[2])) return JsonResponse.error(e,e.status,table ="") except Exception as e: exc = ResponseException(str(e),StatusCode.CLIENT_ERROR,type(e)) manager.markJob(jobId,jobTracker,"invalid",interfaces.errorDb,manager.filename) CloudLogger.logError(str(e),exc,traceback.extract_tb(sys.exc_info()[2])) return JsonResponse.error(exc,exc.status,table="") try: manager.testJobID(jobId,interfaces) except ResponseException as e: open("errorLog","a").write(str(e) + "\n") # Job is not ready to run according to job tracker, do not change status of job in job tracker interfaces.errorDb.writeFileError(jobId,manager.filename,ValidationError.jobError) return JsonResponse.error(e,e.status,table="") except Exception as e: open("errorLog","a").write(str(e) + "\n") exc = ResponseException(str(e),StatusCode.CLIENT_ERROR,type(e)) interfaces.errorDb.writeFileError(jobId,manager.filename,ValidationError.jobError) return JsonResponse.error(exc,exc.status,table="") thread = Thread(target=ThreadedFunction, args= (jobId,)) try : jobTracker.markJobStatus(jobId,"running") except Exception as e: open("errorLog","a").write(str(e) + "\n") exc = ResponseException(str(e),StatusCode.INTERNAL_ERROR,type(e)) return JsonResponse.error(exc,exc.status,table="could not start job") interfaces.close() thread.start() return JsonResponse.create(StatusCode.OK,{"table":"job"+str(jobId)})
def createEmailConfirmation(self, system_email, session): """ Creates user record and email arguments: system_email -- (string) email used to send messages session -- (Session) object from flask """ requestFields = RequestDictionary(self.request) if (not requestFields.exists("email")): exc = ResponseException("Request body must include email", StatusCode.CLIENT_ERROR) return JsonResponse.error(exc, exc.status) email = requestFields.getValue("email") if (not re.match("[^@]+@[^@]+\.[^@]+", email)): return JsonResponse.error(ValueError("Invalid Email Format"), StatusCode.CLIENT_ERROR) try: user = self.interfaces.userDb.getUserByEmail( requestFields.getValue("email")) except ResponseException as e: self.interfaces.userDb.addUnconfirmedEmail(email) else: if (not (user.user_status_id == self.interfaces.userDb. getUserStatusId("awaiting_confirmation") or user.user_status_id == self.interfaces.userDb. getUserStatusId("email_confirmed"))): exc = ResponseException("User already registered", StatusCode.CLIENT_ERROR) return JsonResponse.error(exc, exc.status) emailToken = sesEmail.createToken(email, self.interfaces.userDb, "validate_email") link = "".join( [AccountHandler.FRONT_END, '#/registration/', emailToken]) emailTemplate = {'[USER]': email, '[URL]': link} newEmail = sesEmail(email, system_email, templateType="validate_email", parameters=emailTemplate, database=self.interfaces.userDb) newEmail.send() return JsonResponse.create(StatusCode.OK, {"message": "Email Sent"})
def resetPassword(self, system_email, session): """ Remove old password and email user a token to set a new password. Request should have key "email" arguments: system_email -- (string) email used to send messages session -- (Session) object from flask """ requestDict = RequestDictionary(self.request) if (not (requestDict.exists("email"))): # Don't have the keys we need in request exc = ResponseException( "Reset password route requires key 'email'", StatusCode.CLIENT_ERROR) return JsonResponse.error(exc, exc.status) # Get user object try: user = self.interfaces.userDb.getUserByEmail( requestDict.getValue("email")) except Exception as e: exc = ResponseException("Unknown Error", StatusCode.CLIENT_ERROR, ValueError) return JsonResponse.error(exc, exc.status) LoginSession.logout(session) self.interfaces.userDb.session.commit() email = requestDict.getValue("email") # Send email with token emailToken = sesEmail.createToken(email, self.interfaces.userDb, "password_reset") link = "".join( [AccountHandler.FRONT_END, '#/forgotpassword/', emailToken]) emailTemplate = {'[URL]': link} newEmail = sesEmail(user.email, system_email, templateType="reset_password", parameters=emailTemplate, database=self.interfaces.userDb) newEmail.send() # Return success message return JsonResponse.create(StatusCode.OK, {"message": "Password reset"})
def upload_dabs_files(): if "multipart/form-data" not in request.headers['Content-Type']: return JsonResponse.error( ValueError("Request must be a multipart/form-data type"), StatusCode.CLIENT_ERROR) file_manager = FileHandler(request, is_local=is_local, server_path=server_path) return file_manager.validate_upload_dabs_files()
def setNewPassword(self): """ Set a new password for a user, request should have keys "user_email" and "password" """ requestDict = RequestDictionary(self.request) if(not (requestDict.exists("user_email") and requestDict.exists("password"))): # Don't have the keys we need in request exc = ResponseException("Set password route requires keys user_email and password",StatusCode.CLIENT_ERROR) return JsonResponse.error(exc,exc.status) if(not self.checkPassword(requestDict.getValue("password"))): exc = ResponseException("Invalid Password", StatusCode.CLIENT_ERROR) return JsonResponse.error(exc,exc.status) # Get user from email user = self.interfaces.userDb.getUserByEmail(requestDict.getValue("user_email")) # Set new password self.interfaces.userDb.setPassword(user,requestDict.getValue("password"),self.bcrypt) # Return success message return JsonResponse.create(StatusCode.OK,{"message":"Password successfully changed"})
def listUsersWithStatus(self): """ List all users with the specified status. Associated request body must have key 'status' """ requestDict = RequestDictionary(self.request) if(not (requestDict.exists("status"))): # Missing a required field, return 400 exc = ResponseException("Request body must include status", StatusCode.CLIENT_ERROR) return JsonResponse.error(exc,exc.status) try: users = self.interfaces.userDb.getUsersByStatus(requestDict.getValue("status")) except ValueError as e: # Client provided a bad status exc = ResponseException(str(e),StatusCode.CLIENT_ERROR,ValueError) return JsonResponse.error(exc,exc.status) userInfo = [] for user in users: thisInfo = {"name":user.name, "title":user.title, "agency":user.agency, "email":user.email, "id":user.user_id } userInfo.append(thisInfo) return JsonResponse.create(StatusCode.OK,{"users":userInfo})
def decorated_function(*args, **kwargs): try: errorMessage = "Login Required" if "check_email_token" in permissionList: if(LoginSession.isRegistering(session)) : return f(*args, **kwargs) else : errorMessage = "unauthorized" elif "check_password_token" in permissionList : if(LoginSession.isResetingPassword(session)) : return f(*args, **kwargs) else : errorMessage = "unauthorized" elif LoginSession.isLogin(session): userDb = UserHandler() try: user = userDb.getUserByUID(session["name"]) validUser = True for permission in permissionList : if(not userDb.hasPermission(user, permission)) : validUser = False else: validUser = True break finally: userDb.close() if(validUser) : return f(*args, **kwargs) errorMessage = "Wrong User Type" returnResponse = flask.Response() returnResponse.headers["Content-Type"] = "application/json" returnResponse.status_code = 401 # Error code responseDict = {} responseDict["message"] = errorMessage returnResponse.set_data(json.dumps(responseDict)) return returnResponse except ResponseException as e: return JsonResponse.error(e,e.status) except Exception as e: exc = ResponseException(str(e),StatusCode.INTERNAL_ERROR,type(e)) return JsonResponse.error(exc,exc.status)
def email_users(submission, system_email, template_type, user_ids): """ Send email notification to list of users Args: submission: the submission to send the email about system_email: the address of the system to send the email from template_type: the template type of the email to send user_ids: A list of user IDs denoting who to send the email to Returns: A JsonReponse containing a message that the email sent successfully or the details of the missing or incorrect parameters """ sess = GlobalDB.db().session if submission.cgac_code: agency = sess.query(CGAC).filter_by( cgac_code=submission.cgac_code).first() else: agency = sess.query(FREC).filter_by( frec_code=submission.frec_code).first() if not agency: return JsonResponse.error( ValueError( "The requested submission is not aligned to a valid CGAC or FREC " "agency"), StatusCode.CLIENT_ERROR) # Check if email template type is valid get_email_template(template_type) link = "".join([ AccountHandler.FRONT_END, '#/submission/', str(submission.submission_id) ]) email_template = { '[REV_USER_NAME]': g.user.name, '[REV_AGENCY]': agency.agency_name, '[REV_URL]': link } users = [] for user_id in user_ids: # Check if user id is valid, if so add User object to array users.append( sess.query(User).filter(User.user_id == user_id).one()) for user in users: new_email = SesEmail(user.email, system_email, template_type=template_type, parameters=email_template) new_email.send() return JsonResponse.create(StatusCode.OK, {"message": "Emails successfully sent"})
def finalize(self, jobId=None): """ Set upload job in job tracker database to finished, allowing dependent jobs to be started Flask request should include key "upload_id", which holds the job_id for the file_upload job Returns: A flask response object, if successful just contains key "success" with value True, otherwise value is False """ responseDict = {} try: if jobId is None: inputDictionary = RequestDictionary(self.request) jobId = inputDictionary.getValue("upload_id") # Compare user ID with user who submitted job, if no match return 400 job = self.jobManager.getJobById(jobId) submission = self.jobManager.getSubmissionForJob(job) # Check that user's agency matches submission cgac_code or "SYS", or user id matches submission's user userId = LoginSession.getName(session) userCgac = self.interfaces.userDb.getUserByUID(userId).cgac_code if (submission.user_id != userId and submission.cgac_code != userCgac and userCgac != "SYS"): # This user cannot finalize this job raise ResponseException( "Cannot finalize a job for a different agency", StatusCode.CLIENT_ERROR) # Change job status to finished if (self.jobManager.checkUploadType(jobId)): self.jobManager.changeToFinished(jobId) responseDict["success"] = True return JsonResponse.create(StatusCode.OK, responseDict) else: raise ResponseException("Wrong job type for finalize route", StatusCode.CLIENT_ERROR) except (ValueError, TypeError) as e: return JsonResponse.error(e, StatusCode.CLIENT_ERROR) except ResponseException as e: return JsonResponse.error(e, e.status) except Exception as e: # Unexpected exception, this is a 500 server error return JsonResponse.error(e, StatusCode.INTERNAL_ERROR)
def list_rule_labels(files, error_level='warning', fabs=False): """ Returns a list of rule labels based on the files and error type provided Args: files: A list of files for which to return rule labels. If blank, return all matching other arguments error_level: A string indicating whether to return errors, warnings, or both. Defaults to warning fabs: A boolean indicating whether to return FABS or DABS rules. Defaults to False Returns: JsonResponse of the rule labels the arguments indicate. JsonResponse error if invalid file types are provided or any file types are provided for FABS """ # Make sure list is empty when requesting FABS rules if fabs and len(files) > 0: return JsonResponse.error(ValueError('Files list must be empty for FABS rules'), StatusCode.CLIENT_ERROR) invalid_files = [invalid_file for invalid_file in files if invalid_file not in FILE_TYPES] if invalid_files: return JsonResponse.error(ValueError('The following are not valid file types: {}'. format(', '.join(invalid_files))), StatusCode.CLIENT_ERROR) sess = GlobalDB.db().session rule_label_query = sess.query(RuleSql.rule_label) # If the error level isn't "mixed" add a filter on which severity to pull if error_level == 'error': rule_label_query = rule_label_query.filter_by(rule_severity_id=RULE_SEVERITY_DICT['fatal']) elif error_level == 'warning': rule_label_query = rule_label_query.filter_by(rule_severity_id=RULE_SEVERITY_DICT['warning']) # If specific files have been specified, add a filter to get them if files: rule_label_query = file_filter(rule_label_query, RuleSql, files) elif not fabs: # If not the rules are not FABS, exclude FABS rules rule_label_query = rule_label_query.filter(RuleSql.file_id != FILE_TYPE_DICT_LETTER_ID['FABS']) else: # If the rule is FABS, add a filter to only get FABS rules rule_label_query = rule_label_query.filter_by(file_id=FILE_TYPE_DICT_LETTER_ID['FABS']) return JsonResponse.create(StatusCode.OK, {'labels': [label.rule_label for label in rule_label_query.all()]})
def get_submission_data(submission, file_type=''): """ Get data for the submission specified Args: submission: submission to retrieve metadata for file_type: the type of job to retrieve metadata for Returns: JsonResponse containing the error information or the object containing metadata for all relevant file types """ sess = GlobalDB.db().session file_type = file_type.lower() # Make sure the file type provided is valid if file_type and file_type not in FILE_TYPE_DICT and file_type != 'cross': return JsonResponse.error( ValueError(file_type + ' is not a valid file type'), StatusCode.CLIENT_ERROR) # Make sure the file type provided is valid for the submission type is_fabs = submission.d2_submission if file_type and (is_fabs and file_type != 'fabs') or ( not is_fabs and file_type == 'fabs'): return JsonResponse.error( ValueError(file_type + ' is not a valid file type for this submission'), StatusCode.CLIENT_ERROR) job_query = sess.query(Job).filter( Job.submission_id == submission.submission_id) if not file_type: relevant_job_types = (JOB_TYPE_DICT['csv_record_validation'], JOB_TYPE_DICT['validation']) job_query = job_query.filter(Job.job_type_id.in_(relevant_job_types)) elif file_type == 'cross': job_query = job_query.filter( Job.job_type_id == JOB_TYPE_DICT['validation']) else: job_query = job_query.filter( Job.file_type_id == FILE_TYPE_DICT[file_type]) job_dict = {'jobs': [job_to_dict(job) for job in job_query]} return JsonResponse.create(StatusCode.OK, job_dict)
def upload_fabs_file(): if "multipart/form-data" not in request.headers['Content-Type']: return JsonResponse.error( ValueError("Request must be a multipart/form-data type"), StatusCode.CLIENT_ERROR) params = RequestDictionary.derive(request) fabs = params.get('_files', {}).get('fabs', None) file_manager = FileHandler(request, is_local=is_local, server_path=server_path) return file_manager.upload_fabs_file(fabs)
def email_users(self, system_email): """ Send email notification to list of users """ sess = GlobalDB.db().session request_dict = RequestDictionary.derive(self.request) required = ('users', 'submission_id', 'email_template') try: if any(field not in request_dict for field in required): raise ResponseException( "Email users route requires users, email_template, and " "submission_id", StatusCode.CLIENT_ERROR) except ResponseException as exc: return JsonResponse.error(exc, exc.status) user_ids = request_dict['users'] submission_id = request_dict['submission_id'] # Check if submission id is valid _, agency_name = sess.query(Submission.submission_id, CGAC.agency_name)\ .join(CGAC, Submission.cgac_code == CGAC.cgac_code)\ .filter(Submission.submission_id == submission_id).one() if not agency_name: _, agency_name = sess.query(Submission.submission_id, FREC.agency_name) \ .join(FREC, Submission.frec_code == FREC.frec_code) \ .filter(Submission.submission_id == submission_id).one() template_type = request_dict['email_template'] # Check if email template type is valid get_email_template(template_type) users = [] link = "".join( [AccountHandler.FRONT_END, '#/reviewData/', str(submission_id)]) email_template = { '[REV_USER_NAME]': g.user.name, '[REV_AGENCY]': agency_name, '[REV_URL]': link } for user_id in user_ids: # Check if user id is valid, if so add User object to array users.append( sess.query(User).filter(User.user_id == user_id).one()) for user in users: new_email = SesEmail(user.email, system_email, template_type=template_type, parameters=email_template) new_email.send() return JsonResponse.create(StatusCode.OK, {"message": "Emails successfully sent"})
def deleteUser(self): """ Deletes user specified by 'email' in request """ requestDict = RequestDictionary(self.request) if not requestDict.exists("email"): # missing required fields, return 400 exc = ResponseException( "Request body must include email of user to be deleted", StatusCode.CLIENT_ERROR) return JsonResponse.error(exc, exc.status) email = requestDict.getValue("email") self.interfaces.userDb.deleteUser(email) return JsonResponse.create(StatusCode.OK, {"message": "success"})
def validate(): """Start the validation process on the same threads.""" interfaces = InterfaceHolder() # Create sessions for this route try: return validationManager.validateJob(request,interfaces) except Exception as e: # Something went wrong getting the flask request open("errorLog","a").write(str(e) + "\n") exc = ResponseException(str(e),StatusCode.INTERNAL_ERROR,type(e)) return JsonResponse.error(exc,exc.status,table="") finally: interfaces.close()
def completeGeneration(self, generationId): """ For files D1 and D2, the API uses this route as a callback to load the generated file. Requires an 'href' key in the request that specifies the URL of the file to be downloaded Args: generationId - Unique key stored in file_generation_task table, used in callback to identify which submission this file is for. """ if generationId is None: return JsonResponse.error(ResponseException("Must include a generation ID",StatusCode.CLIENT_ERROR), StatusCode.CLIENT_ERROR) self.smx_log_file_name = "smx_request.log" # Pull url from request safeDictionary = RequestDictionary(self.request) CloudLogger.log("DEBUG: Request content => " + safeDictionary.to_string(), log_type="debug", file_name=self.smx_log_file_name) if not safeDictionary.exists("href"): return JsonResponse.error(ResponseException("Request must include href key with URL of D file", StatusCode.CLIENT_ERROR), StatusCode.CLIENT_ERROR) url = safeDictionary.getValue("href") CloudLogger.log("DEBUG: Download URL => " + url, log_type="debug", file_name=self.smx_log_file_name) #Pull information based on task key try: CloudLogger.log("DEBUG: Pulling information based on task key...", log_type="debug", file_name=self.smx_log_file_name) task = self.interfaces.jobDb.session.query(FileGenerationTask).options(joinedload(FileGenerationTask.file_type)).filter(FileGenerationTask.generation_task_key == generationId).one() job = self.interfaces.jobDb.getJobById(task.job_id) CloudLogger.log("DEBUG: Loading D file...", log_type="debug", file_name=self.smx_log_file_name) result = self.load_d_file(url,job.filename,job.original_filename,job.job_id,self.isLocal) CloudLogger.log("DEBUG: Load D file result => " + str(result), log_type="debug", file_name=self.smx_log_file_name) return JsonResponse.create(StatusCode.OK,{"message":"File loaded successfully"}) except ResponseException as e: return JsonResponse.error(e, e.status) except NoResultFound as e: # Did not find file generation task return JsonResponse.error(ResponseException("Generation task key not found", StatusCode.CLIENT_ERROR), StatusCode.CLIENT_ERROR)
def changeStatus(self,system_email): """ Changes status for specified user. Associated request body should have keys 'uid' and 'new_status' arguments: system_email -- (string) the emaily to send emails from return the reponse object with a success message """ requestDict = RequestDictionary(self.request) if(not (requestDict.exists("uid") and requestDict.exists("new_status"))): # Missing a required field, return 400 exc = ResponseException("Request body must include uid and new_status", StatusCode.CLIENT_ERROR) return JsonResponse.error(exc,exc.status) # Find user that matches specified uid user = self.interfaces.userDb.getUserByUID(int(requestDict.getValue("uid"))) if(user.email == None): return JsonResponse.error(ResponseException("User does not have a defined email",StatusCode.INTERNAL_ERROR),StatusCode.INTERNAL_ERROR) #check if the user is waiting if(self.interfaces.userDb.checkStatus(user,"awaiting_approval")): if(requestDict.getValue("new_status") == "approved"): # Grant agency_user permission to newly approved users self.interfaces.userDb.grantPermission(user,"agency_user") link= AccountHandler.FRONT_END emailTemplate = { '[URL]':link,'[EMAIL]':system_email} newEmail = sesEmail(user.email, system_email,templateType="account_approved",parameters=emailTemplate,database=self.interfaces.userDb) newEmail.send() elif (requestDict.getValue("new_status") == "denied"): emailTemplate = {} newEmail = sesEmail(user.email, system_email,templateType="account_rejected",parameters=emailTemplate,database=self.interfaces.userDb) newEmail.send() # Change user's status self.interfaces.userDb.changeStatus(user,requestDict.getValue("new_status")) return JsonResponse.create(StatusCode.OK,{"message":"Status change successful"})
def listUsersWithStatus(self): """ List all users with the specified status. Associated request body must have key 'status' """ requestDict = RequestDictionary(self.request) if (not (requestDict.exists("status"))): # Missing a required field, return 400 exc = ResponseException("Request body must include status", StatusCode.CLIENT_ERROR) return JsonResponse.error(exc, exc.status) current_user = self.interfaces.userDb.getUserByUID( flaskSession["name"]) try: if self.interfaces.userDb.hasPermission(current_user, "agency_admin"): users = self.interfaces.userDb.getUsersByStatus( requestDict.getValue("status"), current_user.cgac_code) else: users = self.interfaces.userDb.getUsersByStatus( requestDict.getValue("status")) except ValueError as e: # Client provided a bad status exc = ResponseException(str(e), StatusCode.CLIENT_ERROR, ValueError) return JsonResponse.error(exc, exc.status) userInfo = [] for user in users: agency_name = self.interfaces.validationDb.getAgencyName( user.cgac_code) thisInfo = { "name": user.name, "title": user.title, "agency_name": agency_name, "cgac_code": user.cgac_code, "email": user.email, "id": user.user_id } userInfo.append(thisInfo) return JsonResponse.create(StatusCode.OK, {"users": userInfo})
def delete_all_submission_data(submission): """ Delete a submission. Args: submission: submission to delete Returns: JsonResponse object containing a success message or the reason for failure """ # check if the submission has been published, if so, do not allow deletion if submission.publish_status_id != PUBLISH_STATUS_DICT['unpublished']: return JsonResponse.error(ValueError("Submissions that have been certified cannot be deleted"), StatusCode.CLIENT_ERROR) sess = GlobalDB.db().session # check if the submission has any jobs that are currently running, if so, do not allow deletion running_jobs = sess.query(Job).filter(Job.submission_id == submission.submission_id, Job.job_status_id == JOB_STATUS_DICT['running']).all() if running_jobs: return JsonResponse.error(ValueError("Submissions with running jobs cannot be deleted"), StatusCode.CLIENT_ERROR) logger.info({ "message": "Deleting submission with id {}".format(submission.submission_id), "message_type": "BrokerInfo", "submission_id": submission.submission_id }) sess.query(SubmissionSubTierAffiliation).\ filter(SubmissionSubTierAffiliation.submission_id == submission.submission_id).\ delete(synchronize_session=False) sess.query(Submission).filter(Submission.submission_id == submission.submission_id).\ delete(synchronize_session=False) sess.expire_all() return JsonResponse.create(StatusCode.OK, {"message": "Success"})
def delete_all_submission_data(submission): """ Delete a submission. Args: submission: submission to delete Returns: JsonResponse object containing a success message or the reason for failure """ # check if the submission has been published, if so, do not allow deletion if submission.publish_status_id != PUBLISH_STATUS_DICT['unpublished']: return JsonResponse.error(ValueError("Submissions that have been certified cannot be deleted"), StatusCode.CLIENT_ERROR) sess = GlobalDB.db().session # check if the submission has any jobs that are currently running, if so, do not allow deletion running_jobs = sess.query(Job).filter(Job.submission_id == submission.submission_id, Job.job_status_id == JOB_STATUS_DICT['running']).all() if running_jobs: return JsonResponse.error(ValueError("Submissions with running jobs cannot be deleted"), StatusCode.CLIENT_ERROR) logger.info({ "message": "Deleting submission with id {}".format(submission.submission_id), "message_type": "BrokerInfo", "submission_id": submission.submission_id }) sess.query(SubmissionSubTierAffiliation).\ filter(SubmissionSubTierAffiliation.submission_id == submission.submission_id).\ delete(synchronize_session=False) sess.query(Submission).filter(Submission.submission_id == submission.submission_id).\ delete(synchronize_session=False) sess.expire_all() return JsonResponse.create(StatusCode.OK, {"message": "Success"})
def list_user_emails(self): """ List user names and emails """ sess = GlobalDB.db().session user = sess.query(User).filter(User.user_id == LoginSession.getName(flaskSession)).one() try: users = sess.query(User).filter(User.cgac_code == user.cgac_code, User.user_status_id == USER_STATUS_DICT["approved"], User.is_active == True).all() except ValueError as exc: # Client provided a bad status return JsonResponse.error(exc, StatusCode.CLIENT_ERROR) user_info = [] for user in users: this_info = {"id":user.user_id, "name": user.name, "email": user.email} user_info.append(this_info) return JsonResponse.create(StatusCode.OK, {"users": user_info})
def handle_response_exception(error): """Handle exceptions explicitly raised during validation.""" logger.error(str(error)) job = get_current_job() if job: if job.filename is not None: # insert file-level error info to the database writeFileError(job.job_id, job.filename, error.errorType, error.extraInfo) if error.errorType != ValidationError.jobError: # job pass prerequisites for validation, but an error # happened somewhere. mark job as 'invalid' mark_job_status(job.job_id, 'invalid') return JsonResponse.error(error, error.status)
def load_d_file(self, url, upload_name, timestamped_name, job_id, isLocal): """ Pull D file from specified URL and write to S3 """ job_manager = self.interfaces.jobDb try: full_file_path = "".join([CONFIG_BROKER['d_file_storage_path'], timestamped_name]) CloudLogger.log("DEBUG: Downloading file...", log_type="debug", file_name=self.smx_log_file_name) if not self.download_file(full_file_path, url): # Error occurred while downloading file, mark job as failed and record error message job_manager.markJobStatus(job_id, "failed") job = job_manager.getJobById(job_id) file_type = job_manager.getFileType(job_id) if file_type == "award": source= "ASP" elif file_type == "award_procurement": source = "FPDS" else: source = "unknown source" job.error_message = "A problem occurred receiving data from {}".format(source) raise ResponseException(job.error_message, StatusCode.CLIENT_ERROR) lines = self.get_lines_from_csv(full_file_path) write_csv(timestamped_name, upload_name, isLocal, lines[0], lines[1:]) CloudLogger.log("DEBUG: Marking job id of " + str(job_id) + " as finished", log_type="debug", file_name=self.smx_log_file_name) job_manager.markJobStatus(job_id, "finished") return {"message": "Success", "file_name": timestamped_name} except Exception as e: CloudLogger.log("ERROR: Exception caught => " + str(e), log_type="debug", file_name=self.smx_log_file_name) # Log the error JsonResponse.error(e,500) job_manager.getJobById(job_id).error_message = str(e) job_manager.markJobStatus(job_id, "failed") job_manager.session.commit() raise e
def listUsers(self): """ List all users ordered by status. Associated request body must have key 'filter_by' """ requestDict = RequestDictionary(self.request, optionalRequest=True) user_status = requestDict.getValue("status") if requestDict.exists( "status") else "all" user = self.interfaces.userDb.getUserByUID( LoginSession.getName(flaskSession)) isAgencyAdmin = self.userManager.hasPermission( user, "agency_admin") and not self.userManager.hasPermission( user, "website_admin") try: if isAgencyAdmin: users = self.interfaces.userDb.getUsers( cgac_code=user.cgac_code, status=user_status) else: users = self.interfaces.userDb.getUsers(status=user_status) except ValueError as e: # Client provided a bad status exc = ResponseException(str(e), StatusCode.CLIENT_ERROR, ValueError) return JsonResponse.error(exc, exc.status) userInfo = [] for user in users: agency_name = self.interfaces.validationDb.getAgencyName( user.cgac_code) thisInfo = { "name": user.name, "title": user.title, "agency_name": agency_name, "cgac_code": user.cgac_code, "email": user.email, "id": user.user_id, "is_active": user.is_active, "permissions": ",".join(self.interfaces.userDb.getUserPermissions(user)), "status": user.user_status.name } userInfo.append(thisInfo) return JsonResponse.create(StatusCode.OK, {"users": userInfo})
def find_existing_submissions_in_period(cgac_code, frec_code, reporting_fiscal_year, reporting_fiscal_period, submission_id=None): """ Find all the submissions in the given period for the given CGAC or FREC code Args: cgac_code: the CGAC code to check against or None if checking a FREC agency frec_code: the FREC code to check against or None if checking a CGAC agency reporting_fiscal_year: the year to check for reporting_fiscal_period: the period in the year to check for submission_id: the submission ID to check against (used when checking if this submission is being re-certified) Returns: A JsonResponse containing a success message to indicate there are no existing submissions in the given period or the error if there was one """ # We need either a cgac or a frec code for this function if not cgac_code and not frec_code: return JsonResponse.error( ValueError("CGAC or FR Entity Code required"), StatusCode.CLIENT_ERROR) sess = GlobalDB.db().session submission_query = sess.query(Submission).filter( (Submission.cgac_code == cgac_code) if cgac_code else (Submission.frec_code == frec_code), Submission.reporting_fiscal_year == reporting_fiscal_year, Submission.reporting_fiscal_period == reporting_fiscal_period, Submission.publish_status_id != PUBLISH_STATUS_DICT['unpublished']) # Filter out the submission we are potentially re-certifying if one is provided if submission_id: submission_query = submission_query.filter( Submission.submission_id != submission_id) submission_query = submission_query.order_by(desc(Submission.created_at)) if submission_query.count() > 0: data = { "message": "A submission with the same period already exists.", "submissionId": submission_query[0].submission_id } return JsonResponse.create(StatusCode.CLIENT_ERROR, data) return JsonResponse.create(StatusCode.OK, {"message": "Success"})
def handle_validation_exception(error): """Handle uncaught exceptions in validation process.""" logger.error(str(error)) # csv-specific errors get a different job status and response code if isinstance(error, ValueError) or isinstance(error, csv.Error): job_status, response_code = 'invalid', 400 else: job_status, response_code = 'failed', 500 job = get_current_job() if job: if job.filename is not None: writeFileError(job.job_id, job.filename, ValidationError.unknownError) mark_job_status(job.job_id, job_status) return JsonResponse.error(error, response_code)
def emailUsers(self, system_email, session): """ Send email notification to list of users """ requestDict = RequestDictionary(self.request) if not (requestDict.exists("users") and requestDict.exists("submission_id") and requestDict.exists("email_template")): exc = ResponseException( "Email users route requires users, email_template, and submission_id", StatusCode.CLIENT_ERROR) return JsonResponse.error(exc, exc.status) uid = session["name"] current_user = self.interfaces.userDb.getUserByUID(uid) user_ids = requestDict.getValue("users") submission_id = requestDict.getValue("submission_id") # Check if submission id is valid self.jobManager.getSubmissionById(submission_id) template_type = requestDict.getValue("email_template") # Check if email template type is valid self.userManager.getEmailTemplate(template_type) users = [] link = "".join( [AccountHandler.FRONT_END, '#/reviewData/', str(submission_id)]) emailTemplate = { '[REV_USER_NAME]': current_user.name, '[REV_URL]': link } for user_id in user_ids: # Check if user id is valid, if so add User object to array users.append(self.userManager.getUserByUID(user_id)) for user in users: newEmail = sesEmail(user.email, system_email, templateType=template_type, parameters=emailTemplate, database=UserHandler()) newEmail.send() return JsonResponse.create(StatusCode.OK, {"message": "Emails successfully sent"})
def deleteUser(self): """ Deletes user specified by 'email' in request """ sess = GlobalDB.db().session request_dict = RequestDictionary.derive(self.request) try: if 'email' not in request_dict: # missing required fields, return 400 raise ResponseException( "Request body must include email of user to be deleted", StatusCode.CLIENT_ERROR ) except ResponseException as exc: return JsonResponse.error(exc, exc.status) email = request_dict['email'] sess.query(User).filter(User.email == email).delete() sess.commit() return JsonResponse.create(StatusCode.OK,{"message":"success"})
def check_email_confirmation_token(self,session): """ Creates user record and email arguments: session -- (Session) object from flask return the response object with a error code and a message """ sess = GlobalDB.db().session request_fields = RequestDictionary.derive(self.request) try: if 'token' not in request_fields: raise ResponseException( "Request body must include token", StatusCode.CLIENT_ERROR ) except ResponseException as exc: return JsonResponse.error(exc, exc.status) token = request_fields['token'] session["token"] = token success, message, errorCode = sesEmail.check_token(token, "validate_email") if success: #mark session that email can be filled out LoginSession.register(session) #remove token so it cant be used again # The following lines are commented out for issues with registration email links bouncing users back # to the original email input page instead of the registration page # oldToken = sess.query(EmailToken).filter(EmailToken.token == session["token"]).one() # sess.delete(oldToken) # sess.commit() #set the status only if current status is awaiting confirmation user = sess.query(User).filter(func.lower(User.email) == func.lower(message)).one() if user.user_status_id == USER_STATUS_DICT["awaiting_confirmation"]: user.user_status_id = USER_STATUS_DICT["email_confirmed"] sess.commit() return JsonResponse.create(StatusCode.OK,{"email":message,"errorCode":errorCode,"message":"success"}) else: #failure but alert UI of issue return JsonResponse.create(StatusCode.OK,{"errorCode":errorCode,"message":message})
def checkEmailConfirmationToken(self, session): """ Creates user record and email arguments: session -- (Session) object from flask return the reponse object with a error code and a message """ requestFields = RequestDictionary(self.request) if (not requestFields.exists("token")): exc = ResponseException("Request body must include token", StatusCode.CLIENT_ERROR) return JsonResponse.error(exc, exc.status) token = requestFields.getValue("token") session["token"] = token success, message, errorCode = sesEmail.checkToken( token, self.interfaces.userDb, "validate_email") if (success): #mark session that email can be filled out LoginSession.register(session) #remove token so it cant be used again # The following line is commented out for issues with registration email links bouncing users back # to the original email input page instead of the registration page #self.interfaces.userDb.deleteToken(token) #set the status only if current status is awaiting confirmation user = self.interfaces.userDb.getUserByEmail(message) if self.interfaces.userDb.checkStatus(user, "awaiting_confirmation"): self.interfaces.userDb.changeStatus(user, "email_confirmed") return JsonResponse.create(StatusCode.OK, { "email": message, "errorCode": errorCode, "message": "success" }) else: #failure but alert UI of issue return JsonResponse.create(StatusCode.OK, { "errorCode": errorCode, "message": message })
def checkSubmissionById(self, submission_id, file_type): """ Check that submission exists and user has permission to it Args: submission_id: ID of submission to check file_type: file type that has been requested Returns: Tuple of boolean indicating whether submission has passed checks, and http response if not """ error_occurred = False try: submission = self.interfaces.jobDb.getSubmissionById(submission_id) except ResponseException as exc: if isinstance(exc.wrappedException, NoResultFound): # Submission does not exist, change to 400 in this case since route call specified a bad ID exc.status = StatusCode.CLIENT_ERROR message = "Submission does not exist" error_occurred = True error_exc = exc else: raise exc try: self.checkSubmissionPermission(submission) except ResponseException as exc: message = "User does not have permission to view that submission" error_occurred = True error_exc = exc if error_occurred: responseDict = { "message": message, "file_type": file_type, "url": "#", "status": "failed" } if file_type in ["D1", "D2"]: # Add empty start and end dates responseDict["start"] = "" responseDict["end"] = "" return False, JsonResponse.error(error_exc, error_exc.status, **responseDict) return True, None
def email_users(self, system_email): """ Send email notification to list of users """ sess = GlobalDB.db().session request_dict = RequestDictionary.derive(self.request) required = ('users', 'submission_id', 'email_template') try: if any(field not in request_dict for field in required): raise ResponseException( "Email users route requires users, email_template, and " "submission_id", StatusCode.CLIENT_ERROR ) except ResponseException as exc: return JsonResponse.error(exc, exc.status) user_ids = request_dict['users'] submission_id = request_dict['submission_id'] # Check if submission id is valid _, agency_name = sess.query(Submission.submission_id, CGAC.agency_name)\ .join(CGAC, Submission.cgac_code == CGAC.cgac_code)\ .filter(Submission.submission_id == submission_id).one() if not agency_name: _, agency_name = sess.query(Submission.submission_id, FREC.agency_name) \ .join(FREC, Submission.frec_code == FREC.frec_code) \ .filter(Submission.submission_id == submission_id).one() template_type = request_dict['email_template'] # Check if email template type is valid get_email_template(template_type) users = [] link = "".join([AccountHandler.FRONT_END, '#/reviewData/', str(submission_id)]) email_template = {'[REV_USER_NAME]': g.user.name, '[REV_AGENCY]': agency_name, '[REV_URL]': link} for user_id in user_ids: # Check if user id is valid, if so add User object to array users.append(sess.query(User).filter(User.user_id == user_id).one()) for user in users: new_email = SesEmail(user.email, system_email, template_type=template_type, parameters=email_template) new_email.send() return JsonResponse.create(StatusCode.OK, {"message": "Emails successfully sent"})
def set_skip_guide(self): """ Set current user's skip guide parameter """ sess = GlobalDB.db().session request_dict = RequestDictionary.derive(self.request) try: if 'skip_guide' not in request_dict: raise ResponseException("Must include skip_guide parameter", StatusCode.CLIENT_ERROR) skip_guide = str(request_dict['skip_guide']).lower() if skip_guide not in ("true", "false"): raise ResponseException("skip_guide must be true or false", StatusCode.CLIENT_ERROR) g.user.skip_guide = skip_guide == "true" except ResponseException as exc: return JsonResponse.error(exc, exc.status) sess.commit() return JsonResponse.create(StatusCode.OK, { "message": "skip_guide set successfully", "skip_guide": skip_guide })
def generateFile(self): """ Start a file generation job for the specified file type """ self.debug_file_name = "debug.log" CloudLogger.log("DEBUG: Starting D file generation", log_type="debug", file_name=self.debug_file_name) submission_id, file_type = self.getRequestParamsForGenerate() CloudLogger.log("DEBUG: Submission ID = " + str(submission_id) + " / File type = " + str(file_type), log_type="debug", file_name=self.debug_file_name) # Check permission to submission success, error_response = self.checkSubmissionById( submission_id, file_type) if not success: return error_response job = self.interfaces.jobDb.getJobBySubmissionFileTypeAndJobType( submission_id, self.fileTypeMap[file_type], "file_upload") # Check prerequisites on upload job if not self.interfaces.jobDb.runChecks(job.job_id): exc = ResponseException( "Must wait for completion of prerequisite validation job", StatusCode.CLIENT_ERROR) return JsonResponse.error(exc, exc.status) success, error_response = self.startGenerationJob( submission_id, file_type) CloudLogger.log("DEBUG: Finished startGenerationJob method", log_type="debug", file_name=self.debug_file_name) if not success: # If not successful, set job status as "failed" self.interfaces.jobDb.markJobStatus(job.job_id, "failed") return error_response # Return same response as check generation route return self.checkGeneration(submission_id, file_type)
def listUserEmails(self): """ List user names and emails """ user = self.interfaces.userDb.getUserByUID( LoginSession.getName(flaskSession)) try: users = self.interfaces.userDb.getUsers(cgac_code=user.cgac_code, status="approved", only_active=True) except ValueError as e: # Client provided a bad status exc = ResponseException(str(e), StatusCode.CLIENT_ERROR, ValueError) return JsonResponse.error(exc, exc.status) userInfo = [] for user in users: thisInfo = { "id": user.user_id, "name": user.name, "email": user.email } userInfo.append(thisInfo) return JsonResponse.create(StatusCode.OK, {"users": userInfo})