def listUsersWithStatus(self):
""" List all users with the specified status. Associated request body must have key 'status' """
requestDict = RequestDictionary(self.request)
if (not (requestDict.exists("status"))):
# Missing a required field, return 400
exc = ResponseException("Request body must include status",
StatusCode.CLIENT_ERROR)
return JsonResponse.error(exc, exc.status)
try:
users = self.interfaces.userDb.getUsersByStatus(
requestDict.getValue("status"))
except ValueError as e:
# Client provided a bad status
exc = ResponseException(str(e), StatusCode.CLIENT_ERROR,
ValueError)
return JsonResponse.error(exc, exc.status)
userInfo = []
for user in users:
thisInfo = {
"name": user.name,
"title": user.title,
"agency": user.agency,
"email": user.email,
"id": user.user_id
}
userInfo.append(thisInfo)
return JsonResponse.create(StatusCode.OK, {"users": userInfo})
def resetPassword(self, system_email, session):
"""
Remove old password and email user a token to set a new password. Request should have key "email"
arguments:
system_email -- (string) email used to send messages
session -- (Session) object from flask
"""
requestDict = RequestDictionary(self.request)
if (not (requestDict.exists("email"))):
# Don't have the keys we need in request
exc = ResponseException(
"Reset password route requires key 'email'",
StatusCode.CLIENT_ERROR)
return JsonResponse.error(exc, exc.status)
# Get user object
try:
user = self.interfaces.userDb.getUserByEmail(
requestDict.getValue("email"))
except Exception as e:
exc = ResponseException("Unknown Error", StatusCode.CLIENT_ERROR,
ValueError)
return JsonResponse.error(exc, exc.status)
email = requestDict.getValue("email")
LoginSession.logout(session)
self.sendResetPasswordEmail(user, system_email, email)
# Return success message
return JsonResponse.create(StatusCode.OK,
{"message": "Password reset"})
def validate_job(self, request):
""" Gets file for job, validates each row, and sends valid rows to a staging table
Args:
request -- HTTP request containing the jobId
Returns:
Http response object
"""
# Create connection to job tracker database
sess = GlobalDB.db().session
requestDict = RequestDictionary(request)
if requestDict.exists('job_id'):
job_id = requestDict.getValue('job_id')
else:
# Request does not have a job ID, can't validate
validation_error_type = ValidationError.jobError
raise ResponseException('No job ID specified in request',
StatusCode.CLIENT_ERROR, None,
validation_error_type)
# Get the job
job = sess.query(Job).filter_by(job_id=job_id).one_or_none()
if job is None:
validation_error_type = ValidationError.jobError
writeFileError(job_id, None, validation_error_type)
raise ResponseException(
'Job ID {} not found in database'.format(job_id),
StatusCode.CLIENT_ERROR, None, validation_error_type)
# Make sure job's prerequisites are complete
if not run_job_checks(job_id):
validation_error_type = ValidationError.jobError
writeFileError(job_id, None, validation_error_type)
raise ResponseException(
'Prerequisites for Job ID {} are not complete'.format(job_id),
StatusCode.CLIENT_ERROR, None, validation_error_type)
# Make sure this is a validation job
if job.job_type.name in ('csv_record_validation', 'validation'):
job_type_name = job.job_type.name
else:
validation_error_type = ValidationError.jobError
writeFileError(job_id, None, validation_error_type)
raise ResponseException(
'Job ID {} is not a validation job (job type is {})'.format(
job_id, job.job_type.name), StatusCode.CLIENT_ERROR, None,
validation_error_type)
# set job status to running and do validations
mark_job_status(job_id, "running")
if job_type_name == 'csv_record_validation':
self.runValidation(job)
elif job_type_name == 'validation':
self.runCrossValidation(job)
else:
raise ResponseException("Bad job type for validator",
StatusCode.INTERNAL_ERROR)
return JsonResponse.create(StatusCode.OK,
{"message": "Validation complete"})
def finalize(self):
""" Set upload job in job tracker database to finished, allowing dependent jobs to be started
Flask request should include key "upload_id", which holds the job_id for the file_upload job
Returns:
A flask response object, if successful just contains key "success" with value True, otherwise value is False
"""
responseDict = {}
try:
inputDictionary = RequestDictionary(self.request)
jobId = inputDictionary.getValue("upload_id")
# Compare user ID with user who submitted job, if no match return 400
job = self.jobManager.getJobById(jobId)
submission = self.jobManager.getSubmissionForJob(job)
if(submission.user_id != LoginSession.getName(session)):
# This user cannot finalize this job
raise ResponseException("Cannot finalize a job created by a different user", StatusCode.CLIENT_ERROR)
# Change job status to finished
if(self.jobManager.checkUploadType(jobId)):
self.jobManager.changeToFinished(jobId)
responseDict["success"] = True
return JsonResponse.create(StatusCode.OK,responseDict)
else:
raise ResponseException("Wrong job type for finalize route",StatusCode.CLIENT_ERROR)
except ( ValueError , TypeError ) as e:
return JsonResponse.error(e,StatusCode.CLIENT_ERROR)
except ResponseException as e:
return JsonResponse.error(e,e.status)
except Exception as e:
# Unexpected exception, this is a 500 server error
return JsonResponse.error(e,StatusCode.INTERNAL_ERROR)
def setSkipGuide(self, session):
""" Set current user's skip guide parameter """
uid = session["name"]
userDb = self.interfaces.userDb
user = userDb.getUserByUID(uid)
requestDict = RequestDictionary(self.request)
if not requestDict.exists("skip_guide"):
exc = ResponseException("Must include skip_guide parameter",
StatusCode.CLIENT_ERROR)
return JsonResponse.error(exc, exc.status)
skipGuide = requestDict.getValue("skip_guide")
if type(skipGuide) == type(True):
# param is a bool
user.skip_guide = skipGuide
elif type(skipGuide) == type("string"):
# param is a string, allow "true" or "false"
if skipGuide.lower() == "true":
user.skip_guide = True
elif skipGuide.lower() == "false":
user.skip_guide = False
else:
exc = ResponseException("skip_guide must be true or false",
StatusCode.CLIENT_ERROR)
return JsonResponse.error(exc, exc.status)
else:
exc = ResponseException("skip_guide must be a boolean",
StatusCode.CLIENT_ERROR)
return JsonResponse.error(exc, exc.status)
userDb.session.commit()
return JsonResponse.create(StatusCode.OK, {
"message": "skip_guide set successfully",
"skip_guide": skipGuide
})
def checkPasswordToken(self, session):
"""
Checks the password token if its valid
arguments:
session -- (Session) object from flask
return the reponse object with a error code and a message
"""
requestFields = RequestDictionary(self.request)
if (not requestFields.exists("token")):
exc = ResponseException("Request body must include token",
StatusCode.CLIENT_ERROR)
return JsonResponse.error(exc, exc.status)
token = requestFields.getValue("token")
success, message, errorCode = sesEmail.checkToken(
token, self.interfaces.userDb, "password_reset")
if (success):
#mark session that password can be filled out
LoginSession.resetPassword(session)
return JsonResponse.create(StatusCode.OK, {
"email": message,
"errorCode": errorCode,
"message": "success"
})
else:
#failure but alert UI of issue
return JsonResponse.create(StatusCode.OK, {
"errorCode": errorCode,
"message": message
})
def setNewPassword(self, session):
""" Set a new password for a user, request should have keys "user_email" and "password" """
requestDict = RequestDictionary(self.request)
if (not (requestDict.exists("user_email")
and requestDict.exists("password"))):
# Don't have the keys we need in request
exc = ResponseException(
"Set password route requires keys user_email and password",
StatusCode.CLIENT_ERROR)
return JsonResponse.error(exc, exc.status)
if (not self.checkPassword(requestDict.getValue("password"))):
exc = ResponseException("Invalid Password",
StatusCode.CLIENT_ERROR)
return JsonResponse.error(exc, exc.status)
# Get user from email
user = self.interfaces.userDb.getUserByEmail(
requestDict.getValue("user_email"))
# Set new password
self.interfaces.userDb.setPassword(user,
requestDict.getValue("password"),
self.bcrypt)
# Invalidate token
self.interfaces.userDb.deleteToken(session["token"])
session["reset"] = None
# Return success message
return JsonResponse.create(
StatusCode.OK, {"message": "Password successfully changed"})
def getErrorMetrics(self) :
""" Returns an Http response object containing error information for every validation job in specified submission """
responseDict = {}
returnDict = {}
try:
safeDictionary = RequestDictionary(self.request)
submission_id = safeDictionary.getValue("submission_id")
# Check if user has permission to specified submission
self.checkSubmissionPermission(self.jobManager.getSubmissionById(submission_id))
jobIds = self.jobManager.getJobsBySubmission(submission_id)
for currentId in jobIds :
if(self.jobManager.getJobType(currentId) == "csv_record_validation"):
fileName = self.jobManager.getFileType(currentId)
dataList = self.interfaces.errorDb.getErrorMetricsByJobId(currentId)
returnDict[fileName] = dataList
return JsonResponse.create(StatusCode.OK,returnDict)
except ( ValueError , TypeError ) as e:
return JsonResponse.error(e,StatusCode.CLIENT_ERROR)
except ResponseException as e:
return JsonResponse.error(e,e.status)
except Exception as e:
# Unexpected exception, this is a 500 server error
return JsonResponse.error(e,StatusCode.INTERNAL_ERROR)
def getErrorReportURLsForSubmission(self):
"""
Gets the Signed URLs for download based on the submissionId
"""
try :
self.s3manager = s3UrlHandler()
safeDictionary = RequestDictionary(self.request)
submissionId = safeDictionary.getValue("submission_id")
responseDict ={}
for jobId in self.jobManager.getJobsBySubmission(submissionId):
if(self.jobManager.getJobType(jobId) == "csv_record_validation"):
if(not self.isLocal):
responseDict["job_"+str(jobId)+"_error_url"] = self.s3manager.getSignedUrl("errors",self.jobManager.getReportPath(jobId),"GET")
else:
path = os.path.join(self.serverPath, self.jobManager.getReportPath(jobId))
responseDict["job_"+str(jobId)+"_error_url"] = path
if(not self.isLocal):
crossFileReport = self.s3manager.getSignedUrl("errors",self.jobManager.getCrossFileReportPath(submissionId),"GET")
else:
crossFileReport = os.path.join(self.serverPath, self.jobManager.getCrossFileReportPath(submissionId))
responseDict["cross_file_error_url"] = crossFileReport
return JsonResponse.create(StatusCode.OK,responseDict)
except ResponseException as e:
return JsonResponse.error(e,StatusCode.CLIENT_ERROR)
except Exception as e:
# Unexpected exception, this is a 500 server error
return JsonResponse.error(e,StatusCode.INTERNAL_ERROR)
def changeStatus(self, system_email):
"""
Changes status for specified user. Associated request body should have keys 'uid' and 'new_status'
arguments:
system_email -- (string) the emaily to send emails from
return the reponse object with a success message
"""
requestDict = RequestDictionary(self.request)
if (not (requestDict.exists("uid")
and requestDict.exists("new_status"))):
# Missing a required field, return 400
exc = ResponseException(
"Request body must include uid and new_status",
StatusCode.CLIENT_ERROR)
return JsonResponse.error(exc, exc.status)
# Find user that matches specified uid
user = self.interfaces.userDb.getUserByUID(
int(requestDict.getValue("uid")))
if (user.email == None):
return JsonResponse.error(
ResponseException("User does not have a defined email",
StatusCode.INTERNAL_ERROR),
StatusCode.INTERNAL_ERROR)
#check if the user is waiting
if (self.interfaces.userDb.checkStatus(user, "awaiting_approval")):
if (requestDict.getValue("new_status") == "approved"):
# Grant agency_user permission to newly approved users
self.interfaces.userDb.grantPermission(user, "agency_user")
link = AccountHandler.FRONT_END
emailTemplate = {'[URL]': link, '[EMAIL]': system_email}
newEmail = sesEmail(user.email,
system_email,
templateType="account_approved",
parameters=emailTemplate,
database=self.interfaces.userDb)
newEmail.send()
elif (requestDict.getValue("new_status") == "denied"):
emailTemplate = {}
newEmail = sesEmail(user.email,
system_email,
templateType="account_rejected",
parameters=emailTemplate,
database=self.interfaces.userDb)
newEmail.send()
# Change user's status
self.interfaces.userDb.changeStatus(user,
requestDict.getValue("new_status"))
return JsonResponse.create(StatusCode.OK,
{"message": "Status change successful"})
def startGenerationJob(self, submission_id, file_type):
""" Initiates a file generation job
Args:
submission_id: ID of submission to start job for
file_type: Type of file to be generated
Returns:
Tuple of boolean indicating successful start, and error response if False
"""
jobDb = self.interfaces.jobDb
file_type_name = self.fileTypeMap[file_type]
if file_type in ["D1", "D2"]:
# Populate start and end dates, these should be provided in MM/DD/YYYY format, using calendar year (not fiscal year)
requestDict = RequestDictionary(self.request)
start_date = requestDict.getValue("start")
end_date = requestDict.getValue("end")
if not (StringCleaner.isDate(start_date) and StringCleaner.isDate(end_date)):
exc = ResponseException("Start or end date cannot be parsed into a date", StatusCode.CLIENT_ERROR)
return False, JsonResponse.error(exc, exc.status, start = "", end = "", file_type = file_type, status = "failed")
elif file_type not in ["E","F"]:
exc = ResponseException("File type must be either D1, D2, E or F", StatusCode.CLIENT_ERROR)
return False, JsonResponse.error(exc, exc.status, file_type = file_type, status = "failed")
cgac_code = self.jobManager.getSubmissionById(submission_id).cgac_code
# Generate and upload file to S3
user_id = LoginSession.getName(session)
timestamped_name = s3UrlHandler.getTimestampedFilename(CONFIG_BROKER["".join([str(file_type_name),"_file_name"])])
if self.isLocal:
upload_file_name = "".join([CONFIG_BROKER['broker_files'], timestamped_name])
else:
upload_file_name = "".join([str(user_id), "/", timestamped_name])
job = jobDb.getJobBySubmissionFileTypeAndJobType(submission_id, file_type_name, "file_upload")
job.filename = upload_file_name
job.original_filename = timestamped_name
job.job_status_id = jobDb.getJobStatusId("running")
jobDb.session.commit()
if file_type in ["D1", "D2"]:
CloudLogger.log("DEBUG: Adding job info for job id of " + str(job.job_id),
log_type="debug",
file_name=self.debug_file_name)
return self.addJobInfoForDFile(upload_file_name, timestamped_name, submission_id, file_type, file_type_name, start_date, end_date, cgac_code, job)
elif file_type == 'E':
generate_e_file.delay(
submission_id, job.job_id, InterfaceHolder, timestamped_name,
upload_file_name, self.isLocal)
elif file_type == 'F':
generate_f_file.delay(
submission_id, job.job_id, InterfaceHolder, timestamped_name,
upload_file_name, self.isLocal)
return True, None
def getErrorReportURLsForSubmission(self, isWarning = False):
"""
Gets the Signed URLs for download based on the submissionId
"""
try :
self.s3manager = s3UrlHandler()
safeDictionary = RequestDictionary(self.request)
submissionId = safeDictionary.getValue("submission_id")
responseDict ={}
sess = GlobalDB.db().session
for jobId in self.jobManager.getJobsBySubmission(submissionId):
# get the job object here so we can call the refactored getReportPath
# todo: replace other db access functions with job object attributes
job = sess.query(Job).filter(Job.job_id == jobId).one()
if job.job_type.name == 'csv_record_validation':
if isWarning:
reportName = getReportPath(job, 'warning')
key = "job_"+str(jobId)+"_warning_url"
else:
reportName = getReportPath(job, 'error')
key = "job_"+str(jobId)+"_error_url"
if(not self.isLocal):
responseDict[key] = self.s3manager.getSignedUrl("errors",reportName,method="GET")
else:
path = os.path.join(self.serverPath, reportName)
responseDict[key] = path
# For each pair of files, get url for the report
fileTypes = self.interfaces.validationDb.getFileTypeList()
for source in fileTypes:
sourceId = self.interfaces.validationDb.getFileTypeIdByName(source)
for target in fileTypes:
targetId = self.interfaces.validationDb.getFileTypeIdByName(target)
if targetId <= sourceId:
# Skip redundant reports
continue
# Retrieve filename
if isWarning:
reportName = getCrossWarningReportName(submissionId, source, target)
else:
reportName = getCrossReportName(submissionId, source, target)
# If not local, get a signed URL
if self.isLocal:
reportPath = os.path.join(self.serverPath,reportName)
else:
reportPath = self.s3manager.getSignedUrl("errors",reportName,method="GET")
# Assign to key based on source and target
responseDict[self.getCrossReportKey(source,target,isWarning)] = reportPath
return JsonResponse.create(StatusCode.OK,responseDict)
except ResponseException as e:
return JsonResponse.error(e,StatusCode.CLIENT_ERROR)
except Exception as e:
# Unexpected exception, this is a 500 server error
return JsonResponse.error(e,StatusCode.INTERNAL_ERROR)
def deleteUser(self):
""" Deletes user specified by 'email' in request """
requestDict = RequestDictionary(self.request)
if not requestDict.exists("email"):
# missing required fields, return 400
exc = ResponseException(
"Request body must include email of user to be deleted",
StatusCode.CLIENT_ERROR)
return JsonResponse.error(exc, exc.status)
email = requestDict.getValue("email")
self.interfaces.userDb.deleteUser(email)
return JsonResponse.create(StatusCode.OK, {"message": "success"})
def getObligations(self):
input_dictionary = RequestDictionary(self.request)
# Get submission
submission_id = input_dictionary.getValue("submission_id")
submission = self.jobManager.getSubmissionById(submission_id)
# Check that user has access to submission
user = self.checkSubmissionPermission(submission)
obligations_info = obligationStatsForSubmission(submission_id)
return JsonResponse.create(StatusCode.OK, obligations_info)
def getRequestParamsForGenerate(self):
""" Pull information out of request object and return it
Returns: tuple of submission ID and file type
"""
requestDict = RequestDictionary(self.request)
if not (requestDict.exists("submission_id") and requestDict.exists("file_type")):
raise ResponseException("Generate file route requires submission_id and file_type",
StatusCode.CLIENT_ERROR)
submission_id = requestDict.getValue("submission_id")
file_type = requestDict.getValue("file_type")
return submission_id, file_type
def login(self, session):
"""
Logs a user in if their password matches
arguments:
session -- (Session) object from flask
return the response object
"""
try:
sess = GlobalDB.db().session
safe_dictionary = RequestDictionary(self.request)
username = safe_dictionary.get_value('username')
password = safe_dictionary.get_value('password')
try:
user = sess.query(User).filter(
func.lower(User.email) == func.lower(username)).one()
except Exception:
raise ValueError("Invalid username and/or password")
try:
if check_correct_password(user, password, self.bcrypt):
# We have a valid login
return self.create_session_and_response(session, user)
else:
raise ValueError("Invalid username and/or password")
except ValueError as ve:
LoginSession.logout(session)
raise ve
except Exception as e:
LoginSession.logout(session)
raise e
except (TypeError, KeyError, NotImplementedError) as e:
# Return a 400 with appropriate message
return JsonResponse.error(e, StatusCode.CLIENT_ERROR)
except ValueError as e:
# Return a 401 for login denied
return JsonResponse.error(e, StatusCode.LOGIN_REQUIRED)
except Exception as e:
# Return 500
return JsonResponse.error(e, StatusCode.INTERNAL_ERROR)
def getJobID(request):
""" Pull job ID out of request
Args:
request: HTTP request containing the job ID
Returns:
job ID, or raises exception if job ID not found in request
"""
requestDict = RequestDictionary(request)
if (requestDict.exists("job_id")):
jobId = requestDict.getValue("job_id")
return jobId
else:
# Request does not have a job ID, can't validate
raise ResponseException("No job ID specified in request",
StatusCode.CLIENT_ERROR)
def listUsers(self):
""" List all users ordered by status. Associated request body must have key 'filter_by' """
requestDict = RequestDictionary(self.request, optionalRequest=True)
user_status = requestDict.getValue("status") if requestDict.exists(
"status") else "all"
user = self.interfaces.userDb.getUserByUID(
LoginSession.getName(flaskSession))
isAgencyAdmin = self.userManager.hasPermission(
user, "agency_admin") and not self.userManager.hasPermission(
user, "website_admin")
try:
if isAgencyAdmin:
users = self.interfaces.userDb.getUsers(
cgac_code=user.cgac_code, status=user_status)
else:
users = self.interfaces.userDb.getUsers(status=user_status)
except ValueError as e:
# Client provided a bad status
exc = ResponseException(str(e), StatusCode.CLIENT_ERROR,
ValueError)
return JsonResponse.error(exc, exc.status)
userInfo = []
for user in users:
agency_name = self.interfaces.validationDb.getAgencyName(
user.cgac_code)
thisInfo = {
"name":
user.name,
"title":
user.title,
"agency_name":
agency_name,
"cgac_code":
user.cgac_code,
"email":
user.email,
"id":
user.user_id,
"is_active":
user.is_active,
"permissions":
",".join(self.interfaces.userDb.getUserPermissions(user)),
"status":
user.user_status.name
}
userInfo.append(thisInfo)
return JsonResponse.create(StatusCode.OK, {"users": userInfo})
def emailUsers(self, system_email, session):
""" Send email notification to list of users """
requestDict = RequestDictionary(self.request)
if not (requestDict.exists("users")
and requestDict.exists("submission_id")
and requestDict.exists("email_template")):
exc = ResponseException(
"Email users route requires users, email_template, and submission_id",
StatusCode.CLIENT_ERROR)
return JsonResponse.error(exc, exc.status)
uid = session["name"]
current_user = self.interfaces.userDb.getUserByUID(uid)
user_ids = requestDict.getValue("users")
submission_id = requestDict.getValue("submission_id")
# Check if submission id is valid
self.jobManager.getSubmissionById(submission_id)
template_type = requestDict.getValue("email_template")
# Check if email template type is valid
self.userManager.getEmailTemplate(template_type)
users = []
link = "".join(
[AccountHandler.FRONT_END, '#/reviewData/',
str(submission_id)])
emailTemplate = {
'[REV_USER_NAME]': current_user.name,
'[REV_URL]': link
}
for user_id in user_ids:
# Check if user id is valid, if so add User object to array
users.append(self.userManager.getUserByUID(user_id))
for user in users:
newEmail = sesEmail(user.email,
system_email,
templateType=template_type,
parameters=emailTemplate,
database=UserHandler())
newEmail.send()
return JsonResponse.create(StatusCode.OK,
{"message": "Emails successfully sent"})
def login(self, session):
""" Logs a user in if their password matches using local data
Args:
session: the Session object from flask
Returns:
A JsonResponse containing the user information or details on which error occurred, such as whether a
type was wrong, something wasn't implemented, invalid keys were provided, login was denied, or a
different, unexpected error occurred.
"""
try:
sess = GlobalDB.db().session
safe_dictionary = RequestDictionary(self.request)
username = safe_dictionary.get_value('username')
password = safe_dictionary.get_value('password')
try:
user = sess.query(User).filter(
func.lower(User.email) == func.lower(username)).one()
except Exception:
raise ValueError("Invalid username and/or password")
try:
if check_correct_password(user, password, self.bcrypt):
# We have a valid login
return self.create_session_and_response(session, user)
else:
raise ValueError("Invalid username and/or password")
except ValueError as ve:
LoginSession.logout(session)
raise ve
except Exception as e:
LoginSession.logout(session)
raise e
# Catch any specifically raised errors or any other errors that may have happened and return them cleanly
except (TypeError, KeyError, NotImplementedError) as e:
# Return a 400 with appropriate message
return JsonResponse.error(e, StatusCode.CLIENT_ERROR)
except ValueError as e:
# Return a 401 for login denied
return JsonResponse.error(e, StatusCode.LOGIN_REQUIRED)
except Exception as e:
# Return 500
return JsonResponse.error(e, StatusCode.INTERNAL_ERROR)
def checkEmailConfirmationToken(self, session):
"""
Creates user record and email
arguments:
session -- (Session) object from flask
return the reponse object with a error code and a message
"""
requestFields = RequestDictionary(self.request)
if (not requestFields.exists("token")):
exc = ResponseException("Request body must include token",
StatusCode.CLIENT_ERROR)
return JsonResponse.error(exc, exc.status)
token = requestFields.getValue("token")
session["token"] = token
success, message, errorCode = sesEmail.checkToken(
token, self.interfaces.userDb, "validate_email")
if (success):
#mark session that email can be filled out
LoginSession.register(session)
#remove token so it cant be used again
# The following line is commented out for issues with registration email links bouncing users back
# to the original email input page instead of the registration page
#self.interfaces.userDb.deleteToken(token)
#set the status only if current status is awaiting confirmation
user = self.interfaces.userDb.getUserByEmail(message)
if self.interfaces.userDb.checkStatus(user,
"awaiting_confirmation"):
self.interfaces.userDb.changeStatus(user, "email_confirmed")
return JsonResponse.create(StatusCode.OK, {
"email": message,
"errorCode": errorCode,
"message": "success"
})
else:
#failure but alert UI of issue
return JsonResponse.create(StatusCode.OK, {
"errorCode": errorCode,
"message": message
})
def createEmailConfirmation(self, system_email, session):
"""
Creates user record and email
arguments:
system_email -- (string) email used to send messages
session -- (Session) object from flask
"""
requestFields = RequestDictionary(self.request)
if (not requestFields.exists("email")):
exc = ResponseException("Request body must include email",
StatusCode.CLIENT_ERROR)
return JsonResponse.error(exc, exc.status)
email = requestFields.getValue("email")
if (not re.match("[^@]+@[^@]+\.[^@]+", email)):
return JsonResponse.error(ValueError("Invalid Email Format"),
StatusCode.CLIENT_ERROR)
try:
user = self.interfaces.userDb.getUserByEmail(
requestFields.getValue("email"))
except ResponseException as e:
self.interfaces.userDb.addUnconfirmedEmail(email)
else:
if (not (user.user_status_id == self.interfaces.userDb.
getUserStatusId("awaiting_confirmation")
or user.user_status_id == self.interfaces.userDb.
getUserStatusId("email_confirmed"))):
exc = ResponseException("User already registered",
StatusCode.CLIENT_ERROR)
return JsonResponse.error(exc, exc.status)
emailToken = sesEmail.createToken(email, self.interfaces.userDb,
"validate_email")
link = "".join(
[AccountHandler.FRONT_END, '#/registration/', emailToken])
emailTemplate = {'[USER]': email, '[URL]': link}
newEmail = sesEmail(email,
system_email,
templateType="validate_email",
parameters=emailTemplate,
database=self.interfaces.userDb)
newEmail.send()
return JsonResponse.create(StatusCode.OK, {"message": "Email Sent"})
def resetPassword(self, system_email, session):
"""
Remove old password and email user a token to set a new password. Request should have key "email"
arguments:
system_email -- (string) email used to send messages
session -- (Session) object from flask
"""
requestDict = RequestDictionary(self.request)
if (not (requestDict.exists("email"))):
# Don't have the keys we need in request
exc = ResponseException(
"Reset password route requires key 'email'",
StatusCode.CLIENT_ERROR)
return JsonResponse.error(exc, exc.status)
# Get user object
try:
user = self.interfaces.userDb.getUserByEmail(
requestDict.getValue("email"))
except Exception as e:
exc = ResponseException("Unknown Error", StatusCode.CLIENT_ERROR,
ValueError)
return JsonResponse.error(exc, exc.status)
LoginSession.logout(session)
self.interfaces.userDb.session.commit()
email = requestDict.getValue("email")
# Send email with token
emailToken = sesEmail.createToken(email, self.interfaces.userDb,
"password_reset")
link = "".join(
[AccountHandler.FRONT_END, '#/forgotpassword/', emailToken])
emailTemplate = {'[URL]': link}
newEmail = sesEmail(user.email,
system_email,
templateType="reset_password",
parameters=emailTemplate,
database=self.interfaces.userDb)
newEmail.send()
# Return success message
return JsonResponse.create(StatusCode.OK,
{"message": "Password reset"})
def completeGeneration(self, generationId):
""" For files D1 and D2, the API uses this route as a callback to load the generated file.
Requires an 'href' key in the request that specifies the URL of the file to be downloaded
Args:
generationId - Unique key stored in file_generation_task table, used in callback to identify which submission
this file is for.
"""
if generationId is None:
return JsonResponse.error(ResponseException("Must include a generation ID",StatusCode.CLIENT_ERROR), StatusCode.CLIENT_ERROR)
self.smx_log_file_name = "smx_request.log"
# Pull url from request
safeDictionary = RequestDictionary(self.request)
CloudLogger.log("DEBUG: Request content => " + safeDictionary.to_string(), log_type="debug", file_name=self.smx_log_file_name)
if not safeDictionary.exists("href"):
return JsonResponse.error(ResponseException("Request must include href key with URL of D file", StatusCode.CLIENT_ERROR), StatusCode.CLIENT_ERROR)
url = safeDictionary.getValue("href")
CloudLogger.log("DEBUG: Download URL => " + url, log_type="debug", file_name=self.smx_log_file_name)
#Pull information based on task key
try:
CloudLogger.log("DEBUG: Pulling information based on task key...", log_type="debug",
file_name=self.smx_log_file_name)
task = self.interfaces.jobDb.session.query(FileGenerationTask).options(joinedload(FileGenerationTask.file_type)).filter(FileGenerationTask.generation_task_key == generationId).one()
job = self.interfaces.jobDb.getJobById(task.job_id)
CloudLogger.log("DEBUG: Loading D file...", log_type="debug", file_name=self.smx_log_file_name)
result = self.load_d_file(url,job.filename,job.original_filename,job.job_id,self.isLocal)
CloudLogger.log("DEBUG: Load D file result => " + str(result), log_type="debug",
file_name=self.smx_log_file_name)
return JsonResponse.create(StatusCode.OK,{"message":"File loaded successfully"})
except ResponseException as e:
return JsonResponse.error(e, e.status)
except NoResultFound as e:
# Did not find file generation task
return JsonResponse.error(ResponseException("Generation task key not found", StatusCode.CLIENT_ERROR), StatusCode.CLIENT_ERROR)
def listUsersWithStatus(self):
""" List all users with the specified status. Associated request body must have key 'status' """
requestDict = RequestDictionary(self.request)
if (not (requestDict.exists("status"))):
# Missing a required field, return 400
exc = ResponseException("Request body must include status",
StatusCode.CLIENT_ERROR)
return JsonResponse.error(exc, exc.status)
current_user = self.interfaces.userDb.getUserByUID(
flaskSession["name"])
try:
if self.interfaces.userDb.hasPermission(current_user,
"agency_admin"):
users = self.interfaces.userDb.getUsersByStatus(
requestDict.getValue("status"), current_user.cgac_code)
else:
users = self.interfaces.userDb.getUsersByStatus(
requestDict.getValue("status"))
except ValueError as e:
# Client provided a bad status
exc = ResponseException(str(e), StatusCode.CLIENT_ERROR,
ValueError)
return JsonResponse.error(exc, exc.status)
userInfo = []
for user in users:
agency_name = self.interfaces.validationDb.getAgencyName(
user.cgac_code)
thisInfo = {
"name": user.name,
"title": user.title,
"agency_name": agency_name,
"cgac_code": user.cgac_code,
"email": user.email,
"id": user.user_id
}
userInfo.append(thisInfo)
return JsonResponse.create(StatusCode.OK, {"users": userInfo})
def submit(self,name,CreateCredentials):
""" Builds S3 URLs for a set of files and adds all related jobs to job tracker database
Flask request should include keys from FILE_TYPES class variable above
Arguments:
name -- User ID from the session handler
Returns:
Flask response returned will have key_url and key_id for each key in the request
key_url is the S3 URL for uploading
key_id is the job id to be passed to the finalize_submission route
"""
try:
responseDict= {}
fileNameMap = []
safeDictionary = RequestDictionary(self.request)
submissionId = self.jobManager.createSubmission(name, safeDictionary)
existingSubmission = False
if safeDictionary.exists("existing_submission_id"):
existingSubmission = True
# Check if user has permission to specified submission
self.checkSubmissionPermission(self.jobManager.getSubmissionById(submissionId))
for fileType in FileHandler.FILE_TYPES :
# If filetype not included in request, and this is an update to an existing submission, skip it
if not safeDictionary.exists(fileType):
if existingSubmission:
continue
else:
# This is a new submission, all files are required
raise ResponseException("Must include all files for new submission",StatusCode.CLIENT_ERROR)
filename = safeDictionary.getValue(fileType)
if( safeDictionary.exists(fileType)) :
if(not self.isLocal):
uploadName = str(name)+"/"+s3UrlHandler.getTimestampedFilename(filename)
else:
uploadName = filename
responseDict[fileType+"_key"] = uploadName
fileNameMap.append((fileType,uploadName,filename))
fileJobDict = self.jobManager.createJobs(fileNameMap,submissionId,existingSubmission)
for fileType in fileJobDict.keys():
if (not "submission_id" in fileType) :
responseDict[fileType+"_id"] = fileJobDict[fileType]
if(CreateCredentials and not self.isLocal) :
self.s3manager = s3UrlHandler(CONFIG_BROKER["aws_bucket"])
responseDict["credentials"] = self.s3manager.getTemporaryCredentials(name)
else :
responseDict["credentials"] ={"AccessKeyId" : "local","SecretAccessKey" :"local","SessionToken":"local" ,"Expiration" :"local"}
responseDict["submission_id"] = fileJobDict["submission_id"]
if self.isLocal:
responseDict["bucket_name"] = CONFIG_BROKER["broker_files"]
else:
responseDict["bucket_name"] = CONFIG_BROKER["aws_bucket"]
return JsonResponse.create(StatusCode.OK,responseDict)
except (ValueError , TypeError, NotImplementedError) as e:
return JsonResponse.error(e,StatusCode.CLIENT_ERROR)
except Exception as e:
# Unexpected exception, this is a 500 server error
return JsonResponse.error(e,StatusCode.INTERNAL_ERROR)
except:
return JsonResponse.error(Exception("Failed to catch exception"),StatusCode.INTERNAL_ERROR)
def getStatus(self):
""" Get description and status of all jobs in the submission specified in request object
Returns:
A flask response object to be sent back to client, holds a JSON where each job ID has a dictionary holding file_type, job_type, status, and filename
"""
try:
inputDictionary = RequestDictionary(self.request)
# Get submission
submissionId = inputDictionary.getValue("submission_id")
submission = self.jobManager.getSubmissionById(submissionId)
# Check that user has access to submission
user = self.checkSubmissionPermission(submission)
# Get jobs in this submission
jobs = self.jobManager.getJobsBySubmission(submissionId)
# Build dictionary of submission info with info about each job
submissionInfo = {}
submissionInfo["jobs"] = []
submissionInfo["agency_name"] = submission.agency_name
submissionInfo["reporting_period_start_date"] = submission.reporting_start_date.strftime("%m/%d/%Y")
submissionInfo["reporting_period_end_date"] = submission.reporting_end_date.strftime("%m/%d/%Y")
submissionInfo["created_on"] = self.interfaces.jobDb.getFormattedDatetimeBySubmissionId(submissionId)
# Include number of errors in submission
submissionInfo["number_of_errors"] = self.interfaces.errorDb.sumNumberOfErrorsForJobList(jobs)
submissionInfo["number_of_rows"] = self.interfaces.jobDb.sumNumberOfRowsForJobList(jobs)
for jobId in jobs:
jobInfo = {}
if(self.jobManager.getJobType(jobId) != "csv_record_validation"):
continue
jobInfo["job_id"] = jobId
jobInfo["job_status"] = self.jobManager.getJobStatus(jobId)
jobInfo["job_type"] = self.jobManager.getJobType(jobId)
jobInfo["filename"] = self.jobManager.getOriginalFilenameById(jobId)
try:
jobInfo["file_status"] = self.interfaces.errorDb.getStatusLabelByJobId(jobId)
except ResponseException as e:
# Job ID not in error database, probably did not make it to validation, or has not yet been validated
jobInfo["file_status"] = ""
jobInfo["missing_headers"] = []
jobInfo["duplicated_headers"] = []
jobInfo["error_type"] = ""
jobInfo["error_data"] = []
else:
# If job ID was found in file_status, we should be able to get header error lists and file data
# Get string of missing headers and parse as a list
missingHeaderString = self.interfaces.errorDb.getMissingHeadersByJobId(jobId)
if missingHeaderString is not None:
# Split header string into list, excluding empty strings
jobInfo["missing_headers"] = [n.strip() for n in missingHeaderString.split(",") if len(n) > 0]
else:
jobInfo["missing_headers"] = []
# Get string of duplicated headers and parse as a list
duplicatedHeaderString = self.interfaces.errorDb.getDuplicatedHeadersByJobId(jobId)
if duplicatedHeaderString is not None:
# Split header string into list, excluding empty strings
jobInfo["duplicated_headers"] = [n.strip() for n in duplicatedHeaderString.split(",") if len(n) > 0]
else:
jobInfo["duplicated_headers"] = []
jobInfo["error_type"] = self.interfaces.errorDb.getErrorType(jobId)
jobInfo["error_data"] = self.interfaces.errorDb.getErrorMetricsByJobId(jobId)
# File size and number of rows not dependent on error DB
# Get file size
jobInfo["file_size"] = self.jobManager.getFileSizeById(jobId)
# Get number of rows in file
jobInfo["number_of_rows"] = self.jobManager.getNumberOfRowsById(jobId)
try :
jobInfo["file_type"] = self.jobManager.getFileType(jobId)
except Exception as e:
jobInfo["file_type"] = ''
submissionInfo["jobs"].append(jobInfo)
# Build response object holding dictionary
return JsonResponse.create(StatusCode.OK,submissionInfo)
except ResponseException as e:
return JsonResponse.error(e,e.status)
except Exception as e:
# Unexpected exception, this is a 500 server error
return JsonResponse.error(e,StatusCode.INTERNAL_ERROR)
def max_login(self, session):
""" Logs a user in if their password matches using MAX
Args:
session: Session object from flask
Returns:
A JsonResponse containing the user information or details on which error occurred, such as whether a
type was wrong, something wasn't implemented, invalid keys were provided, login was denied, or a
different, unexpected error occurred.
"""
try:
safe_dictionary = RequestDictionary(self.request)
ticket = safe_dictionary.get_value("ticket")
service = safe_dictionary.get_value('service')
# Call MAX's serviceValidate endpoint and retrieve the response
max_dict = get_max_dict(ticket, service)
if 'cas:authenticationSuccess' not in max_dict[
'cas:serviceResponse']:
raise ValueError(
"The Max CAS endpoint was unable to locate your session "
"using the ticket/service combination you provided.")
cas_attrs = max_dict['cas:serviceResponse'][
'cas:authenticationSuccess']['cas:attributes']
# Grab MAX ID to see if a service account is being logged in
max_id_components = cas_attrs['maxAttribute:MAX-ID'].split('_')
service_account_flag = (len(max_id_components) > 1
and max_id_components[0].lower() == 's')
# Grab the email and list of groups from MAX's response
email = cas_attrs['maxAttribute:Email-Address']
try:
sess = GlobalDB.db().session
user = sess.query(User).filter(
func.lower(User.email) == func.lower(email)).one_or_none()
# If the user does not exist, create them since they are allowed to access the site because they got
# past the above group membership checks
if user is None:
user = User()
user.email = email
set_user_name(user, cas_attrs)
set_max_perms(user, cas_attrs['maxAttribute:GroupList'],
service_account_flag)
sess.add(user)
sess.commit()
except MultipleResultsFound:
raise ValueError("An error occurred during login.")
return self.create_session_and_response(session, user)
# Catch any specifically raised errors or any other errors that may have happened and return them cleanly.
# We add the error parameter here because this endpoint needs to provide better feedback, and to avoid changing
# the default behavior of the JsonResponse class globally.
except (TypeError, KeyError, NotImplementedError) as e:
# Return a 400 with appropriate message
return JsonResponse.error(e, StatusCode.CLIENT_ERROR, error=str(e))
except ValueError as e:
# Return a 401 for login denied
return JsonResponse.error(e,
StatusCode.LOGIN_REQUIRED,
error=str(e))
except Exception as e:
# Return 500
return JsonResponse.error(e,
StatusCode.INTERNAL_ERROR,
error=str(e))
def validateJob(self, request, interfaces):
""" Gets file for job, validates each row, and sends valid rows to staging database
Args:
request -- HTTP request containing the jobId
interfaces -- InterfaceHolder object to the databases
Returns:
Http response object
"""
# Create connection to job tracker database
self.filename = None
tableName = ""
jobId = None
jobTracker = None
try:
jobTracker = interfaces.jobDb
requestDict = RequestDictionary(request)
tableName = ""
if (requestDict.exists("job_id")):
jobId = requestDict.getValue("job_id")
else:
# Request does not have a job ID, can't validate
raise ResponseException("No job ID specified in request",
StatusCode.CLIENT_ERROR)
# Check that job exists and is ready
if (not (jobTracker.runChecks(jobId))):
raise ResponseException("Checks failed on Job ID",
StatusCode.CLIENT_ERROR)
tableName = interfaces.stagingDb.getTableName(jobId)
jobType = interfaces.jobDb.checkJobType(jobId)
except ResponseException as e:
CloudLogger.logError(str(e), e,
traceback.extract_tb(sys.exc_info()[2]))
if (e.errorType == None):
# Error occurred while trying to get and check job ID
e.errorType = ValidationError.jobError
interfaces.errorDb.writeFileError(jobId, self.filename,
e.errorType, e.extraInfo)
return JsonResponse.error(e, e.status, table=tableName)
except Exception as e:
exc = ResponseException(str(e), StatusCode.INTERNAL_ERROR, type(e))
CloudLogger.logError(str(e), e,
traceback.extract_tb(sys.exc_info()[2]))
self.markJob(jobId, jobTracker, "failed", interfaces.errorDb,
self.filename, ValidationError.unknownError)
return JsonResponse.error(exc, exc.status, table=tableName)
try:
jobTracker.markJobStatus(jobId, "running")
if jobType == interfaces.jobDb.getJobTypeId(
"csv_record_validation"):
self.runValidation(jobId, interfaces)
elif jobType == interfaces.jobDb.getJobTypeId("validation"):
self.runCrossValidation(jobId, interfaces)
else:
raise ResponseException("Bad job type for validator",
StatusCode.INTERNAL_ERROR)
interfaces.errorDb.markFileComplete(jobId, self.filename)
return JsonResponse.create(StatusCode.OK, {"table": tableName})
except ResponseException as e:
CloudLogger.logError(str(e), e,
traceback.extract_tb(sys.exc_info()[2]))
self.markJob(jobId, jobTracker, "invalid", interfaces.errorDb,
self.filename, e.errorType, e.extraInfo)
return JsonResponse.error(e, e.status, table=tableName)
except ValueError as e:
CloudLogger.logError(str(e), e,
traceback.extract_tb(sys.exc_info()[2]))
# Problem with CSV headers
exc = ResponseException(
str(e), StatusCode.CLIENT_ERROR, type(e),
ValidationError.unknownError) #"Internal value error"
self.markJob(jobId, jobTracker, "invalid", interfaces.errorDb,
self.filename, ValidationError.unknownError)
return JsonResponse.error(exc, exc.status, table=tableName)
except Error as e:
CloudLogger.logError(str(e), e,
traceback.extract_tb(sys.exc_info()[2]))
# CSV file not properly formatted (usually too much in one field)
exc = ResponseException("Internal error", StatusCode.CLIENT_ERROR,
type(e), ValidationError.unknownError)
self.markJob(jobId, jobTracker, "invalid", interfaces.errorDb,
self.filename, ValidationError.unknownError)
return JsonResponse.error(exc, exc.status, table=tableName)
except Exception as e:
CloudLogger.logError(str(e), e,
traceback.extract_tb(sys.exc_info()[2]))
exc = ResponseException(str(e), StatusCode.INTERNAL_ERROR, type(e),
ValidationError.unknownError)
self.markJob(jobId, jobTracker, "failed", interfaces.errorDb,
self.filename, ValidationError.unknownError)
return JsonResponse.error(exc, exc.status, table=tableName)
def login(self,session):
"""
Logs a user in if their password matches
arguments:
session -- (Session) object from flask
return the response object
"""
try:
sess = GlobalDB.db().session
safe_dictionary = RequestDictionary(self.request)
username = safe_dictionary.getValue('username')
password = safe_dictionary.getValue('password')
try:
user = sess.query(User).filter(func.lower(User.email) == func.lower(username)).one()
except Exception:
raise ValueError("Invalid username and/or password")
if user.user_status_id != USER_STATUS_DICT["approved"]:
raise ValueError("Invalid username and/or password")
# Only check if user is active after they've logged in for the first time
if user.last_login_date is not None and self.isAccountExpired(user):
raise ValueError("Your account has expired. Please contact an administrator.")
# for whatever reason, your account is not active, therefore it's locked
if not user.is_active:
raise ValueError("Your account has been locked. Please contact an administrator.")
try:
if check_correct_password(user,password,self.bcrypt):
# We have a valid login
# Reset incorrect password attempt count to 0
self.reset_password_count(user)
return self.create_session_and_response(session, user)
else :
# increase incorrect password attempt count by 1
# if this is the 3rd incorrect attempt, lock account
self.incrementPasswordCount(user)
if user.incorrect_password_attempts == 3:
raise ValueError("Your account has been locked due to too many failed login attempts. Please contact an administrator.")
raise ValueError("Invalid username and/or password")
except ValueError as ve:
LoginSession.logout(session)
raise ve
except Exception as e:
LoginSession.logout(session)
raise e
except (TypeError, KeyError, NotImplementedError) as e:
# Return a 400 with appropriate message
return JsonResponse.error(e,StatusCode.CLIENT_ERROR)
except ValueError as e:
# Return a 401 for login denied
return JsonResponse.error(e,StatusCode.LOGIN_REQUIRED)
except Exception as e:
# Return 500
return JsonResponse.error(e,StatusCode.INTERNAL_ERROR)
