def find_existing_submissions_in_period(cgac_code, frec_code, reporting_fiscal_year, reporting_fiscal_period,
submission_id=None):
""" Find all the submissions in the given period for the given CGAC or FREC code
Args:
cgac_code: the CGAC code to check against or None if checking a FREC agency
frec_code: the FREC code to check against or None if checking a CGAC agency
reporting_fiscal_year: the year to check for
reporting_fiscal_period: the period in the year to check for
submission_id: the submission ID to check against (used when checking if this submission is being
re-certified)
Returns:
A JsonResponse containing a success message to indicate there are no existing submissions in the given
period or the error if there was one
"""
# We need either a cgac or a frec code for this function
if not cgac_code and not frec_code:
return JsonResponse.error(ValueError("CGAC or FR Entity Code required"), StatusCode.CLIENT_ERROR)
submission_query = get_existing_submission_list(cgac_code, frec_code, reporting_fiscal_year,
reporting_fiscal_period, submission_id)
if submission_query.count() > 0:
data = {
"message": "A submission with the same period already exists.",
"submissionId": submission_query[0].submission_id
}
return JsonResponse.create(StatusCode.CLIENT_ERROR, data)
return JsonResponse.create(StatusCode.OK, {"message": "Success"})
def set_skip_guide(self):
""" Set current user's skip guide parameter
Returns:
JsonResponse object containing results of setting the skip guide or details of the error that occurred.
Possible errors include the request not containing a skip_guide parameter or it not being a boolean
value
"""
sess = GlobalDB.db().session
request_dict = RequestDictionary.derive(self.request)
try:
if 'skip_guide' not in request_dict:
raise ResponseException(
"Must include skip_guide parameter",
StatusCode.CLIENT_ERROR
)
skip_guide = str(request_dict['skip_guide']).lower()
if skip_guide not in ("true", "false"):
raise ResponseException(
"skip_guide must be true or false",
StatusCode.CLIENT_ERROR
)
g.user.skip_guide = skip_guide == "true"
except ResponseException as exc:
return JsonResponse.error(exc, exc.status)
sess.commit()
return JsonResponse.create(StatusCode.OK, {"message": "skip_guide set successfully", "skip_guide": skip_guide})
def delete_submission(submission):
""" Deletes all data associated with the specified submission
NOTE: THERE IS NO WAY TO UNDO THIS """
if submission.publish_status_id != PUBLISH_STATUS_DICT['unpublished']:
return JsonResponse.error(ValueError("Submissions that have been certified cannot be deleted"),
StatusCode.CLIENT_ERROR)
sess = GlobalDB.db().session
# Check if the submission has any jobs that are currently running, if so, do not allow deletion
jobs = sess.query(Job).filter(Job.submission_id == submission.submission_id,
Job.job_status_id == JOB_STATUS_DICT['running']).all()
if jobs:
return JsonResponse.error(ValueError("Submissions with running jobs cannot be deleted"),
StatusCode.CLIENT_ERROR)
sess.query(SubmissionSubTierAffiliation).filter(
SubmissionSubTierAffiliation.submission_id == submission.submission_id).delete(
synchronize_session=False)
sess.query(Submission).filter(Submission.submission_id == submission.submission_id).delete(
synchronize_session=False)
sess.expire_all()
return JsonResponse.create(StatusCode.OK, {"message": "Success"})
def resetPassword(self,system_email,session):
"""
Remove old password and email user a token to set a new password. Request should have key "email"
arguments:
system_email -- (string) email used to send messages
session -- (Session) object from flask
"""
requestDict = RequestDictionary(self.request)
if(not (requestDict.exists("email"))):
# Don't have the keys we need in request
exc = ResponseException("Reset password route requires key 'email'",StatusCode.CLIENT_ERROR)
return JsonResponse.error(exc,exc.status)
# Get user object
try:
user = self.interfaces.userDb.getUserByEmail(requestDict.getValue("email"))
except Exception as e:
exc = ResponseException("Unknown Error",StatusCode.CLIENT_ERROR,ValueError)
return JsonResponse.error(exc,exc.status)
email = requestDict.getValue("email")
LoginSession.logout(session)
self.sendResetPasswordEmail(user, system_email, email)
# Return success message
return JsonResponse.create(StatusCode.OK,{"message":"Password reset"})
def listUsersWithStatus(self):
""" List all users with the specified status. Associated request body must have key 'status' """
requestDict = RequestDictionary(self.request)
if(not (requestDict.exists("status"))):
# Missing a required field, return 400
exc = ResponseException("Request body must include status", StatusCode.CLIENT_ERROR)
return JsonResponse.error(exc,exc.status)
current_user = self.interfaces.userDb.getUserByUID(flaskSession["name"])
try:
if self.interfaces.userDb.hasPermission(current_user, "agency_admin"):
users = self.interfaces.userDb.getUsersByStatus(requestDict.getValue("status"), current_user.cgac_code)
else:
users = self.interfaces.userDb.getUsersByStatus(requestDict.getValue("status"))
except ValueError as e:
# Client provided a bad status
exc = ResponseException(str(e),StatusCode.CLIENT_ERROR,ValueError)
return JsonResponse.error(exc,exc.status)
userInfo = []
for user in users:
agency_name = self.interfaces.validationDb.getAgencyName(user.cgac_code)
thisInfo = {"name":user.name, "title":user.title, "agency_name":agency_name, "cgac_code":user.cgac_code,
"email":user.email, "id":user.user_id }
userInfo.append(thisInfo)
return JsonResponse.create(StatusCode.OK,{"users":userInfo})
def run_instance_function(accountManager,
accountFunction,
getSystemEmail=False,
getSession=False,
getUser=False,
getCredentials=False):
""" Standard error handling around each route """
interfaces = InterfaceHolder()
try:
accountManager.addInterfaces(interfaces)
if (getSystemEmail and getSession):
return accountFunction(RouteUtils.SYSTEM_EMAIL, session)
elif (getSystemEmail):
return accountFunction(RouteUtils.SYSTEM_EMAIL)
elif (getSession):
return accountFunction(session)
elif (getUser):
if (getCredentials):
return accountFunction(LoginSession.getName(session),
RouteUtils.CREATE_CREDENTIALS)
else:
# Currently no functions with user but not credentials flag
raise ValueError(
"Invalid combination of flags to run_instance_function"
)
else:
return accountFunction()
except ResponseException as e:
return JsonResponse.error(e, e.status)
except Exception as e:
exc = ResponseException(str(e), StatusCode.INTERNAL_ERROR, type(e))
return JsonResponse.error(exc, exc.status)
finally:
interfaces.close()
def test_max_login_success(monkeypatch):
ah = accountHandler.AccountHandler(Mock())
mock_dict = Mock()
mock_dict.return_value.safeDictionary.side_effect = {'ticket': '', 'service': ''}
monkeypatch.setattr(accountHandler, 'RequestDictionary', mock_dict)
max_dict = {'cas:serviceResponse': {}}
monkeypatch.setattr(accountHandler, 'get_max_dict', Mock(return_value=max_dict))
config = {'parent_group': 'parent-group'}
monkeypatch.setattr(accountHandler, 'CONFIG_BROKER', config)
max_dict = make_max_dict('parent-group,parent-group-CGAC_SYS')
monkeypatch.setattr(accountHandler, 'get_max_dict', Mock(return_value=max_dict))
# If it gets to this point, that means the user was in all the right groups aka successful login
monkeypatch.setattr(ah, 'create_session_and_response',
Mock(return_value=JsonResponse.create(StatusCode.OK, {"message": "Login successful"})))
json_response = ah.max_login(Mock())
assert "Login successful" == json.loads(json_response.get_data().decode("utf-8"))['message']
max_dict = make_max_dict('')
monkeypatch.setattr(accountHandler, 'get_max_dict', Mock(return_value=max_dict))
# If it gets to this point, that means the user was in all the right groups aka successful login
monkeypatch.setattr(ah, 'create_session_and_response',
Mock(return_value=JsonResponse.create(StatusCode.OK, {"message": "Login successful"})))
json_response = ah.max_login(Mock())
assert "Login successful" == json.loads(json_response.get_data().decode("utf-8"))['message']
def setSkipGuide(self, session):
""" Set current user's skip guide parameter """
uid = session["name"]
userDb = self.interfaces.userDb
user = userDb.getUserByUID(uid)
requestDict = RequestDictionary(self.request)
if not requestDict.exists("skip_guide"):
exc = ResponseException("Must include skip_guide parameter", StatusCode.CLIENT_ERROR)
return JsonResponse.error(exc, exc.status)
skipGuide = requestDict.getValue("skip_guide")
if type(skipGuide) == type(True):
# param is a bool
user.skip_guide = skipGuide
elif type(skipGuide) == type("string"):
# param is a string, allow "true" or "false"
if skipGuide.lower() == "true":
user.skip_guide = True
elif skipGuide.lower() == "false":
user.skip_guide = False
else:
exc = ResponseException("skip_guide must be true or false", StatusCode.CLIENT_ERROR)
return JsonResponse.error(exc, exc.status)
else:
exc = ResponseException("skip_guide must be a boolean", StatusCode.CLIENT_ERROR)
return JsonResponse.error(exc, exc.status)
userDb.session.commit()
return JsonResponse.create(StatusCode.OK,{"message":"skip_guide set successfully","skip_guide":skipGuide})
def checkPasswordToken(self,session):
"""
Checks the password token if its valid
arguments:
session -- (Session) object from flask
return the reponse object with a error code and a message
"""
request_fields = RequestDictionary.derive(self.request)
try:
if 'token' not in request_fields:
raise ResponseException(
"Request body must include token", StatusCode.CLIENT_ERROR)
except ResponseException as exc:
return JsonResponse.error(exc, exc.status)
token = request_fields['token']
# Save token to be deleted after reset
session["token"] = token
success, message, errorCode = sesEmail.check_token(token, "password_reset")
if success:
#mark session that password can be filled out
LoginSession.reset_password(session)
return JsonResponse.create(StatusCode.OK,{"email":message,"errorCode":errorCode,"message":"success"})
else:
#failure but alert UI of issue
return JsonResponse.create(StatusCode.OK,{"errorCode":errorCode,"message":message})
def email_users(self, system_email):
""" Send email notification to list of users
Args:
system_email: the address of the system to send the email from
Returns:
A JsonReponse containing a message that the email sent successfully or the details of the missing
parameters
"""
sess = GlobalDB.db().session
request_dict = RequestDictionary.derive(self.request)
required = ('users', 'submission_id', 'email_template')
try:
if any(field not in request_dict for field in required):
raise ResponseException(
"Email users route requires users, email_template, and submission_id",
StatusCode.CLIENT_ERROR)
except ResponseException as exc:
return JsonResponse.error(exc, exc.status)
user_ids = request_dict['users']
submission_id = request_dict['submission_id']
# Check if submission id is valid
_, agency_name = sess.query(Submission.submission_id, CGAC.agency_name).\
join(CGAC, Submission.cgac_code == CGAC.cgac_code).filter(Submission.submission_id == submission_id).one()
if not agency_name:
_, agency_name = sess.query(Submission.submission_id, FREC.agency_name).\
join(FREC, Submission.frec_code == FREC.frec_code).\
filter(Submission.submission_id == submission_id).one()
template_type = request_dict['email_template']
# Check if email template type is valid
get_email_template(template_type)
users = []
link = "".join(
[AccountHandler.FRONT_END, '#/reviewData/',
str(submission_id)])
email_template = {
'[REV_USER_NAME]': g.user.name,
'[REV_AGENCY]': agency_name,
'[REV_URL]': link
}
for user_id in user_ids:
# Check if user id is valid, if so add User object to array
users.append(
sess.query(User).filter(User.user_id == user_id).one())
for user in users:
new_email = SesEmail(user.email,
system_email,
template_type=template_type,
parameters=email_template)
new_email.send()
return JsonResponse.create(StatusCode.OK,
{"message": "Emails successfully sent"})
def reset_password(self,system_email,session):
"""
Remove old password and email user a token to set a new password. Request should have key "email"
arguments:
system_email -- (string) email used to send messages
session -- (Session) object from flask
"""
sess = GlobalDB.db().session
request_dict = RequestDictionary.derive(self.request)
try:
if 'email' not in request_dict:
# Don't have the keys we need in request
raise ResponseException(
"Reset password route requires key 'email'",
StatusCode.CLIENT_ERROR
)
user = sess.query(User).filter(
func.lower(User.email) == func.lower(request_dict['email'])
).one()
except Exception as exc:
return JsonResponse.error(exc, StatusCode.CLIENT_ERROR)
email = request_dict['email']
LoginSession.logout(session)
self.send_reset_password_email(user, system_email, email)
# Return success message
return JsonResponse.create(StatusCode.OK,{"message":"Password reset"})
def get_submission_data(submission, file_type=''):
""" Get data for the submission specified
Args:
submission: submission to retrieve metadata for
file_type: the type of job to retrieve metadata for
Returns:
JsonResponse containing the error information or the object containing metadata for all relevant file types
"""
sess = GlobalDB.db().session
file_type = file_type.lower()
# Make sure the file type provided is valid
if file_type and file_type not in FILE_TYPE_DICT and file_type != 'cross':
return JsonResponse.error(ValueError(file_type + ' is not a valid file type'), StatusCode.CLIENT_ERROR)
# Make sure the file type provided is valid for the submission type
is_fabs = submission.d2_submission
if file_type and (is_fabs and file_type != 'fabs') or (not is_fabs and file_type == 'fabs'):
return JsonResponse.error(ValueError(file_type + ' is not a valid file type for this submission'),
StatusCode.CLIENT_ERROR)
job_query = sess.query(Job).filter(Job.submission_id == submission.submission_id)
if not file_type:
relevant_job_types = (JOB_TYPE_DICT['csv_record_validation'], JOB_TYPE_DICT['validation'])
job_query = job_query.filter(Job.job_type_id.in_(relevant_job_types))
elif file_type == 'cross':
job_query = job_query.filter(Job.job_type_id == JOB_TYPE_DICT['validation'])
else:
job_query = job_query.filter(Job.file_type_id == FILE_TYPE_DICT[file_type])
job_dict = {'jobs': [job_to_dict(job) for job in job_query]}
return JsonResponse.create(StatusCode.OK, job_dict)
def setSkipGuide(self, session):
""" Set current user's skip guide parameter """
uid = session["name"]
userDb = self.interfaces.userDb
user = userDb.getUserByUID(uid)
requestDict = RequestDictionary(self.request)
if not requestDict.exists("skip_guide"):
exc = ResponseException("Must include skip_guide parameter",
StatusCode.CLIENT_ERROR)
return JsonResponse.error(exc, exc.status)
skipGuide = requestDict.getValue("skip_guide")
if type(skipGuide) == type(True):
# param is a bool
user.skip_guide = skipGuide
elif type(skipGuide) == type("string"):
# param is a string, allow "true" or "false"
if skipGuide.lower() == "true":
user.skip_guide = True
elif skipGuide.lower() == "false":
user.skip_guide = False
else:
exc = ResponseException("skip_guide must be true or false",
StatusCode.CLIENT_ERROR)
return JsonResponse.error(exc, exc.status)
else:
exc = ResponseException("skip_guide must be a boolean",
StatusCode.CLIENT_ERROR)
return JsonResponse.error(exc, exc.status)
userDb.session.commit()
return JsonResponse.create(StatusCode.OK, {
"message": "skip_guide set successfully",
"skip_guide": skipGuide
})
def run_instance_function(
accountManager, accountFunction, getSystemEmail=False, getSession=False, getUser=False, getCredentials=False
):
""" Standard error handling around each route """
interfaces = InterfaceHolder()
try:
accountManager.addInterfaces(interfaces)
if getSystemEmail and getSession:
return accountFunction(RouteUtils.SYSTEM_EMAIL, session)
elif getSystemEmail:
return accountFunction(RouteUtils.SYSTEM_EMAIL)
elif getSession:
return accountFunction(session)
elif getUser:
if getCredentials:
return accountFunction(LoginSession.getName(session), RouteUtils.CREATE_CREDENTIALS)
else:
# Currently no functions with user but not credentials flag
raise ValueError("Invalid combination of flags to run_instance_function")
else:
return accountFunction()
except ResponseException as e:
return JsonResponse.error(e, e.status)
except Exception as e:
exc = ResponseException(str(e), StatusCode.INTERNAL_ERROR, type(e))
return JsonResponse.error(exc, exc.status)
finally:
interfaces.close()
def listUsersWithStatus(self):
""" List all users with the specified status. Associated request body must have key 'status' """
requestDict = RequestDictionary(self.request)
if (not (requestDict.exists("status"))):
# Missing a required field, return 400
exc = ResponseException("Request body must include status",
StatusCode.CLIENT_ERROR)
return JsonResponse.error(exc, exc.status)
try:
users = self.interfaces.userDb.getUsersByStatus(
requestDict.getValue("status"))
except ValueError as e:
# Client provided a bad status
exc = ResponseException(str(e), StatusCode.CLIENT_ERROR,
ValueError)
return JsonResponse.error(exc, exc.status)
userInfo = []
for user in users:
thisInfo = {
"name": user.name,
"title": user.title,
"agency": user.agency,
"email": user.email,
"id": user.user_id
}
userInfo.append(thisInfo)
return JsonResponse.create(StatusCode.OK, {"users": userInfo})
def checkPasswordToken(self, session):
"""
Checks the password token if its valid
arguments:
session -- (Session) object from flask
return the reponse object with a error code and a message
"""
requestFields = RequestDictionary(self.request)
if (not requestFields.exists("token")):
exc = ResponseException("Request body must include token",
StatusCode.CLIENT_ERROR)
return JsonResponse.error(exc, exc.status)
token = requestFields.getValue("token")
success, message, errorCode = sesEmail.checkToken(
token, self.interfaces.userDb, "password_reset")
if (success):
#mark session that password can be filled out
LoginSession.resetPassword(session)
return JsonResponse.create(StatusCode.OK, {
"email": message,
"errorCode": errorCode,
"message": "success"
})
else:
#failure but alert UI of issue
return JsonResponse.create(StatusCode.OK, {
"errorCode": errorCode,
"message": message
})
def getErrorReportURLsForSubmission(self):
"""
Gets the Signed URLs for download based on the submissionId
"""
try :
self.s3manager = s3UrlHandler()
safeDictionary = RequestDictionary(self.request)
submissionId = safeDictionary.getValue("submission_id")
responseDict ={}
for jobId in self.jobManager.getJobsBySubmission(submissionId):
if(self.jobManager.getJobType(jobId) == "csv_record_validation"):
if(not self.isLocal):
responseDict["job_"+str(jobId)+"_error_url"] = self.s3manager.getSignedUrl("errors",self.jobManager.getReportPath(jobId),"GET")
else:
path = os.path.join(self.serverPath, self.jobManager.getReportPath(jobId))
responseDict["job_"+str(jobId)+"_error_url"] = path
if(not self.isLocal):
crossFileReport = self.s3manager.getSignedUrl("errors",self.jobManager.getCrossFileReportPath(submissionId),"GET")
else:
crossFileReport = os.path.join(self.serverPath, self.jobManager.getCrossFileReportPath(submissionId))
responseDict["cross_file_error_url"] = crossFileReport
return JsonResponse.create(StatusCode.OK,responseDict)
except ResponseException as e:
return JsonResponse.error(e,StatusCode.CLIENT_ERROR)
except Exception as e:
# Unexpected exception, this is a 500 server error
return JsonResponse.error(e,StatusCode.INTERNAL_ERROR)
def list_users_with_status(self):
""" List all users with the specified status. Associated request body must have key 'status' """
request_dict = RequestDictionary.derive(self.request)
try:
if 'status' not in request_dict:
# Missing a required field, return 400
raise ResponseException(
"Request body must include status", StatusCode.CLIENT_ERROR)
except ResponseException as exc:
return JsonResponse.error(exc, exc.status)
sess = GlobalDB.db().session
try:
users = sess.query(User).filter_by(
user_status_id=USER_STATUS_DICT[request_dict['status']]
).all()
except ValueError as exc:
# Client provided a bad status
return JsonResponse.error(exc, StatusCode.CLIENT_ERROR)
user_info = []
for user in users:
agency_name = sess.query(CGAC.agency_name).\
filter(CGAC.cgac_code == user.cgac_code).\
one_or_none()
this_info = {"name":user.name, "title":user.title, "agency_name":agency_name, "cgac_code":user.cgac_code,
"email":user.email, "id":user.user_id }
user_info.append(this_info)
return JsonResponse.create(StatusCode.OK,{"users":user_info})
def checkPasswordToken(self,session):
"""
Checks the password token if its valid
arguments:
session -- (Session) object from flask
return the reponse object with a error code and a message
"""
requestFields = RequestDictionary(self.request)
if(not requestFields.exists("token")):
exc = ResponseException("Request body must include token", StatusCode.CLIENT_ERROR)
return JsonResponse.error(exc,exc.status)
token = requestFields.getValue("token")
success,message,errorCode = sesEmail.checkToken(token,self.interfaces.userDb,"password_reset")
if(success):
#mark session that password can be filled out
LoginSession.resetPassword(session)
return JsonResponse.create(StatusCode.OK,{"email":message,"errorCode":errorCode,"message":"success"})
else:
#failure but alert UI of issue
return JsonResponse.create(StatusCode.OK,{"errorCode":errorCode,"message":message})
def markJob(jobId,
jobTracker,
status,
errorDb,
filename=None,
fileError=ValidationError.unknownError,
extraInfo=None):
""" Update status of a job in job tracker database
Args:
jobId: Job to be updated
jobTracker: Interface object for job tracker
status: New status for specified job
errorDb: Interface object for error database
filename: Filename of file to be validated
fileError: Type of error that occurred if this is an invalid or failed status
extraInfo: Dict of extra fields to attach to exception
"""
try:
if filename != None and (status == "invalid"
or status == "failed"):
# Mark the file error that occurred
errorDb.writeFileError(jobId, filename, fileError, extraInfo)
jobTracker.markJobStatus(jobId, status)
except ResponseException as e:
# Could not get a unique job ID in the database, either a bad job ID was passed in
# or the record of that job was lost.
# Either way, cannot mark status of a job that does not exist
# Log error
JsonResponse.error(e, e.status)
def finalize(self):
""" Set upload job in job tracker database to finished, allowing dependent jobs to be started
Flask request should include key "upload_id", which holds the job_id for the file_upload job
Returns:
A flask response object, if successful just contains key "success" with value True, otherwise value is False
"""
responseDict = {}
try:
inputDictionary = RequestDictionary(self.request)
jobId = inputDictionary.getValue("upload_id")
# Compare user ID with user who submitted job, if no match return 400
job = self.jobManager.getJobById(jobId)
submission = self.jobManager.getSubmissionForJob(job)
if(submission.user_id != LoginSession.getName(session)):
# This user cannot finalize this job
raise ResponseException("Cannot finalize a job created by a different user", StatusCode.CLIENT_ERROR)
# Change job status to finished
if(self.jobManager.checkUploadType(jobId)):
self.jobManager.changeToFinished(jobId)
responseDict["success"] = True
return JsonResponse.create(StatusCode.OK,responseDict)
else:
raise ResponseException("Wrong job type for finalize route",StatusCode.CLIENT_ERROR)
except ( ValueError , TypeError ) as e:
return JsonResponse.error(e,StatusCode.CLIENT_ERROR)
except ResponseException as e:
return JsonResponse.error(e,e.status)
except Exception as e:
# Unexpected exception, this is a 500 server error
return JsonResponse.error(e,StatusCode.INTERNAL_ERROR)
def get_submission_data(submission, file_type=''):
""" Get data for the submission specified
Args:
submission: submission to retrieve metadata for
file_type: the type of job to retrieve metadata for
Returns:
JsonResponse containing the error information or the object containing metadata for all relevant file types
"""
sess = GlobalDB.db().session
file_type = file_type.lower()
# Make sure the file type provided is valid
if file_type and file_type not in FILE_TYPE_DICT and file_type != 'cross':
return JsonResponse.error(ValueError(file_type + ' is not a valid file type'), StatusCode.CLIENT_ERROR)
# Make sure the file type provided is valid for the submission type
is_fabs = submission.d2_submission
if file_type and (is_fabs and file_type != 'fabs') or (not is_fabs and file_type == 'fabs'):
return JsonResponse.error(ValueError(file_type + ' is not a valid file type for this submission'),
StatusCode.CLIENT_ERROR)
job_query = sess.query(Job).filter(Job.submission_id == submission.submission_id)
if not file_type:
relevant_job_types = (JOB_TYPE_DICT['csv_record_validation'], JOB_TYPE_DICT['validation'])
job_query = job_query.filter(Job.job_type_id.in_(relevant_job_types))
elif file_type == 'cross':
job_query = job_query.filter(Job.job_type_id == JOB_TYPE_DICT['validation'])
else:
job_query = job_query.filter(Job.file_type_id == FILE_TYPE_DICT[file_type])
job_dict = {'jobs': [job_to_dict(job) for job in job_query]}
return JsonResponse.create(StatusCode.OK, job_dict)
def getErrorMetrics(self) :
""" Returns an Http response object containing error information for every validation job in specified submission """
responseDict = {}
returnDict = {}
try:
safeDictionary = RequestDictionary(self.request)
submission_id = safeDictionary.getValue("submission_id")
# Check if user has permission to specified submission
self.checkSubmissionPermission(self.jobManager.getSubmissionById(submission_id))
jobIds = self.jobManager.getJobsBySubmission(submission_id)
for currentId in jobIds :
if(self.jobManager.getJobType(currentId) == "csv_record_validation"):
fileName = self.jobManager.getFileType(currentId)
dataList = self.interfaces.errorDb.getErrorMetricsByJobId(currentId)
returnDict[fileName] = dataList
return JsonResponse.create(StatusCode.OK,returnDict)
except ( ValueError , TypeError ) as e:
return JsonResponse.error(e,StatusCode.CLIENT_ERROR)
except ResponseException as e:
return JsonResponse.error(e,e.status)
except Exception as e:
# Unexpected exception, this is a 500 server error
return JsonResponse.error(e,StatusCode.INTERNAL_ERROR)
def createEmailConfirmation(self,system_email,session):
"""
Creates user record and email
arguments:
system_email -- (string) email used to send messages
session -- (Session) object from flask
"""
requestFields = RequestDictionary(self.request)
if(not requestFields.exists("email")):
exc = ResponseException("Request body must include email", StatusCode.CLIENT_ERROR)
return JsonResponse.error(exc,exc.status)
email = requestFields.getValue("email")
if( not re.match("[^@]+@[^@]+\.[^@]+",email)) :
return JsonResponse.error(ValueError("Invalid Email Format"),StatusCode.CLIENT_ERROR)
try :
user = self.interfaces.userDb.getUserByEmail(requestFields.getValue("email"))
except ResponseException as e:
self.interfaces.userDb.addUnconfirmedEmail(email)
else:
if(not (user.user_status_id == self.interfaces.userDb.getUserStatusId("awaiting_confirmation") or user.user_status_id == self.interfaces.userDb.getUserStatusId("email_confirmed"))):
exc = ResponseException("User already registered", StatusCode.CLIENT_ERROR)
return JsonResponse.error(exc,exc.status)
emailToken = sesEmail.createToken(email,self.interfaces.userDb,"validate_email")
link= "".join([AccountHandler.FRONT_END,'#/registration/',emailToken])
emailTemplate = {'[USER]': email, '[URL]':link}
newEmail = sesEmail(email, system_email,templateType="validate_email",parameters=emailTemplate,database=self.interfaces.userDb)
newEmail.send()
return JsonResponse.create(StatusCode.OK,{"message":"Email Sent"})
def setNewPassword(self, session):
""" Set a new password for a user, request should have keys "user_email" and "password" """
requestDict = RequestDictionary(self.request)
if (not (requestDict.exists("user_email")
and requestDict.exists("password"))):
# Don't have the keys we need in request
exc = ResponseException(
"Set password route requires keys user_email and password",
StatusCode.CLIENT_ERROR)
return JsonResponse.error(exc, exc.status)
if (not self.checkPassword(requestDict.getValue("password"))):
exc = ResponseException("Invalid Password",
StatusCode.CLIENT_ERROR)
return JsonResponse.error(exc, exc.status)
# Get user from email
user = self.interfaces.userDb.getUserByEmail(
requestDict.getValue("user_email"))
# Set new password
self.interfaces.userDb.setPassword(user,
requestDict.getValue("password"),
self.bcrypt)
# Invalidate token
self.interfaces.userDb.deleteToken(session["token"])
session["reset"] = None
# Return success message
return JsonResponse.create(
StatusCode.OK, {"message": "Password successfully changed"})
def checkEmailConfirmationToken(self,session):
"""
Creates user record and email
arguments:
session -- (Session) object from flask
return the reponse object with a error code and a message
"""
requestFields = RequestDictionary(self.request)
if(not requestFields.exists("token")):
exc = ResponseException("Request body must include token", StatusCode.CLIENT_ERROR)
return JsonResponse.error(exc,exc.status)
token = requestFields.getValue("token")
success,message,errorCode = sesEmail.checkToken(token,self.interfaces.userDb,"validate_email")
if(success):
#mark session that email can be filled out
LoginSession.register(session)
#remove token so it cant be used again
# The following line is commented out for issues with registration email links bouncing users back
# to the original email input page instead of the registration page
#self.interfaces.userDb.deleteToken(token)
#set the status only if current status is awaiting confirmation
user = self.interfaces.userDb.getUserByEmail(message)
if self.interfaces.userDb.checkStatus(user,"awaiting_confirmation"):
self.interfaces.userDb.changeStatus(user,"email_confirmed")
return JsonResponse.create(StatusCode.OK,{"email":message,"errorCode":errorCode,"message":"success"})
else:
#failure but alert UI of issue
return JsonResponse.create(StatusCode.OK,{"errorCode":errorCode,"message":message})
def set_new_password(self, session):
""" Set a new password for a user, request should have keys "user_email" and "password" """
sess = GlobalDB.db().session
request_dict = RequestDictionary.derive(self.request)
required = ('user_email', 'password')
try:
if any(field not in request_dict for field in required):
# Don't have the keys we need in request
raise ResponseException(
"Set password route requires keys user_email and password",
StatusCode.CLIENT_ERROR
)
if not self.checkPassword(request_dict['password']):
raise ResponseException(
"Invalid Password", StatusCode.CLIENT_ERROR)
except ResponseException as exc:
return JsonResponse.error(exc,exc.status)
# Get user from email
user = sess.query(User).filter(
func.lower(User.email) == func.lower(request_dict["user_email"])
).one()
# Set new password
set_user_password(user,request_dict["password"],self.bcrypt)
# Invalidate token
oldToken = sess.query(EmailToken).filter(EmailToken.token == session["token"]).one()
sess.delete(oldToken)
sess.commit()
session["reset"] = None
# Return success message
return JsonResponse.create(StatusCode.OK,{"message":"Password successfully changed"})
def list_users(self):
""" List all users ordered by status. Associated request body must have key 'filter_by' """
request_dict = RequestDictionary.derive(
self.request, optional_request=True)
user_status = request_dict.get('status', 'all')
sess = GlobalDB.db().session
try:
user_query = sess.query(User)
if user_status != "all":
user_query = user_query.filter(User.user_status_id == USER_STATUS_DICT[user_status])
users = user_query.all()
except ValueError as exc:
# Client provided a bad status
return JsonResponse.error(exc, StatusCode.CLIENT_ERROR)
user_info = []
for user in users:
agency_name = sess.query(CGAC.agency_name).\
filter(CGAC.cgac_code == user.cgac_code).\
one_or_none()
thisInfo = {"name":user.name, "title":user.title, "agency_name":agency_name, "cgac_code":user.cgac_code,
"email":user.email, "id":user.user_id, "is_active":user.is_active,
"permission": PERMISSION_TYPE_DICT_ID.get(user.permission_type_id), "status": user.user_status.name}
user_info.append(thisInfo)
return JsonResponse.create(StatusCode.OK,{"users":user_info})
def set_skip_guide(self):
""" Set current user's skip guide parameter
Returns:
JsonResponse object containing results of setting the skip guide or details of the error that occurred.
Possible errors include the request not containing a skip_guide parameter or it not being a boolean
value
"""
sess = GlobalDB.db().session
request_dict = RequestDictionary.derive(self.request)
try:
if 'skip_guide' not in request_dict:
raise ResponseException("Must include skip_guide parameter",
StatusCode.CLIENT_ERROR)
skip_guide = str(request_dict['skip_guide']).lower()
if skip_guide not in ("true", "false"):
raise ResponseException("skip_guide must be true or false",
StatusCode.CLIENT_ERROR)
g.user.skip_guide = skip_guide == "true"
except ResponseException as exc:
return JsonResponse.error(exc, exc.status)
sess.commit()
return JsonResponse.create(StatusCode.OK, {
"message": "skip_guide set successfully",
"skip_guide": skip_guide
})
def resetPassword(self, system_email, session):
"""
Remove old password and email user a token to set a new password. Request should have key "email"
arguments:
system_email -- (string) email used to send messages
session -- (Session) object from flask
"""
requestDict = RequestDictionary(self.request)
if (not (requestDict.exists("email"))):
# Don't have the keys we need in request
exc = ResponseException(
"Reset password route requires key 'email'",
StatusCode.CLIENT_ERROR)
return JsonResponse.error(exc, exc.status)
# Get user object
try:
user = self.interfaces.userDb.getUserByEmail(
requestDict.getValue("email"))
except Exception as e:
exc = ResponseException("Unknown Error", StatusCode.CLIENT_ERROR,
ValueError)
return JsonResponse.error(exc, exc.status)
email = requestDict.getValue("email")
LoginSession.logout(session)
self.sendResetPasswordEmail(user, system_email, email)
# Return success message
return JsonResponse.create(StatusCode.OK,
{"message": "Password reset"})
def resetPassword(self,system_email,session):
"""
Remove old password and email user a token to set a new password. Request should have key "email"
arguments:
system_email -- (string) email used to send messages
session -- (Session) object from flask
"""
requestDict = RequestDictionary(self.request)
if(not (requestDict.exists("email"))):
# Don't have the keys we need in request
exc = ResponseException("Reset password route requires key 'email'",StatusCode.CLIENT_ERROR)
return JsonResponse.error(exc,exc.status)
# Get user object
try:
user = self.interfaces.userDb.getUserByEmail(requestDict.getValue("email"))
except Exception as e:
exc = ResponseException("Unknown Error",StatusCode.CLIENT_ERROR,ValueError)
return JsonResponse.error(exc,exc.status)
LoginSession.logout(session)
self.interfaces.userDb.session.commit()
email = requestDict.getValue("email")
# Send email with token
emailToken = sesEmail.createToken(email,self.interfaces.userDb,"password_reset")
link= "".join([ AccountHandler.FRONT_END,'#/forgotpassword/',emailToken])
emailTemplate = { '[URL]':link}
newEmail = sesEmail(user.email, system_email,templateType="reset_password",parameters=emailTemplate,database=self.interfaces.userDb)
newEmail.send()
# Return success message
return JsonResponse.create(StatusCode.OK,{"message":"Password reset"})
def delete_submission(submission):
""" Deletes all data associated with the specified submission
NOTE: THERE IS NO WAY TO UNDO THIS """
if submission.publish_status_id != PUBLISH_STATUS_DICT['unpublished']:
return JsonResponse.error(ValueError("Submissions that have been certified cannot be deleted"),
StatusCode.CLIENT_ERROR)
sess = GlobalDB.db().session
# Check if the submission has any jobs that are currently running, if so, do not allow deletion
jobs = sess.query(Job).filter(Job.submission_id == submission.submission_id,
Job.job_status_id == JOB_STATUS_DICT['running']).all()
if jobs:
return JsonResponse.error(ValueError("Submissions with running jobs cannot be deleted"),
StatusCode.CLIENT_ERROR)
sess.query(SubmissionSubTierAffiliation).filter(
SubmissionSubTierAffiliation.submission_id == submission.submission_id).delete(
synchronize_session=False)
sess.query(Submission).filter(Submission.submission_id == submission.submission_id).delete(
synchronize_session=False)
sess.expire_all()
return JsonResponse.create(StatusCode.OK, {"message": "Success"})
def inner(*args, **kwargs):
if g.user is None:
return JsonResponse.create(StatusCode.LOGIN_REQUIRED, {'message': "Login Required"})
if not g.user.website_admin:
return JsonResponse.create(StatusCode.LOGIN_REQUIRED, {'message': NOT_AUTHORIZED_MSG})
return func(*args, **kwargs)
def login(self,session):
"""
Logs a user in if their password matches
arguments:
session -- (Session) object from flask
return the reponse object
"""
try:
safeDictionary = RequestDictionary(self.request)
username = safeDictionary.getValue('username')
password = safeDictionary.getValue('password')
try:
user = self.interfaces.userDb.getUserByEmail(username)
except Exception as e:
raise ValueError("user name and or password invalid")
if(not self.interfaces.userDb.checkStatus(user,"approved")):
raise ValueError("user name and or password invalid")
# Only check if user is active after they've logged in for the first time
if user.last_login_date is not None and not self.isUserActive(user):
raise ValueError("Your account has expired. Please contact an administrator.")
try:
if(self.interfaces.userDb.checkPassword(user,password,self.bcrypt)):
# We have a valid login
LoginSession.login(session,user.user_id)
permissionList = []
for permission in self.interfaces.userDb.getPermssionList():
if(self.interfaces.userDb.hasPermission(user, permission.name)):
permissionList.append(permission.permission_type_id)
self.interfaces.userDb.updateLastLogin(user)
return JsonResponse.create(StatusCode.OK,{"message":"Login successful","user_id": int(user.user_id),"name":user.name,"title":user.title ,"agency":user.agency, "permissions" : permissionList})
else :
raise ValueError("user name and or password invalid")
except Exception as e:
LoginSession.logout(session)
raise ValueError("user name and or password invalid")
except (TypeError, KeyError, NotImplementedError) as e:
# Return a 400 with appropriate message
return JsonResponse.error(e,StatusCode.CLIENT_ERROR)
except ValueError as e:
# Return a 401 for login denied
return JsonResponse.error(e,StatusCode.LOGIN_REQUIRED)
except Exception as e:
# Return 500
return JsonResponse.error(e,StatusCode.INTERNAL_ERROR)
return self.response
def startGenerationJob(self, submission_id, file_type):
""" Initiates a file generation job
Args:
submission_id: ID of submission to start job for
file_type: Type of file to be generated
Returns:
Tuple of boolean indicating successful start, and error response if False
"""
jobDb = self.interfaces.jobDb
file_type_name = self.fileTypeMap[file_type]
if file_type in ["D1", "D2"]:
# Populate start and end dates, these should be provided in MM/DD/YYYY format, using calendar year (not fiscal year)
requestDict = RequestDictionary(self.request)
start_date = requestDict.getValue("start")
end_date = requestDict.getValue("end")
if not (StringCleaner.isDate(start_date) and StringCleaner.isDate(end_date)):
exc = ResponseException("Start or end date cannot be parsed into a date", StatusCode.CLIENT_ERROR)
return False, JsonResponse.error(exc, exc.status, start = "", end = "", file_type = file_type, status = "failed")
elif file_type not in ["E","F"]:
exc = ResponseException("File type must be either D1, D2, E or F", StatusCode.CLIENT_ERROR)
return False, JsonResponse.error(exc, exc.status, file_type = file_type, status = "failed")
cgac_code = self.jobManager.getSubmissionById(submission_id).cgac_code
# Generate and upload file to S3
user_id = LoginSession.getName(session)
timestamped_name = s3UrlHandler.getTimestampedFilename(CONFIG_BROKER["".join([str(file_type_name),"_file_name"])])
if self.isLocal:
upload_file_name = "".join([CONFIG_BROKER['broker_files'], timestamped_name])
else:
upload_file_name = "".join([str(user_id), "/", timestamped_name])
job = jobDb.getJobBySubmissionFileTypeAndJobType(submission_id, file_type_name, "file_upload")
job.filename = upload_file_name
job.original_filename = timestamped_name
job.job_status_id = jobDb.getJobStatusId("running")
jobDb.session.commit()
if file_type in ["D1", "D2"]:
CloudLogger.log("DEBUG: Adding job info for job id of " + str(job.job_id),
log_type="debug",
file_name=self.debug_file_name)
return self.addJobInfoForDFile(upload_file_name, timestamped_name, submission_id, file_type, file_type_name, start_date, end_date, cgac_code, job)
elif file_type == 'E':
generate_e_file.delay(
submission_id, job.job_id, InterfaceHolder, timestamped_name,
upload_file_name, self.isLocal)
elif file_type == 'F':
generate_f_file.delay(
submission_id, job.job_id, InterfaceHolder, timestamped_name,
upload_file_name, self.isLocal)
return True, None
def changeStatus(self, system_email):
"""
Changes status for specified user. Associated request body should have keys 'uid' and 'new_status'
arguments:
system_email -- (string) the emaily to send emails from
return the reponse object with a success message
"""
requestDict = RequestDictionary(self.request)
if (not (requestDict.exists("uid")
and requestDict.exists("new_status"))):
# Missing a required field, return 400
exc = ResponseException(
"Request body must include uid and new_status",
StatusCode.CLIENT_ERROR)
return JsonResponse.error(exc, exc.status)
# Find user that matches specified uid
user = self.interfaces.userDb.getUserByUID(
int(requestDict.getValue("uid")))
if (user.email == None):
return JsonResponse.error(
ResponseException("User does not have a defined email",
StatusCode.INTERNAL_ERROR),
StatusCode.INTERNAL_ERROR)
#check if the user is waiting
if (self.interfaces.userDb.checkStatus(user, "awaiting_approval")):
if (requestDict.getValue("new_status") == "approved"):
# Grant agency_user permission to newly approved users
self.interfaces.userDb.grantPermission(user, "agency_user")
link = AccountHandler.FRONT_END
emailTemplate = {'[URL]': link, '[EMAIL]': system_email}
newEmail = sesEmail(user.email,
system_email,
templateType="account_approved",
parameters=emailTemplate,
database=self.interfaces.userDb)
newEmail.send()
elif (requestDict.getValue("new_status") == "denied"):
emailTemplate = {}
newEmail = sesEmail(user.email,
system_email,
templateType="account_rejected",
parameters=emailTemplate,
database=self.interfaces.userDb)
newEmail.send()
# Change user's status
self.interfaces.userDb.changeStatus(user,
requestDict.getValue("new_status"))
return JsonResponse.create(StatusCode.OK,
{"message": "Status change successful"})
def getProtectedFiles(self):
""" Returns a set of urls to protected files on the help page """
response = {}
if self.isLocal:
response["urls"] = {}
return JsonResponse.create(StatusCode.CLIENT_ERROR, response)
response["urls"] = self.s3manager.getFileUrls(bucket_name=CONFIG_BROKER["static_files_bucket"], path=CONFIG_BROKER["help_files_path"])
return JsonResponse.create(StatusCode.OK, response)
def email_users(submission, system_email, template_type, user_ids):
""" Send email notification to list of users
Args:
submission: the submission to send the email about
system_email: the address of the system to send the email from
template_type: the template type of the email to send
user_ids: A list of user IDs denoting who to send the email to
Returns:
A JsonReponse containing a message that the email sent successfully or the details of the missing
or incorrect parameters
"""
sess = GlobalDB.db().session
if submission.cgac_code:
agency = sess.query(CGAC).filter_by(
cgac_code=submission.cgac_code).first()
else:
agency = sess.query(FREC).filter_by(
frec_code=submission.frec_code).first()
if not agency:
return JsonResponse.error(
ValueError(
"The requested submission is not aligned to a valid CGAC or FREC "
"agency"), StatusCode.CLIENT_ERROR)
# Check if email template type is valid
get_email_template(template_type)
link = "".join([
AccountHandler.FRONT_END, '#/submission/',
str(submission.submission_id)
])
email_template = {
'[REV_USER_NAME]': g.user.name,
'[REV_AGENCY]': agency.agency_name,
'[REV_URL]': link
}
users = []
for user_id in user_ids:
# Check if user id is valid, if so add User object to array
users.append(
sess.query(User).filter(User.user_id == user_id).one())
for user in users:
new_email = SesEmail(user.email,
system_email,
template_type=template_type,
parameters=email_template)
new_email.send()
return JsonResponse.create(StatusCode.OK,
{"message": "Emails successfully sent"})
def inner(*args, **kwargs):
if g.user is None:
return JsonResponse.create(StatusCode.LOGIN_REQUIRED,
{'message': "Login Required"})
if not g.user.website_admin:
return JsonResponse.create(StatusCode.LOGIN_REQUIRED,
{'message': NOT_AUTHORIZED_MSG})
return func(*args, **kwargs)
def getErrorReportURLsForSubmission(self, isWarning = False):
"""
Gets the Signed URLs for download based on the submissionId
"""
try :
self.s3manager = s3UrlHandler()
safeDictionary = RequestDictionary(self.request)
submissionId = safeDictionary.getValue("submission_id")
responseDict ={}
sess = GlobalDB.db().session
for jobId in self.jobManager.getJobsBySubmission(submissionId):
# get the job object here so we can call the refactored getReportPath
# todo: replace other db access functions with job object attributes
job = sess.query(Job).filter(Job.job_id == jobId).one()
if job.job_type.name == 'csv_record_validation':
if isWarning:
reportName = getReportPath(job, 'warning')
key = "job_"+str(jobId)+"_warning_url"
else:
reportName = getReportPath(job, 'error')
key = "job_"+str(jobId)+"_error_url"
if(not self.isLocal):
responseDict[key] = self.s3manager.getSignedUrl("errors",reportName,method="GET")
else:
path = os.path.join(self.serverPath, reportName)
responseDict[key] = path
# For each pair of files, get url for the report
fileTypes = self.interfaces.validationDb.getFileTypeList()
for source in fileTypes:
sourceId = self.interfaces.validationDb.getFileTypeIdByName(source)
for target in fileTypes:
targetId = self.interfaces.validationDb.getFileTypeIdByName(target)
if targetId <= sourceId:
# Skip redundant reports
continue
# Retrieve filename
if isWarning:
reportName = getCrossWarningReportName(submissionId, source, target)
else:
reportName = getCrossReportName(submissionId, source, target)
# If not local, get a signed URL
if self.isLocal:
reportPath = os.path.join(self.serverPath,reportName)
else:
reportPath = self.s3manager.getSignedUrl("errors",reportName,method="GET")
# Assign to key based on source and target
responseDict[self.getCrossReportKey(source,target,isWarning)] = reportPath
return JsonResponse.create(StatusCode.OK,responseDict)
except ResponseException as e:
return JsonResponse.error(e,StatusCode.CLIENT_ERROR)
except Exception as e:
# Unexpected exception, this is a 500 server error
return JsonResponse.error(e,StatusCode.INTERNAL_ERROR)
def delete_all_submission_data(submission):
""" Delete a submission.
Args:
submission: submission to delete
Returns:
JsonResponse object containing a success message or the reason for failure
"""
# check if the submission has been published, if so, do not allow deletion
if submission.publish_status_id != PUBLISH_STATUS_DICT['unpublished']:
return JsonResponse.error(
ValueError(
"Submissions that have been certified cannot be deleted"),
StatusCode.CLIENT_ERROR)
sess = GlobalDB.db().session
all_jobs = sess.query(Job).filter(
Job.submission_id == submission.submission_id)
# check if the submission has any jobs that are currently running, if so, do not allow deletion
running_jobs = all_jobs.filter(
Job.job_status_id == JOB_STATUS_DICT['running']).all()
if running_jobs:
return JsonResponse.error(
ValueError("Submissions with running jobs cannot be deleted"),
StatusCode.CLIENT_ERROR)
logger.info({
"message":
"Deleting submission with id {}".format(submission.submission_id),
"message_type":
"BrokerInfo",
"submission_id":
submission.submission_id
})
for job in all_jobs.all():
# check if the submission has any cached D files, if so, disconnect that job from the submission
cached_file = sess.query(FileRequest).filter(
FileRequest.job_id == job.job_id,
FileRequest.is_cached_file.is_(True)).all()
if cached_file:
job.submission_id = None
sess.commit()
sess.query(SubmissionSubTierAffiliation).\
filter(SubmissionSubTierAffiliation.submission_id == submission.submission_id).\
delete(synchronize_session=False)
sess.query(Submission).filter(Submission.submission_id == submission.submission_id).\
delete(synchronize_session=False)
sess.expire_all()
return JsonResponse.create(StatusCode.OK, {"message": "Success"})
def list_rule_labels(files, error_level='warning', fabs=False):
""" Returns a list of rule labels based on the files and error type provided
Args:
files: A list of files for which to return rule labels. If blank, return all matching other arguments
error_level: A string indicating whether to return errors, warnings, or both. Defaults to warning
fabs: A boolean indicating whether to return FABS or DABS rules. Defaults to False
Returns:
JsonResponse of the rule labels the arguments indicate. JsonResponse error if invalid file types are
provided or any file types are provided for FABS
"""
# Make sure list is empty when requesting FABS rules
if fabs and len(files) > 0:
return JsonResponse.error(
ValueError('Files list must be empty for FABS rules'),
StatusCode.CLIENT_ERROR)
invalid_files = [
invalid_file for invalid_file in files
if invalid_file not in FILE_TYPES
]
if invalid_files:
return JsonResponse.error(
ValueError('The following are not valid file types: {}'.format(
', '.join(invalid_files))), StatusCode.CLIENT_ERROR)
sess = GlobalDB.db().session
rule_label_query = sess.query(RuleSql.rule_label)
# If the error level isn't "mixed" add a filter on which severity to pull
if error_level == 'error':
rule_label_query = rule_label_query.filter_by(
rule_severity_id=RULE_SEVERITY_DICT['fatal'])
elif error_level == 'warning':
rule_label_query = rule_label_query.filter_by(
rule_severity_id=RULE_SEVERITY_DICT['warning'])
# If specific files have been specified, add a filter to get them
if files:
rule_label_query = file_filter(rule_label_query, RuleSql, files)
elif not fabs:
# If not the rules are not FABS, exclude FABS rules
rule_label_query = rule_label_query.filter(
RuleSql.file_id != FILE_TYPE_DICT_LETTER_ID['FABS'])
else:
# If the rule is FABS, add a filter to only get FABS rules
rule_label_query = rule_label_query.filter_by(
file_id=FILE_TYPE_DICT_LETTER_ID['FABS'])
return JsonResponse.create(
StatusCode.OK,
{'labels': [label.rule_label for label in rule_label_query.all()]})
def decorated_function(*args, **kwargs):
try:
sess = GlobalDB.db().session
error_message = "Login Required"
if permission == "check_email_token":
if LoginSession.isRegistering(session):
return f(*args, **kwargs)
else:
error_message = "unauthorized"
elif permission == "check_password_token":
if LoginSession.isResetingPassword(session):
return f(*args, **kwargs)
else:
error_message = "unauthorized"
elif LoginSession.isLogin(session):
user = sess.query(User).filter(
User.user_id == session["name"]).one()
valid_user = True
if permission is not None:
perm_hierarchy = {
d['name']: d['order']
for d in PERMISSION_MAP.values()
}
# if the users permission is not higher than the one specified, check their permission
# if user's perm order is < than what's passed in, it means they have higher permissions
if perm_hierarchy[PERMISSION_TYPE_DICT_ID[
user.permission_type_id]] > perm_hierarchy[
permission]:
if not user.permission_type_id == PERMISSION_TYPE_DICT[
permission]:
valid_user = False
if valid_user:
return f(*args, **kwargs)
error_message = "You are not authorized to perform the requested task. Please contact your administrator."
# No user logged in
return_response = flask.Response()
return_response.headers["Content-Type"] = "application/json"
return_response.status_code = 401 # Error code
response_dict = {}
response_dict["message"] = error_message
return_response.set_data(json.dumps(response_dict))
return return_response
except ResponseException as e:
return JsonResponse.error(e, e.status)
except InvalidUsage:
raise
except Exception as e:
exc = ResponseException(str(e), StatusCode.INTERNAL_ERROR,
type(e))
return JsonResponse.error(exc, exc.status)
def deleteUser(self):
""" Deletes user specified by 'email' in request """
requestDict = RequestDictionary(self.request)
if not requestDict.exists("email"):
# missing required fields, return 400
exc = ResponseException(
"Request body must include email of user to be deleted",
StatusCode.CLIENT_ERROR)
return JsonResponse.error(exc, exc.status)
email = requestDict.getValue("email")
self.interfaces.userDb.deleteUser(email)
return JsonResponse.create(StatusCode.OK, {"message": "success"})
def validate_threaded():
"""Start the validation process on a new thread."""
@copy_current_request_context
def ThreadedFunction(arg):
"""The new thread."""
threadedManager = ValidationManager(local, error_report_path)
threadedManager.threadedValidateJob(arg)
try:
interfaces = InterfaceHolder()
jobTracker = interfaces.jobDb
except ResponseException as e:
open("errorLog","a").write(str(e) + "\n")
return JsonResponse.error(e,e.status,table = "cannot connect to job database")
except Exception as e:
open("errorLog","a").write(str(e) + "\n")
exc = ResponseException(str(e),StatusCode.INTERNAL_ERROR,type(e))
return JsonResponse.error(exc,exc.status,table= "cannot connect to job database")
jobId = None
manager = ValidationManager(local, error_report_path)
try:
jobId = manager.getJobID(request)
except ResponseException as e:
manager.markJob(jobId,jobTracker,"invalid",interfaces.errorDb,manager.filename)
CloudLogger.logError(str(e),e,traceback.extract_tb(sys.exc_info()[2]))
return JsonResponse.error(e,e.status,table ="")
except Exception as e:
exc = ResponseException(str(e),StatusCode.CLIENT_ERROR,type(e))
manager.markJob(jobId,jobTracker,"invalid",interfaces.errorDb,manager.filename)
CloudLogger.logError(str(e),exc,traceback.extract_tb(sys.exc_info()[2]))
return JsonResponse.error(exc,exc.status,table="")
try:
manager.testJobID(jobId,interfaces)
except ResponseException as e:
open("errorLog","a").write(str(e) + "\n")
# Job is not ready to run according to job tracker, do not change status of job in job tracker
interfaces.errorDb.writeFileError(jobId,manager.filename,ValidationError.jobError)
return JsonResponse.error(e,e.status,table="")
except Exception as e:
open("errorLog","a").write(str(e) + "\n")
exc = ResponseException(str(e),StatusCode.CLIENT_ERROR,type(e))
interfaces.errorDb.writeFileError(jobId,manager.filename,ValidationError.jobError)
return JsonResponse.error(exc,exc.status,table="")
thread = Thread(target=ThreadedFunction, args= (jobId,))
try :
jobTracker.markJobStatus(jobId,"running")
except Exception as e:
open("errorLog","a").write(str(e) + "\n")
exc = ResponseException(str(e),StatusCode.INTERNAL_ERROR,type(e))
return JsonResponse.error(exc,exc.status,table="could not start job")
interfaces.close()
thread.start()
return JsonResponse.create(StatusCode.OK,{"table":"job"+str(jobId)})
def submission_list_certifications(submission):
if submission.d2_submission:
return JsonResponse.error(ValueError("FABS submissions do not have a certification history"),
StatusCode.CLIENT_ERROR)
sess = GlobalDB.db().session
certify_history = sess.query(CertifyHistory).filter_by(submission_id=submission.submission_id)
if certify_history.count() == 0:
return JsonResponse.error(ValueError("This submission has no certification history"),
StatusCode.CLIENT_ERROR)
return list_certifications(submission)
def login(self, session):
"""
Logs a user in if their password matches
arguments:
session -- (Session) object from flask
return the response object
"""
try:
sess = GlobalDB.db().session
safe_dictionary = RequestDictionary(self.request)
username = safe_dictionary.get_value('username')
password = safe_dictionary.get_value('password')
try:
user = sess.query(User).filter(func.lower(User.email) == func.lower(username)).one()
except Exception:
raise ValueError("Invalid username and/or password")
try:
if check_correct_password(user, password, self.bcrypt):
# We have a valid login
return self.create_session_and_response(session, user)
else:
raise ValueError("Invalid username and/or password")
except ValueError as ve:
LoginSession.logout(session)
raise ve
except Exception as e:
LoginSession.logout(session)
raise e
except (TypeError, KeyError, NotImplementedError) as e:
# Return a 400 with appropriate message
return JsonResponse.error(e, StatusCode.CLIENT_ERROR)
except ValueError as e:
# Return a 401 for login denied
return JsonResponse.error(e, StatusCode.LOGIN_REQUIRED)
except Exception as e:
# Return 500
return JsonResponse.error(e, StatusCode.INTERNAL_ERROR)
def list_all_agencies():
sess = GlobalDB.db().session
cgacs = sess.query(CGAC).all()
agency_list = [{'agency_name': cgac.agency_name, 'cgac_code': cgac.cgac_code} for cgac in cgacs]
frecs = sess.query(FREC).all()
shared_list = [{'agency_name': frec.agency_name, 'frec_code': frec.frec_code} for frec in frecs]
return JsonResponse.create(StatusCode.OK, {'agency_list': agency_list, 'shared_agency_list': shared_list})
def list_sub_tier_agencies(sub_tier_agencies):
""" List all Sub-Tier Agencies user has FABS permissions for
Args:
sub_tier_agencies - List of all SubTierAgencies generated by the get_fabs_sub_tier_agencies decorator,
required to list only sub_tier_agencies that user has FABS permissions for
"""
return JsonResponse.create(StatusCode.OK, organize_sub_tier_agencies(sub_tier_agencies))
def upload_fabs_file():
if "multipart/form-data" not in request.headers['Content-Type']:
return JsonResponse.error(ValueError("Request must be a multipart/form-data type"), StatusCode.CLIENT_ERROR)
params = RequestDictionary.derive(request)
fabs = params.get('_files', {}).get('fabs', None)
file_manager = FileHandler(request, is_local=is_local, server_path=server_path)
return file_manager.upload_fabs_file(fabs)
def submit_files():
file_manager = FileHandler(request, is_local=is_local, server_path=server_path)
sess = GlobalDB.db().session
start_date = request.json.get('reporting_period_start_date')
end_date = request.json.get('reporting_period_end_date')
is_quarter = request.json.get('is_quarter_format', False)
if not (start_date is None or end_date is None):
formatted_start_date, formatted_end_date = FileHandler.check_submission_dates(start_date,
end_date, is_quarter)
submissions = sess.query(Submission).filter(
Submission.cgac_code == request.json.get('cgac_code'),
Submission.frec_code == request.json.get('frec_code'),
Submission.reporting_start_date == formatted_start_date,
Submission.reporting_end_date == formatted_end_date,
Submission.is_quarter_format == request.json.get('is_quarter'),
Submission.publish_status_id != PUBLISH_STATUS_DICT['unpublished'])
if 'existing_submission_id' in request.json:
submissions.filter(Submission.submission_id !=
request.json['existing_submission_id'])
submissions = submissions.order_by(desc(Submission.created_at))
if submissions.count() > 0:
data = {
"message": "A submission with the same period already exists.",
"submissionId": submissions[0].submission_id
}
return JsonResponse.create(StatusCode.CLIENT_ERROR, data)
return file_manager.submit(create_credentials)
def listAgencies(self):
agencies = self.validationManager.getAllAgencies()
agency_list = []
for agency in agencies:
agency_list.append({"agency_name": agency.agency_name, "cgac_code": agency.cgac_code})
return JsonResponse.create(StatusCode.OK, {"cgac_agency_list": agency_list})
def getRss(self):
response = {}
if self.isLocal:
response["rss_url"] = os.path.join(self.serverPath, CONFIG_BROKER["rss_folder"],CONFIG_BROKER["rss_file"])
else:
self.s3manager = s3UrlHandler()
response["rss_url"] = self.s3manager.getSignedUrl(CONFIG_BROKER["rss_folder"],CONFIG_BROKER["rss_file"],"GET")
return JsonResponse.create(200,response)
def login(self, session):
""" Logs a user in if their password matches using local data
Args:
session: the Session object from flask
Returns:
A JsonResponse containing the user information or details on which error occurred, such as whether a
type was wrong, something wasn't implemented, invalid keys were provided, login was denied, or a
different, unexpected error occurred.
"""
try:
sess = GlobalDB.db().session
safe_dictionary = RequestDictionary(self.request)
username = safe_dictionary.get_value('username')
password = safe_dictionary.get_value('password')
try:
user = sess.query(User).filter(func.lower(User.email) == func.lower(username)).one()
except Exception:
raise ValueError("Invalid username and/or password")
try:
if check_correct_password(user, password, self.bcrypt):
# We have a valid login
return self.create_session_and_response(session, user)
else:
raise ValueError("Invalid username and/or password")
except ValueError as ve:
LoginSession.logout(session)
raise ve
except Exception as e:
LoginSession.logout(session)
raise e
# Catch any specifically raised errors or any other errors that may have happened and return them cleanly
except (TypeError, KeyError, NotImplementedError) as e:
# Return a 400 with appropriate message
return JsonResponse.error(e, StatusCode.CLIENT_ERROR)
except ValueError as e:
# Return a 401 for login denied
return JsonResponse.error(e, StatusCode.LOGIN_REQUIRED)
except Exception as e:
# Return 500
return JsonResponse.error(e, StatusCode.INTERNAL_ERROR)
def listSubmissionsByCurrentUser(self):
""" List all submission IDs associated with the current user ID """
userId = LoginSession.getName(flaskSession)
submissions = self.interfaces.jobDb.getSubmissionsByUserId(userId)
submissionIdList = []
for submission in submissions:
submissionIdList.append(submission.submission_id)
return JsonResponse.create(StatusCode.OK,{"submission_id_list": submissionIdList})
def generate_detached_file(file_type, cgac_code, frec_code, start, end):
""" Generate a file from external API, independent from a submission """
if not cgac_code and not frec_code:
return JsonResponse.error(ValueError("Detached file generation requires CGAC or FR Entity Code"),
StatusCode.CLIENT_ERROR)
file_manager = FileHandler(request, is_local=is_local, server_path=server_path)
return file_manager.generate_detached_file(file_type, cgac_code, frec_code, start, end)
def check_year_and_quarter(cgac_code, frec_code, reporting_fiscal_year, reporting_fiscal_period):
""" Check if cgac (or frec) code, year, and quarter already has a published submission """
if not cgac_code and not frec_code:
return JsonResponse.error(ValueError("CGAC or FR Entity Code required"), StatusCode.CLIENT_ERROR)
sess = GlobalDB.db().session
return find_existing_submissions_in_period(sess, cgac_code, frec_code, reporting_fiscal_year,
reporting_fiscal_period)