def finalize(self):
""" Set upload job in job tracker database to finished, allowing dependent jobs to be started
Flask request should include key "upload_id", which holds the job_id for the file_upload job
Returns:
A flask response object, if successful just contains key "success" with value True, otherwise value is False
"""
responseDict = {}
try:
inputDictionary = RequestDictionary(self.request)
jobId = inputDictionary.getValue("upload_id")
# Compare user ID with user who submitted job, if no match return 400
job = self.jobManager.getJobById(jobId)
submission = self.jobManager.getSubmissionForJob(job)
if(submission.user_id != LoginSession.getName(session)):
# This user cannot finalize this job
raise ResponseException("Cannot finalize a job created by a different user", StatusCode.CLIENT_ERROR)
# Change job status to finished
if(self.jobManager.checkUploadType(jobId)):
self.jobManager.changeToFinished(jobId)
responseDict["success"] = True
return JsonResponse.create(StatusCode.OK,responseDict)
else:
raise ResponseException("Wrong job type for finalize route",StatusCode.CLIENT_ERROR)
except ( ValueError , TypeError ) as e:
return JsonResponse.error(e,StatusCode.CLIENT_ERROR)
except ResponseException as e:
return JsonResponse.error(e,e.status)
except Exception as e:
# Unexpected exception, this is a 500 server error
return JsonResponse.error(e,StatusCode.INTERNAL_ERROR)
def run_instance_function(
accountManager, accountFunction, getSystemEmail=False, getSession=False, getUser=False, getCredentials=False
):
""" Standard error handling around each route """
interfaces = InterfaceHolder()
try:
accountManager.addInterfaces(interfaces)
if getSystemEmail and getSession:
return accountFunction(RouteUtils.SYSTEM_EMAIL, session)
elif getSystemEmail:
return accountFunction(RouteUtils.SYSTEM_EMAIL)
elif getSession:
return accountFunction(session)
elif getUser:
if getCredentials:
return accountFunction(LoginSession.getName(session), RouteUtils.CREATE_CREDENTIALS)
else:
# Currently no functions with user but not credentials flag
raise ValueError("Invalid combination of flags to run_instance_function")
else:
return accountFunction()
except ResponseException as e:
return JsonResponse.error(e, e.status)
except Exception as e:
exc = ResponseException(str(e), StatusCode.INTERNAL_ERROR, type(e))
return JsonResponse.error(exc, exc.status)
finally:
interfaces.close()
def run_instance_function(accountManager,
accountFunction,
getSystemEmail=False,
getSession=False,
getUser=False,
getCredentials=False):
""" Standard error handling around each route """
interfaces = InterfaceHolder()
try:
accountManager.addInterfaces(interfaces)
if (getSystemEmail and getSession):
return accountFunction(RouteUtils.SYSTEM_EMAIL, session)
elif (getSystemEmail):
return accountFunction(RouteUtils.SYSTEM_EMAIL)
elif (getSession):
return accountFunction(session)
elif (getUser):
if (getCredentials):
return accountFunction(LoginSession.getName(session),
RouteUtils.CREATE_CREDENTIALS)
else:
# Currently no functions with user but not credentials flag
raise ValueError(
"Invalid combination of flags to run_instance_function"
)
else:
return accountFunction()
except ResponseException as e:
return JsonResponse.error(e, e.status)
except Exception as e:
exc = ResponseException(str(e), StatusCode.INTERNAL_ERROR, type(e))
return JsonResponse.error(exc, exc.status)
finally:
interfaces.close()
def listSubmissionsByCurrentUser(self):
""" List all submission IDs associated with the current user ID """
userId = LoginSession.getName(flaskSession)
submissions = self.interfaces.jobDb.getSubmissionsByUserId(userId)
submissionIdList = []
for submission in submissions:
submissionIdList.append(submission.submission_id)
return JsonResponse.create(StatusCode.OK,{"submission_id_list": submissionIdList})
def submit_files():
fileManager = FileHandler(request,
isLocal=IS_LOCAL,
serverPath=SERVER_PATH)
return RouteUtils.run_instance_function(fileManager,
fileManager.submit,
LoginSession.getName(session),
RouteUtils.CREATE_CREDENTIALS)
def checkSubmissionPermission(self,submission):
""" Check if current user has permisson to access submission and return user object. """
userId = LoginSession.getName(session)
user = self.interfaces.userDb.getUserByUID(userId)
# Check that user has permission to see this submission, user must either own the submission or be an admin
if(submission.user_id != userId and not self.interfaces.userDb.hasPermission(user,"website_admin")):
raise ResponseException("User does not have permission to view that submission",StatusCode.CLIENT_ERROR)
return user
def listSubmissionsByCurrentUser(self):
""" List all submission IDs associated with the current user ID """
userId = LoginSession.getName(flaskSession)
submissions = self.interfaces.jobDb.getSubmissionsByUserId(userId)
submissionIdList = []
for submission in submissions:
submissionIdList.append(submission.submission_id)
return JsonResponse.create(StatusCode.OK,
{"submission_id_list": submissionIdList})
def startGenerationJob(self, submission_id, file_type):
""" Initiates a file generation job
Args:
submission_id: ID of submission to start job for
file_type: Type of file to be generated
Returns:
Tuple of boolean indicating successful start, and error response if False
"""
jobDb = self.interfaces.jobDb
file_type_name = self.fileTypeMap[file_type]
if file_type in ["D1", "D2"]:
# Populate start and end dates, these should be provided in MM/DD/YYYY format, using calendar year (not fiscal year)
requestDict = RequestDictionary(self.request)
start_date = requestDict.getValue("start")
end_date = requestDict.getValue("end")
if not (StringCleaner.isDate(start_date) and StringCleaner.isDate(end_date)):
exc = ResponseException("Start or end date cannot be parsed into a date", StatusCode.CLIENT_ERROR)
return False, JsonResponse.error(exc, exc.status, start = "", end = "", file_type = file_type, status = "failed")
elif file_type not in ["E","F"]:
exc = ResponseException("File type must be either D1, D2, E or F", StatusCode.CLIENT_ERROR)
return False, JsonResponse.error(exc, exc.status, file_type = file_type, status = "failed")
cgac_code = self.jobManager.getSubmissionById(submission_id).cgac_code
# Generate and upload file to S3
user_id = LoginSession.getName(session)
timestamped_name = s3UrlHandler.getTimestampedFilename(CONFIG_BROKER["".join([str(file_type_name),"_file_name"])])
if self.isLocal:
upload_file_name = "".join([CONFIG_BROKER['broker_files'], timestamped_name])
else:
upload_file_name = "".join([str(user_id), "/", timestamped_name])
job = jobDb.getJobBySubmissionFileTypeAndJobType(submission_id, file_type_name, "file_upload")
job.filename = upload_file_name
job.original_filename = timestamped_name
job.job_status_id = jobDb.getJobStatusId("running")
jobDb.session.commit()
if file_type in ["D1", "D2"]:
CloudLogger.log("DEBUG: Adding job info for job id of " + str(job.job_id),
log_type="debug",
file_name=self.debug_file_name)
return self.addJobInfoForDFile(upload_file_name, timestamped_name, submission_id, file_type, file_type_name, start_date, end_date, cgac_code, job)
elif file_type == 'E':
generate_e_file.delay(
submission_id, job.job_id, InterfaceHolder, timestamped_name,
upload_file_name, self.isLocal)
elif file_type == 'F':
generate_f_file.delay(
submission_id, job.job_id, InterfaceHolder, timestamped_name,
upload_file_name, self.isLocal)
return True, None
def checkSubmissionPermission(self,submission):
""" Check if current user has permisson to access submission and return user object.
Args:
submission - Submission model object
"""
userId = LoginSession.getName(session)
user = self.interfaces.userDb.getUserByUID(userId)
# Check that user has permission to see this submission, user must be within the agency of the submission
if(submission.cgac_code != user.cgac_code and submission.user_id != user.user_id):
raise ResponseException("User does not have permission to view that submission",StatusCode.CLIENT_ERROR)
return user
def list_user_emails(self):
""" List user names and emails """
sess = GlobalDB.db().session
user = sess.query(User).filter(User.user_id == LoginSession.getName(flaskSession)).one()
try:
users = sess.query(User).filter(User.cgac_code == user.cgac_code, User.user_status_id == USER_STATUS_DICT["approved"], User.is_active == True).all()
except ValueError as exc:
# Client provided a bad status
return JsonResponse.error(exc, StatusCode.CLIENT_ERROR)
user_info = []
for user in users:
this_info = {"id":user.user_id, "name": user.name, "email": user.email}
user_info.append(this_info)
return JsonResponse.create(StatusCode.OK, {"users": user_info})
def listUsers(self):
""" List all users ordered by status. Associated request body must have key 'filter_by' """
requestDict = RequestDictionary(self.request, optionalRequest=True)
user_status = requestDict.getValue("status") if requestDict.exists(
"status") else "all"
user = self.interfaces.userDb.getUserByUID(
LoginSession.getName(flaskSession))
isAgencyAdmin = self.userManager.hasPermission(
user, "agency_admin") and not self.userManager.hasPermission(
user, "website_admin")
try:
if isAgencyAdmin:
users = self.interfaces.userDb.getUsers(
cgac_code=user.cgac_code, status=user_status)
else:
users = self.interfaces.userDb.getUsers(status=user_status)
except ValueError as e:
# Client provided a bad status
exc = ResponseException(str(e), StatusCode.CLIENT_ERROR,
ValueError)
return JsonResponse.error(exc, exc.status)
userInfo = []
for user in users:
agency_name = self.interfaces.validationDb.getAgencyName(
user.cgac_code)
thisInfo = {
"name":
user.name,
"title":
user.title,
"agency_name":
agency_name,
"cgac_code":
user.cgac_code,
"email":
user.email,
"id":
user.user_id,
"is_active":
user.is_active,
"permissions":
",".join(self.interfaces.userDb.getUserPermissions(user)),
"status":
user.user_status.name
}
userInfo.append(thisInfo)
return JsonResponse.create(StatusCode.OK, {"users": userInfo})
def checkSubmissionPermission(self,submission):
""" Check if current user has permisson to access submission and return user object.
Args:
submission - Submission model object
"""
userId = LoginSession.getName(session)
user = self.interfaces.userDb.getUserByUID(userId)
# Check that user has permission to see this submission, user must be within the agency of the submission, or be
# the original user, or be in the 'SYS' agency
submissionCgac = StringCleaner.cleanString(submission.cgac_code)
userCgac = StringCleaner.cleanString(user.cgac_code)
if(submissionCgac != userCgac and submission.user_id != user.user_id
and userCgac != "sys"):
raise ResponseException("User does not have permission to view that submission",
StatusCode.PERMISSION_DENIED)
return user
def listSubmissionsByCurrentUserAgency(self):
""" List all submission IDs associated with the current user's agency """
userId = LoginSession.getName(flaskSession)
user = self.interfaces.userDb.getUserByUID(userId)
submissions = self.interfaces.jobDb.getSubmissionsByUserAgency(user)
submissionDetails = []
for submission in submissions:
jobIds = self.interfaces.jobDb.getJobsBySubmission(
submission.submission_id)
total_size = 0
for jobId in jobIds:
file_size = self.interfaces.jobDb.getFileSize(jobId)
total_size += file_size if file_size is not None else 0
status = self.interfaces.jobDb.getSubmissionStatus(
submission.submission_id, self.interfaces)
error_count = self.interfaces.errorDb.sumNumberOfErrorsForJobList(
jobIds, self.interfaces.validationDb)
if submission.user_id is None:
submission_user_name = "No user"
else:
submission_user_name = self.interfaces.userDb.getUserByUID(
submission.user_id).name
submissionDetails.append({
"submission_id":
submission.submission_id,
"last_modified":
submission.updated_at.strftime('%m/%d/%Y'),
"size":
total_size,
"status":
status,
"errors":
error_count,
"reporting_start_date":
str(submission.reporting_start_date),
"reporting_end_date":
str(submission.reporting_end_date),
"user": {
"user_id": submission.user_id,
"name": submission_user_name
}
})
return JsonResponse.create(StatusCode.OK,
{"submissions": submissionDetails})
def listAgencies(self):
""" Retrieves a list of all agency names and their cgac codes. If there is
a user logged in, it will check if that user is part of the 'SYS' agency.
If so, 'SYS' will be added to the agency_list. """
agencies = self.validationManager.getAllAgencies()
agency_list = []
for agency in agencies:
agency_list.append({
"agency_name": agency.agency_name,
"cgac_code": agency.cgac_code
})
if LoginSession.isLogin(session):
user_id = LoginSession.getName(session)
user = self.userManager.getUserByUID(user_id)
if user.cgac_code.lower() == "sys":
agency_list.append({"agency_name": "SYS", "cgac_code": "SYS"})
return JsonResponse.create(StatusCode.OK,
{"cgac_agency_list": agency_list})
def listUsers(self):
""" List all users ordered by status. Associated request body must have key 'filter_by' """
user = self.interfaces.userDb.getUserByUID(LoginSession.getName(flaskSession))
isAgencyAdmin = True if self.interfaces.userDb.hasPermission(user, "agency_admin") else False
try:
if isAgencyAdmin:
users = self.interfaces.userDb.getUsers(cgac_code=user.cgac_code)
else:
users = self.interfaces.userDb.getUsers()
except ValueError as e:
# Client provided a bad status
exc = ResponseException(str(e),StatusCode.CLIENT_ERROR,ValueError)
return JsonResponse.error(exc,exc.status)
userInfo = []
for user in users:
agency_name = self.interfaces.validationDb.getAgencyName(user.cgac_code)
thisInfo = {"name":user.name, "title":user.title, "agency_name":agency_name, "cgac_code":user.cgac_code,
"email":user.email, "id":user.user_id, "is_active":user.is_active,
"permissions": ",".join(self.interfaces.userDb.getUserPermissions(user)), "status": user.user_status.name}
userInfo.append(thisInfo)
return JsonResponse.create(StatusCode.OK,{"users":userInfo})
def listSubmissionsByCurrentUser(self):
""" List all submission IDs associated with the current user ID """
userId = LoginSession.getName(flaskSession)
user = self.interfaces.userDb.getUserByUID(userId)
submissions = self.interfaces.jobDb.getSubmissionsByUserId(userId)
submissionDetails = []
for submission in submissions:
jobIds = self.interfaces.jobDb.getJobsBySubmission(submission.submission_id)
total_size = 0
for jobId in jobIds:
file_size = self.interfaces.jobDb.getFileSize(jobId)
total_size += file_size if file_size is not None else 0
status = self.interfaces.jobDb.getSubmissionStatus(submission.submission_id)
error_count = self.interfaces.errorDb.sumNumberOfErrorsForJobList(jobIds)
submissionDetails.append(
{"submission_id": submission.submission_id, "last_modified": submission.updated_at.strftime('%m/%d/%Y'),
"size": total_size, "status": status, "errors": error_count, "reporting_start_date": str(submission.reporting_start_date),
"reporting_end_date": str(submission.reporting_end_date), "user": {"user_id": str(userId),
"name": user.name}})
return JsonResponse.create(StatusCode.OK, {"submissions": submissionDetails})
def list_agencies(self):
""" Retrieves a list of all agency names and their cgac codes. If there is
a user logged in, it will check if that user is part of the 'SYS' agency.
If so, 'SYS' will be added to the agency_list. """
sess = GlobalDB.db().session
agencies = sess.query(CGAC).all()
agency_list = []
for agency in agencies:
agency_list.append({
"agency_name": agency.agency_name,
"cgac_code": agency.cgac_code
})
if LoginSession.isLogin(session):
user = sess.query(User).filter(
User.user_id == LoginSession.getName(session)).one()
if user.cgac_code.lower() == "sys":
agency_list.append({"agency_name": "SYS", "cgac_code": "SYS"})
return JsonResponse.create(StatusCode.OK,
{"cgac_agency_list": agency_list})
def listUserEmails(self):
""" List user names and emails """
user = self.interfaces.userDb.getUserByUID(
LoginSession.getName(flaskSession))
try:
users = self.interfaces.userDb.getUsers(cgac_code=user.cgac_code,
status="approved",
only_active=True)
except ValueError as e:
# Client provided a bad status
exc = ResponseException(str(e), StatusCode.CLIENT_ERROR,
ValueError)
return JsonResponse.error(exc, exc.status)
userInfo = []
for user in users:
thisInfo = {
"id": user.user_id,
"name": user.name,
"email": user.email
}
userInfo.append(thisInfo)
return JsonResponse.create(StatusCode.OK, {"users": userInfo})
def submit_files():
fileManager = FileHandler(request,
isLocal=IS_LOCAL,
serverPath=SERVER_PATH)
return fileManager.submit(LoginSession.getName(session),
CreateCredentials)
