def read_all(self, sources, extension_map=None):
text = ""
for source in sources:
if source.startswith('/'):
source = os.path.join(self.root, source[1:])
else:
source = os.path.join(self.root, source)
source = resolve_file(source, extension_map)
source = os.path.abspath(source)
if not source.startswith(self.root):
raise errors.InternalModelError(
"Resource configuration is invalid",
log_message="%s is not in %s" % (source, self.root))
with open(source, 'r', encoding='utf8') as f:
text += f.read()
text += "\n"
return text
def process_submission_file(self, all_rows, session, submission, user):
program_qnodes = (session.query(
model.QuestionNode).filter_by(program_id=submission.program.id))
try:
order = title = ''
for row_num in range(0, len(all_rows) - 1):
order, title = self.parse_order_title(all_rows, row_num, "A")
function = program_qnodes.filter_by(parent_id=None,
title=title).one()
log.debug("function: %s", function)
function_order = order
order, title = self.parse_order_title(all_rows, row_num, "B")
process = program_qnodes.filter_by(parent_id=function.id,
title=title).one()
log.debug("process: %s", process)
order, title = self.parse_order_title(all_rows, row_num, "C")
subprocess = program_qnodes.filter_by(parent_id=process.id,
title=title).one()
log.debug("subprocess: %s", subprocess)
order, title = self.parse_order_title(all_rows, row_num, "D")
measure = [
qm.measure for qm in subprocess.qnode_measures
if qm.measure.title.split('\n')[0] == title
]
if len(measure) == 1:
measure = measure[0]
else:
raise Exception(
"This survey does not match the target survey.")
log.debug("measure: %s", measure)
log.debug("measure response_type: %s",
measure.response_type.name)
response = model.Response()
response.program_id = submission.program.id
response.survey_id = submission.survey.id
response.measure_id = measure.id
response.submission_id = submission.id
response.user_id = user.id
response.comment = all_rows[row_num][col2num("K")]
# FIXME: Hard-coded; should be read from file
response.not_relevant = False
response.modified = datetime.datetime.utcnow()
response.approval = 'draft'
response_part = []
response_part.append(
self.parse_response_type(all_rows, row_num,
measure.response_type, "E"))
if function_order != "7":
response_part.append(
self.parse_response_type(all_rows, row_num,
measure.response_type, "F"))
response_part.append(
self.parse_response_type(all_rows, row_num,
measure.response_type, "G"))
response_part.append(
self.parse_response_type(all_rows, row_num,
measure.response_type, "H"))
response.response_parts = response_part
response.audit_reason = "Import"
session.add(response)
except sqlalchemy.orm.exc.NoResultFound:
raise errors.ModelError(
"Survey structure does not match: Row %d: %s %s" %
(row_num + 2, order, title))
except ImportError as e:
raise errors.ModelError("Row %d: %s %s: %s" %
(row_num + 2, order, title, str(e)))
except Exception as e:
raise errors.InternalModelError(
"Row %d: %s %s: %s" % (row_num + 2, order, title, str(e)))
calculator = Calculator.scoring(submission)
calculator.mark_entire_survey_dirty(submission.survey)
calculator.execute()
def post(self, submission_id, measure_id,submeasure_id):
fileinfo = self.request.files['file'][0]
with model.session_scope() as session:
user_session = self.get_user_session(session)
response = (
session.query(model.Response)
.get((submission_id, measure_id)))
if response is None:
raise errors.MissingDocError("No such response")
org = response.submission.organisation
policy = user_session.policy.derive({
'org': org,
'surveygroups': org.surveygroups,
})
policy.verify('surveygroup_interact')
policy.verify('attachment_add')
if aws.session is not None:
s3 = aws.session.resource('s3', verify=False)
bucket = os.environ.get('AWS_BUCKET')
hex_key = hashlib.sha256(bytes(fileinfo['body'])).hexdigest()
s3_path = "{0}/{1}".format(
response.submission.organisation_id, hex_key)
# Metadata can not contain non-ASCII characters - so encode
# higher Unicode characters :/
# https://github.com/boto/boto3/issues/478#issuecomment-180608544
file_name_enc = (
fileinfo["filename"]
.encode('ascii', errors='backslashreplace')
.decode('ascii')
[:1024])
try:
s3.Bucket(bucket).put_object(
Key=s3_path,
Metadata={'filename': file_name_enc},
Body=bytes(fileinfo['body']))
except botocore.exceptions.ClientError as e:
raise errors.InternalModelError(
"Failed to write to data store", log_message=str(e))
if measure_id != submeasure_id:
attachment = model.Attachment(
organisation=response.submission.organisation,
response=response,
submeasure_id=submeasure_id,
file_name=fileinfo["filename"]
)
else:
attachment = model.Attachment(
organisation=response.submission.organisation,
response=response,
file_name=fileinfo["filename"]
)
if aws.session is not None:
attachment.storage = "aws"
aws_url = aws.s3_url.format(
region=aws.region_name,
bucket=bucket,
s3_path=s3_path)
attachment.url = aws_url
else:
attachment.storage = "database"
attachment.blob = bytes(fileinfo['body'])
session.add(attachment)
session.flush()
attachment_id = str(attachment.id)
self.set_header("Content-Type", "text/plain")
self.write(attachment_id)
self.finish()
def process_tabular(self, file_name, program_id, survey_id, submission_id,
user_role, base_url):
"""
Open and write an Excel file
"""
workbook = xlsxwriter.Workbook(file_name)
worksheet = workbook.add_worksheet('Response')
worksheet_metadata = workbook.add_worksheet('Metadata')
format = workbook.add_format()
format.set_text_wrap()
format_no_wrap = workbook.add_format()
format_comment = workbook.add_format()
format_comment.set_text_wrap()
format_no_wrap = workbook.add_format()
format_percent = workbook.add_format()
format_percent.set_num_format(10)
format_int = workbook.add_format()
format_int.set_num_format(1)
format_date = workbook.add_format({'num_format': 'dd/mmm/yy'})
format_2_decimal = workbook.add_format({'num_format': '0.00'})
url_format = workbook.add_format({
'font_color': 'blue',
'underline': 1
})
line = 1
with model.session_scope() as session:
survey = (session.query(model.Survey).get((survey_id, program_id)))
if not survey:
raise errors.MissingDocError("No such survey")
if not 'levels' in survey.structure:
raise errors.InternalModelError("Survey is misconfigured")
if submission_id:
submission = (session.query(
model.Submission).get(submission_id))
if submission.survey_id != survey.id:
raise errors.MissingDocError(
"That submission does not belong to that survey")
self.write_metadata(workbook, worksheet_metadata, submission)
else:
submission = None
levels = survey.structure["levels"]
level_length = len(levels)
worksheet.set_column(0, level_length, 50)
measures = [qm.measure for qm in survey.ordered_qnode_measures]
max_parts = 0
longest_response_type = None
for m in measures:
if len(m.response_type.parts) > max_parts:
longest_response_type = m.response_type
max_parts = len(m.response_type.parts)
if longest_response_type:
response_parts = [
p.get('name') or "Part %s" % string.ascii_uppercase[i]
for i, p in enumerate(longest_response_type.parts)
]
else:
response_parts = []
worksheet.set_column(level_length + 1,
level_length + max_parts + 11, 15)
worksheet.set_column(level_length + max_parts + 12,
level_length + max_parts + 12, 100)
# Header from heirarchy levels
self.write_response_header(workbook, worksheet, levels, max_parts,
response_parts)
for measure in measures:
qnode_measure = measure.get_qnode_measure(survey)
self.write_qnode(worksheet, qnode_measure.qnode, line, format,
level_length - 1)
seq = qnode_measure.seq + 1
worksheet.write(line, level_length,
"%d. %s" % (seq, measure.title), format)
importance = None
urgency = None
if submission:
response = model.Response.from_measure(
qnode_measure, submission)
url = base_url + "/#/3/measure/{}?submission={}".format(
measure.id, submission.id)
# Walk up the tree to get the importance and target from the
# parent rnodes
parent = qnode_measure.qnode
while parent and (importance is None or urgency is None):
rnode = model.ResponseNode.from_qnode(
parent, submission)
if rnode is not None:
if importance is None:
importance = rnode.importance
if urgency is None:
urgency = rnode.urgency
parent = parent.parent
else:
response = None
url = base_url + '/#/3/measure/{}?program={}&survey={}'.format(
measure.id, program_id, qnode_measure.survey_id)
worksheet.write(line, level_length + max_parts + 1, importance,
format_int)
worksheet.write(line, level_length + max_parts + 2, urgency,
format_int)
score = None
comment = ''
quality = None
self.comment = ''
if response:
if not response.not_relevant:
self.write_response_parts(worksheet,
response.response_parts,
line, format_no_wrap,
level_length + 1)
export_approval_status = ['final', 'reviewed', 'approved']
self.write_approval(worksheet, line,
level_length + max_parts + 2, response,
response.user, format, format_date)
if response.approval in export_approval_status:
export_approval_status.remove(response.approval)
for approval_status in export_approval_status:
res = (session.query(model.ResponseHistory).filter_by(
submission_id=response.submission_id,
measure_id=response.measure_id,
approval=approval_status).order_by(
model.ResponseHistory.modified.desc()).first())
if res:
user = session.query(model.AppUser).\
filter_by(id=res.user_id).\
first()
self.write_approval(worksheet, line,
level_length + max_parts + 2,
res, user, format, format_date)
if user_role != 'clerk':
if response.measure.weight != 0:
score = response.score #/ response.measure.weight
else:
score = 0
if response.comment or submission_id:
comment = response.comment + '; '
quality = response.quality
if user_role in {'clerk', 'org_admin'}:
weight = None
else:
weight = measure.weight
worksheet.write(line, level_length + max_parts + 9, score,
format_2_decimal)
# score, format_percent)
worksheet.write(line, level_length + max_parts + 10, weight,
format_no_wrap)
worksheet.write(line, level_length + max_parts + 11, quality,
format_no_wrap)
worksheet.write(line, level_length + max_parts + 12,
comment + self.comment, format_comment)
#comment, format_comment)
worksheet.write_url(line, level_length + max_parts + 13, url,
url_format, "Link")
line = line + 1
workbook.close()