def create_submission(self, survey, user):
session = object_session(survey)
program = survey.program
submission = model.Submission(program=program,
organisation=user.organisation,
survey=survey,
title="First submission",
approval='draft')
session.add(submission)
for m in program.measures:
# Preload response type to avoid autoflush
response_type = m.response_type
qnode_measure = m.get_qnode_measure(survey)
if not qnode_measure:
continue
response = model.Response(qnode_measure=qnode_measure,
submission=submission,
user=user)
response.attachments = []
response.not_relevant = False
response.modified = datetime.datetime.utcnow()
response.approval = 'final'
response.comment = "Response for %s" % m.title
session.add(response)
if response_type.name == 'Yes / No':
response.response_parts = [{'index': 1, 'note': "Yes"}]
elif response_type.name in {
'Numerical', 'External Numerical', 'Planned', 'Actual'
}:
response.response_parts = [{'value': 1}]
else:
raise ValueError("Unknown response type")
response.attachments.append(
model.Attachment(file_name="File %s 1" % m.title,
url="Bar",
storage='external',
organisation=user.organisation))
response.attachments.append(
model.Attachment(file_name="File %s 2" % m.title,
url="Baz",
storage='external',
organisation=user.organisation))
response.attachments.append(
model.Attachment(file_name="File %s 3" % m.title,
blob=b'A blob',
storage='external',
organisation=user.organisation))
session.flush()
calculator = Calculator.scoring(submission)
calculator.mark_entire_survey_dirty(submission.survey)
calculator.execute()
submission.approval = 'final'
session.flush()
return submission
def put(self, submission_id, measure_id, submeasure_id):
son = self.request_son
externals = son["externals"]
with model.session_scope() as session:
user_session = self.get_user_session(session)
response = (
session.query(model.Response)
.get((submission_id, measure_id)))
if response is None:
raise errors.MissingDocError("No such response")
org = response.submission.organisation
policy = user_session.policy.derive({
'org': org,
'surveygroups': org.surveygroups,
})
policy.verify('surveygroup_interact')
policy.verify('attachment_add')
for external in externals:
url = external.get('url', '').strip()
file_name = external.get('file_name', '').strip()
if url == '' and file_name == '':
continue
if url == '':
raise errors.ModelError(
"URL required for link '%s'" % file_name)
if measure_id!=submeasure_id:
attachment = model.Attachment(
organisation=response.submission.organisation,
response=response,
url=url,
file_name=file_name,
submeasure_id=submeasure_id,
storage='external'
)
else:
attachment = model.Attachment(
organisation=response.submission.organisation,
response=response,
url=url,
file_name=file_name,
storage='external'
)
session.add(attachment)
self.get(submission_id, measure_id, submeasure_id)
def add_attachments(db, user_id, project_id, document_id, attachments):
user = db.query(model.User).filter(model.User.id == user_id).first()
if user is None:
raise QueryException("Authentication failed")
project, access = get_project_access(db, project_id, user_id, ACCESS_WRITE)
document = db.query(model.Document).filter(
(model.Document.id == document_id)
& (model.Document.project_id == project_id)).first()
if document is None:
raise QueryException("Invalid document")
# ensure existence of user specific asset dir
root = os.path.abspath(os.path.join(config.ASSETS, user_id, "attachments"))
if not os.path.exists(root):
os.makedirs(root)
# each attachment
total_size = 0
saved = []
try:
for attachment in attachments:
# check size is ok
item = model.Attachment(project=project,
document=document,
name=attachment["name"],
size=attachment["size"],
location="server")
db.add(item)
db.flush() # get item id
# save to filesystem
target_filename = os.path.join(root, item.id)
decoded = base64.b64decode(attachment["content"])
if len(decoded) > config.MAX_ATTACHMENT_SIZE:
raise QueryException("File too large")
total_size += len(decoded)
if user.category == 'free' and total_size > config.MAX_FREE_USER_STORAGE:
raise QueryException("Quota exceeded")
with open(target_filename, "wb") as fh:
fh.write(decoded)
saved.append(target_filename)
except:
# deal with already saved files if an exception occurs
for filename in saved:
os.remove(filename)
raise
user.storage_used += total_size
db.commit()
def import_project(db, user_id, name, data, attachment_data):
user = db.query(model.User).filter(model.User.id == user_id).first()
if user is None:
raise QueryException("Authentication failed")
# for attachments
root = os.path.abspath(os.path.join(config.ASSETS, user_id, "attachments"))
if not os.path.exists(root):
os.makedirs(root)
project = model.Project(name=name, renderer='Markdown',
owner=user) # renderer
db.add(project)
id_map = {}
ids_added = set()
queue = [x for x in data['documents']]
last_added = 0
stats = {'documents': 0, 'attachments': 0}
while len(queue) > 0:
item = queue.pop(0)
if item['id'] not in id_map:
id_map[item['id']] = model.generate_id()
if (item['parent_id'] is None or item['parent_id']
in ids_added) and (item['predecessor_id'] is None
or item['predecessor_id'] in ids_added):
document = model.Document(project=project,
id=id_map.get(item['id']),
name=item['name'],
parent_id=id_map.get(item['parent_id']),
predecessor_id=id_map.get(
item['predecessor_id']),
document_type=item['document_type'],
renderer=item['renderer'],
content=item['content'],
updated=item['updated'],
rating=item['rating'])
db.add(document)
stats['documents'] += 1
ids_added.add(item['id'])
last_added = 0
for attachment in item['attachments']:
new_attachment_id = model.generate_id()
# extract and save zipped file to new id
target_filename = os.path.join(root, new_attachment_id)
file_data = attachment_data.open(attachment['id'], 'r').read()
with open(target_filename, "wb") as fh:
fh.write(file_data)
new_attachment = model.Attachment(id=new_attachment_id,
project=project,
document=document,
name=attachment["name"],
size=len(file_data),
location="server")
db.add(new_attachment)
stats['attachments'] += 1
else:
queue.append(item)
last_added += 1
if last_added > 2 * len(queue):
raise QueryException('Circular document definition')
db.commit()
return stats
def post(self, submission_id, measure_id,submeasure_id):
fileinfo = self.request.files['file'][0]
with model.session_scope() as session:
user_session = self.get_user_session(session)
response = (
session.query(model.Response)
.get((submission_id, measure_id)))
if response is None:
raise errors.MissingDocError("No such response")
org = response.submission.organisation
policy = user_session.policy.derive({
'org': org,
'surveygroups': org.surveygroups,
})
policy.verify('surveygroup_interact')
policy.verify('attachment_add')
if aws.session is not None:
s3 = aws.session.resource('s3', verify=False)
bucket = os.environ.get('AWS_BUCKET')
hex_key = hashlib.sha256(bytes(fileinfo['body'])).hexdigest()
s3_path = "{0}/{1}".format(
response.submission.organisation_id, hex_key)
# Metadata can not contain non-ASCII characters - so encode
# higher Unicode characters :/
# https://github.com/boto/boto3/issues/478#issuecomment-180608544
file_name_enc = (
fileinfo["filename"]
.encode('ascii', errors='backslashreplace')
.decode('ascii')
[:1024])
try:
s3.Bucket(bucket).put_object(
Key=s3_path,
Metadata={'filename': file_name_enc},
Body=bytes(fileinfo['body']))
except botocore.exceptions.ClientError as e:
raise errors.InternalModelError(
"Failed to write to data store", log_message=str(e))
if measure_id != submeasure_id:
attachment = model.Attachment(
organisation=response.submission.organisation,
response=response,
submeasure_id=submeasure_id,
file_name=fileinfo["filename"]
)
else:
attachment = model.Attachment(
organisation=response.submission.organisation,
response=response,
file_name=fileinfo["filename"]
)
if aws.session is not None:
attachment.storage = "aws"
aws_url = aws.s3_url.format(
region=aws.region_name,
bucket=bucket,
s3_path=s3_path)
attachment.url = aws_url
else:
attachment.storage = "database"
attachment.blob = bytes(fileinfo['body'])
session.add(attachment)
session.flush()
attachment_id = str(attachment.id)
self.set_header("Content-Type", "text/plain")
self.write(attachment_id)
self.finish()
def _instantiate_attachment(self, article, zendesk_attachment):
attachment = model.Attachment(article, zendesk_attachment['file_name'])
attachment.meta = zendesk_attachment
return attachment