def create_reports(self):
(other, teammate, contact, captain, super_admin, team,
classroom) = self.create()
classReport1 = Report.create(
team_id=team.uid,
classroom_id=classroom.uid,
filename='foo.pdf',
gcs_path='/mybucket/upload/12345.pdf',
size=1000000,
content_type='application/pdf',
preview=False,
)
classReport2 = Report.create(
team_id=team.uid,
classroom_id=classroom.uid,
filename='bar.pdf',
gcs_path='/mybucket/upload/23456.pdf',
size=1000000,
content_type='application/pdf',
preview=False,
)
teamReport = Report.create(
team_id=team.uid,
classroom_id=None,
filename='team.pdf',
gcs_path='/mybucket/upload/34567.pdf',
size=1000000,
content_type='application/pdf',
preview=False,
)
Report.put_multi((classReport1, classReport2, teamReport))
return (other, teammate, contact, captain, super_admin, team,
classroom, classReport1, classReport2, teamReport)
def create_dataset_reports(self):
(other, teammate, contact, captain, super_admin, team,
classroom) = self.create()
classReport1 = Report.create(
team_id=team.uid,
classroom_id=classroom.uid,
filename='foo.html',
dataset_id='Dataset_class1',
template='class_template',
content_type='text/html',
preview=False,
)
classReport2 = Report.create(
team_id=team.uid,
classroom_id=classroom.uid,
filename='bar.html',
dataset_id='Dataset_class2',
template='class_template',
content_type='text/html',
preview=False,
)
teamReport = Report.create(
team_id=team.uid,
classroom_id=None,
filename='team.html',
dataset_id='Dataset_team',
template='team_template',
content_type='text/html',
preview=False,
)
Report.put_multi((classReport1, classReport2, teamReport))
return (other, teammate, contact, captain, super_admin, team,
classroom, classReport1, classReport2, teamReport)
def test_rserve_skips_existing(self):
program = Program.create(
name="The Engagement Project",
label="ep19",
preview_url='foo.com',
)
week = util.datelike_to_iso_string(datetime.date.today())
org = Organization.create(name="Organization",
captain_id="User_cap",
program_id=program.uid)
org_to_skip = Organization.create(name="Organization",
captain_id="User_cap",
program_id=program.uid)
Organization.put_multi([org, org_to_skip])
team = Team.create(name="Team",
captain_id="User_cap",
program_id=program.uid)
team_to_skip = Team.create(name="Team",
captain_id="User_cap",
program_id=program.uid)
Team.put_multi([team, team_to_skip])
cl = Classroom.create(name="Classroom",
team_id=team.uid,
code="foo",
contact_id="User_contact")
cl_to_skip = Classroom.create(name="Classroom",
team_id=team.uid,
code="foo",
contact_id="User_contact")
Classroom.put_multi([cl, cl_to_skip])
Report.put_multi([
Report.create(parent_id=org_to_skip.uid,
filename="foo",
issue_date=week),
Report.create(parent_id=team_to_skip.uid,
filename="foo",
issue_date=week),
Report.create(parent_id=cl_to_skip.uid,
filename="foo",
issue_date=week),
])
# Skips all the parents who have reports already this week.
orgs, teams, classes = cron_rserve.get_report_parents(
program, week, False)
self.assertEqual(len(orgs), 1)
self.assertEqual(len(teams), 1)
self.assertEqual(len(classes), 1)
# ...unless you force it, then they're all there.
orgs, teams, classes = cron_rserve.get_report_parents(
program, week, True)
self.assertEqual(len(orgs), 2)
self.assertEqual(len(teams), 2)
self.assertEqual(len(classes), 2)
def save_report(self, *ignore):
text_buffer = self.report.get_buffer()
report_data = text_buffer.get_text(
text_buffer.get_start_iter(),
text_buffer.get_end_iter(),
False)
with database.transaction():
Report.create(
report=report_data,
created=self.datetime_create
)
self.quit(self)
def test_get_pdf_token_allowed(self):
"""You don't need to be logged in at all, if your token is right."""
(other, teammate, contact, captain, super_admin, team, classroom,
report_dict) = self.test_post_team_pdf()
# Put in a classroom report for the same team so we know we can
# correctly select the team-level one.
classReport = Report.create(
team_id=team.uid,
classroom_id=classroom.uid,
filename='classroom.pdf',
gcs_path='/mybucket/upload/classroom.pdf',
size=1000000,
content_type='application/pdf',
)
classReport.put()
path = '/api/teams/{team_id}/reports/{filename}'.format(
team_id=team.uid,
filename=report_dict['filename'],
)
endpoint_str = util.get_endpoint_str(method='GET', path=path)
jwt = jwt_helper.encode({'allowed_endpoints': [endpoint_str]}),
url = util.set_query_parameters(path, token=jwt)
response = self.testapp.get(url) # asserts 200
self.assert_pdf_response(report_dict, response)
def test_list_network_reports(self):
network = Network.create(name="Foo Net", program_id="Program_foo")
network.put()
admin = User.create(email="*****@*****.**",
owned_networks=[network.uid])
admin.put()
template = 'network_tempate'
filename = 'foo.html'
report = Report.create(
network_id=network.uid,
filename=filename,
dataset_id='Dataset_class1',
template=template,
content_type='text/html',
preview=False,
)
report.put()
response = self.testapp.get(
'/api/networks/{}/reports'.format(network.uid),
headers=self.login_headers(admin),
)
reports = json.loads(response.body)
self.assertEqual(
urlparse(reports[0]['link']).path,
'/datasets/{}/{}/{}'.format(report.uid, template, filename),
)
url, token = reports[0]['link'].split('?token=')
payload, error = jwt_helper.decode(token)
self.assertIn('GET //neptune' + urlparse(url).path,
payload['allowed_endpoints'])
def create_for_delete(self):
"""Should delete the classroom and associated reports."""
contact = User.create(name='contact', email='*****@*****.**')
captain = User.create(name='foo', email='*****@*****.**')
team = Team.create(name='Team Foo',
captain_id=captain.uid,
program_id=self.program.uid)
team.put()
captain.owned_teams = [team.uid]
contact.owned_teams = [team.uid]
User.put_multi([captain, contact])
classroom = Classroom.create(
name='Classroom Foo',
code='trout viper',
team_id=team.uid,
contact_id=contact.uid,
num_students=22,
grade_level='9-12',
)
classroom.put()
report1 = Report.create(
team_id=team.uid,
classroom_id=classroom.uid,
filename='report1.pdf',
gcs_path='/upload/abc',
size=10,
content_type='application/pdf',
)
report1.put()
report2 = Report.create(
team_id=team.uid,
classroom_id=classroom.uid,
filename='report2.pdf',
gcs_path='/upload/def',
size=10,
content_type='application/pdf',
)
report2.put()
return (captain, contact, classroom, report1, report2)
def test_list_preview(self):
"""Preview reports only visible to super admins."""
(other, teammate, contact, captain, super_admin, team,
classroom) = self.create()
classReport1 = Report.create(
team_id=team.uid,
classroom_id=classroom.uid,
filename='foo.pdf',
gcs_path='/mybucket/upload/12345.pdf',
size=1000000,
content_type='application/pdf',
preview=True,
)
teamReport = Report.create(
team_id=team.uid,
classroom_id=None,
filename='team.pdf',
gcs_path='/mybucket/upload/34567.pdf',
size=1000000,
content_type='application/pdf',
preview=True,
)
Report.put_multi((classReport1, teamReport))
# Contact and captain see an empty list.
for user in (contact, captain):
response = self.testapp.get(
'/api/teams/{}/reports'.format(team.uid),
headers=self.login_headers(user),
)
self.assertEqual(response.body, '[]')
# Super sees preview reports.
response = self.testapp.get(
'/api/teams/{}/reports'.format(team.uid),
headers=self.login_headers(super_admin),
)
self.assertEqual(
set(d['uid'] for d in json.loads(response.body)),
{teamReport.uid, classReport1.uid},
)
class Base:
def __init__(self, database):
self.model = Report(database)
def on_get(self, req, resp):
if authorize_as(req.auth, 'player'):
resp.body = dumps(self.model.all())
else:
raise HTTPUnauthorized('unauthorized', 'unauthorized')
def on_post(self, req, resp):
if authorize_as(req.auth, 'player'):
body = loads(req.stream.read().decode('utf-8'))
created = self.model.create(body)
resp.status = HTTP_201
resp.body = dumps({'id': created.inserted_id})
else:
raise HTTPUnauthorized('unauthorized', 'unauthorized')
def test_delete_removes_related(self):
user = User.create(name='foo', email='*****@*****.**')
team = Team.create(name='Team Foo', captain_id=user.uid,
program_id=self.demo_program.uid)
team.put()
survey = Survey.create(team_id=team.uid)
survey.put()
user.owned_teams = [team.uid]
user.put()
classroom1 = Classroom.create(
name='Classroom One',
code='trout viper',
team_id=team.uid,
contact_id='User_contact',
num_students=22,
grade_level='9-12',
)
classroom1.put()
classroom2 = Classroom.create(
name='Classroom Two',
code='trout viper',
team_id=team.uid,
contact_id='User_contact',
num_students=22,
grade_level='9-12',
)
classroom2.put()
report1 = Report.create(
team_id=team.uid,
classroom_id=classroom1.uid,
filename='report1.pdf',
gcs_path='/upload/abc',
size=10,
content_type='application/pdf',
)
report1.put()
report2 = Report.create(
team_id=team.uid,
classroom_id=classroom2.uid,
filename='report2.pdf',
gcs_path='/upload/def',
size=10,
content_type='application/pdf',
)
report2.put()
url = '/api/teams/{}'.format(team.uid)
headers = self.login_headers(user)
# Delete the team.
self.testapp.delete(url, headers=headers, status=204)
# Expect the survey, classrooms, and related reports are gone from the
# db.
self.assertIsNone(Survey.get_by_id(survey.uid))
self.assertIsNone(Classroom.get_by_id(classroom1.uid))
self.assertIsNone(Classroom.get_by_id(classroom2.uid))
self.assertIsNone(Report.get_by_id(report1.uid))
self.assertIsNone(Report.get_by_id(report2.uid))
def post(self):
"""Save references to reports, and perhaps the report file itself.
Has two modes: accept a file, in which case the file is saved to GCS, or
a dataset id. In both cases a Report is inserted referencing the
file/dataset.
Either `team_id` or `classroom_id` must be provided. If team_id, then
the report's classroom_id is empty (these are "team-level" reports). If
classroom_id, the corresponding team_id is looked up (these are
"classroom-level" reports).
"""
# Allow RServe to call this endpoint, then fall back on regular auth.
user, error = self.authenticate_rserve()
if not user:
user = self.get_current_user()
error = ''
# Replaces function of `requires_auth = True`.
if user.user_type == 'public':
return self.http_unauthorized()
if not user.super_admin:
return self.http_forbidden()
org_id = self.get_param('organization_id', str, None)
team_id = self.get_param('team_id', str, None)
classroom_id = self.get_param('classroom_id', str, None)
params = self.get_params({
'dataset_id': str,
'filename': unicode,
'issue_date': str,
'notes': unicode,
'preview': bool,
'template': str,
})
# Lookup related objects.
classroom = None
if classroom_id:
classroom = Classroom.get_by_id(classroom_id)
if not classroom:
return self.http_bad_request(
"Classroom not found: {}".format(classroom_id))
team_id = classroom.team_id
team = None
if team_id:
# May be set for team reports, or via lookup for class reports.
team = Team.get_by_id(team_id)
if not team:
return self.http_bad_request(
"Team not found: {}".format(team_id))
org = None
if org_id:
org = Organization.get_by_id(org_id)
if not org:
return self.http_bad_request(
"Organization not found: {}".format(org_id))
content_type = self.request.headers['Content-Type']
is_form = 'multipart/form-data' in content_type
is_json = 'application/json' in content_type
if is_form:
report = self.save_file(
params['filename'],
self.request.POST['file'],
org_id,
team_id,
classroom_id,
)
elif is_json:
kwargs = {
'classroom_id': classroom_id,
'dataset_id': params['dataset_id'],
'filename': params['filename'],
'issue_date': params.get('issue_date', None),
'notes': params.get('notes', None),
'organization_id': org_id,
'team_id': team_id,
'template': params['template'],
}
# Some params we may want to avoid including at all, so that if
# they're absent they'll take the default value defined in the db.
if params.get('preview', None) is not None:
kwargs['preview'] = params['preview']
report = Report.create(**kwargs)
else:
return self.http_bad_request(
"Only supported content types are 1) multipart/form-data for "
"uploading files and 2) application/json for datasets. Got {}".
format(self.request.headers['Content-Type']))
saved_report = Report.put_for_index(report, 'parent-file')
self.write(saved_report)
def save_file(self,
filename,
field_storage,
org_id=None,
team_id=None,
classroom_id=None):
"""GCS files are saved by their md5 hash, so uploading the same file
many times has no effect (other than io). Uploading a different file to
the same classroom and filename changes the reference on the report
object in the db, but doesn't delete previous uploads in cloud storage.
A history of uploads for a given report can be found by searching the
upload bucket for files with the header 'x-goog-meta-task-id:[classroom
uid]'.
Filenames as uploaded are preserved:
* in the Content-Disposition header of the GCS file
* in the report object, which has properties gcs_path, filename,
content_type
To avoid file names getting out of sync, only the one in the report
object is actually used.
"""
# Which variant of report, team- or classroom-level.
meta_headers = {}
if org_id:
meta_headers['x-goog-meta-organization-id'] = org_id
if team_id:
meta_headers['x-goog-meta-team-id'] = team_id
if classroom_id:
meta_headers['x-goog-meta-classroom-id'] = classroom_id
if len(meta_headers) == 0:
return self.http_bad_request("Missing ids.")
# FieldStorageClass object has potentially useful properties:
# file: cStringIO.StringO object
# headers: rfc822.Message instance
# type: str, MIME type, e.g. 'application/pdf'
# filename: str, file name as uploaded
file_contents = field_storage.file.read()
file_hash = hashlib.md5(file_contents).hexdigest()
field_storage.file.seek(0, os.SEEK_END)
file_size = field_storage.file.tell()
content_type = field_storage.type
gcs_path = '/{}{}/{}'.format(
util.get_upload_bucket(),
os.environ['GCS_UPLOAD_PREFIX'],
file_hash,
)
open_kwargs = {
'content_type':
content_type,
# These will be headers on the stored file.
'options':
dict(
{
# Not actually used, but seems smart to keep track of.
'Content-Disposition': 'attachment; filename=' + filename,
},
# Theoretically allows figuring out an attachment history for
# a given classroom.
**meta_headers),
'retry_params':
gcs.RetryParams(backoff_factor=1.1),
}
with gcs.open(gcs_path, 'w', **open_kwargs) as gcs_file:
gcs_file.write(file_contents)
report = Report.create(
organization_id=org_id,
team_id=team_id,
classroom_id=classroom_id,
filename=filename,
gcs_path=gcs_path,
size=file_size,
content_type=content_type,
)
return report
