def create_study():
if request.method == 'GET':
studies = [
study.as_native_python()
for study in Study.get_all_studies_by_name()
]
return render_template('create_study.html',
studies=json.dumps(studies),
allowed_studies=get_admins_allowed_studies(),
system_admin=admin_is_system_admin())
name = request.form.get('name', '')
encryption_key = request.form.get('encryption_key', '')
is_test = request.form.get(
'is_test') == 'true' # 'true' -> True, 'false' -> False
try:
study = Study.create_with_object_id(name=name,
encryption_key=encryption_key,
is_test=is_test)
copy_existing_study_if_asked_to(study)
flash('Successfully created study {}.'.format(name), 'success')
return redirect('/device_settings/{:d}'.format(study.pk))
except ValidationError as ve:
for field, message in ve.message_dict.iteritems():
flash('{}: {}'.format(field, message[0]), 'danger')
return redirect('/create_study')
def create_study():
# ONLY THE SITE ADMIN CAN CREATE NEW STUDIES.
if not get_session_researcher().site_admin:
return abort(403)
if request.method == 'GET':
studies = [
study.as_native_python()
for study in Study.get_all_studies_by_name()
]
return render_template(
'create_study.html',
studies=json.dumps(studies),
allowed_studies=get_researcher_allowed_studies(),
is_admin=researcher_is_an_admin())
name = request.form.get('name', '')
encryption_key = request.form.get('encryption_key', '')
is_test = request.form.get(
'is_test') == 'true' # 'true' -> True, 'false' -> False
assert len(name) <= 2**16, "safety check on new study name failed"
try:
study = Study.create_with_object_id(name=name,
encryption_key=encryption_key,
is_test=is_test)
copy_existing_study_if_asked_to(study)
flash('Successfully created study {}.'.format(name), 'success')
return redirect('/device_settings/{:d}'.format(study.pk))
except ValidationError as ve:
for field, message in ve.message_dict.iteritems():
flash('{}: {}'.format(field, message[0]), 'danger')
return redirect('/create_study')
def get_researcher_allowed_studies_as_query_set():
session_researcher = get_session_researcher()
if session_researcher.site_admin:
return Study.get_all_studies_by_name()
return Study.get_all_studies_by_name().filter(
id__in=session_researcher.study_relations.values_list("study", flat=True)
)
def test_s3_upload(self):
study = Study(object_id='0vsvxgyx5skpI0ndOSAk1Duf',
encryption_key='aabbccddefggiijjkklmnoppqqrrsstt',
name='TEST_STUDY_FOR_TESTS')
study.save()
test_data = "THIS IS TEST DATA"
s3_upload("test_file_for_tests.txt", test_data, study.object_id)
s3_data = s3_retrieve("test_file_for_tests.txt", study.object_id)
self.assertEqual(s3_data, test_data)
def create_dummy_study(object_id):
s = DStudy(
name="Old Unnamed Study %s" % object_id,
encryption_key="0"*32,
object_id=str(object_id),
deleted=True,
)
s.save()
study_id_dict[object_id] = {"pk": s.id}
return s
def test_get_all_studies_by_name(self):
study_names = ['My studies', 'MY STUDY', 'my_study', 'your study']
encryption_key = 'aabbccddeeffgghhiijjkkllmmnnoopp'
for name in study_names:
good_study = Study.create_with_object_id(name=name, encryption_key=encryption_key)
self.assertIn(good_study, Study.get_all_studies_by_name())
self.assertEqual(list(Study.get_all_studies_by_name().values_list('name', flat=True)), study_names)
bad_study = Study.create_with_object_id(name='name', encryption_key=encryption_key, deleted=True)
self.assertNotIn(bad_study, Study.get_all_studies_by_name())
def test_settings_mongo_integrity(self):
study = Study(**self.translated_reference_study)
study.save()
mongo_reference_settings = self.translated_reference_device_settings
mongo_reference_settings['study'] = study
django_settings = DeviceSettings(**mongo_reference_settings)
x = compare_dictionaries(mongo_reference_settings,
django_settings.as_dict(),
ignore=['deleted', 'id'])
self.assertTrue(x)
def test_participant_mongo_integrity(self):
study = Study(**self.translated_reference_study)
study.save()
reference_participant = self.translated_reference_participant
django_participant = Participant(study=study,
**reference_participant).as_unpacked_native_python()
x = compare_dictionaries(reference_participant,
django_participant,
ignore=['id', 'deleted'])
self.assertTrue(x)
def test_survey_mongo_integrity(self):
study = Study(**self.translated_reference_study)
study.save()
mongo_reference_survey = self.translated_reference_survey
mongo_reference_survey['study'] = study
django_reference_survey = Survey(study=study, **self.translated_reference_survey)
# This comparison requires the as_dict comparison because it has jsonfields
x = compare_dictionaries(mongo_reference_survey,
django_reference_survey.as_dict(),
ignore=['deleted', 'id', 'last_modified'])
self.assertTrue(x)
def edit_researcher(researcher_pk):
researcher = Researcher.objects.get(pk=researcher_pk)
admin_is_current_user = (researcher.username == session['admin_username'])
current_studies = Study.get_all_studies_by_name().filter(researchers=researcher)
return render_template(
'edit_researcher.html',
admin=researcher,
current_studies=current_studies,
all_studies=Study.get_all_studies_by_name(),
allowed_studies=get_admins_allowed_studies(),
admin_is_current_user=admin_is_current_user,
system_admin=admin_is_system_admin(),
redirect_url='/edit_researcher/{:s}'.format(researcher_pk),
)
def test_study_create_with_object_id(self):
self.assertEqual(Study.objects.count(), 0)
self.assertEqual(DeviceSettings.objects.count(), 0)
study_name = 'my study'
encryption_key = 'aabbccddeeffgghhiijjkkllmmnnoopp'
Study.create_with_object_id(name=study_name, encryption_key=encryption_key)
new_study = Study.objects.get()
new_ds = DeviceSettings.objects.get()
self.assertEqual(Study.objects.count(), 1)
self.assertEqual(DeviceSettings.objects.count(), 1)
self.assertEqual(new_study.name, study_name)
self.assertEqual(new_study.encryption_key, encryption_key)
self.assertEqual(len(new_study.object_id), 24)
self.assertEqual(new_study.device_settings, new_ds)
self.assertFalse(new_study.deleted)
def setup(self, researcher=True, apikey=True, study=True):
"""
This function is used to initialize that database for each test.
This was written in this fashion because of a known issue with the HybridTest class that does not consistently
use database changes that persist between tests
"""
if apikey and not researcher:
raise Exception("invalid setup criteria")
if researcher:
self.researcher = Researcher.create_with_password(
username=TEST_USERNAME, password=TEST_PASSWORD)
if apikey:
self.api_key = ApiKey.generate(self.researcher,
has_tableau_api_permissions=True)
self.api_key_public = self.api_key.access_key_id
self.api_key_private = self.api_key.access_key_secret_plaintext
if study:
self.study = Study.create_with_object_id(
device_settings=DeviceSettings(),
encryption_key=TEST_STUDY_ENCRYPTION_KEY,
name=TEST_STUDY_NAME,
)
if researcher:
self.study_relation = StudyRelation(
study=self.study,
researcher=self.researcher,
relationship="researcher",
).save()
def manage_studies():
studies = [study.as_native_python() for study in Study.get_all_studies_by_name()]
return render_template(
'manage_studies.html',
studies=json.dumps(studies),
allowed_studies=get_admins_allowed_studies(),
system_admin=admin_is_system_admin()
)
def get_administerable_studies_by_name():
""" Site admins see all studies, study admins see only studies they are admins on. """
researcher_admin = get_session_researcher()
if researcher_admin.site_admin:
studies = Study.get_all_studies_by_name()
else:
studies = researcher_admin.get_administered_studies_by_name()
return studies
def manage_researchers():
researcher_list = []
for researcher in Researcher.get_all_researchers_by_username():
allowed_studies = Study.get_all_studies_by_name().filter(researchers=researcher).values_list('name', flat=True)
researcher_list.append((researcher.as_native_python(), list(allowed_studies)))
return render_template(
'manage_researchers.html',
admins=json.dumps(researcher_list),
allowed_studies=get_admins_allowed_studies(),
system_admin=admin_is_system_admin()
)
def get_researcher_allowed_studies() -> List[Dict]:
"""
Return a list of studies which the currently logged-in researcher is authorized to view and edit.
"""
session_researcher = get_session_researcher()
kwargs = {}
if not session_researcher.site_admin:
kwargs = dict(study_relations__researcher=session_researcher)
return [
study_info_dict for study_info_dict in
Study.get_all_studies_by_name().filter(**kwargs).values("name", "object_id", "id", "is_test")
]
def create_study():
# Only a SITE admin can create new studies.
if not get_session_researcher().site_admin:
return abort(403)
if request.method == 'GET':
studies = [
study.as_unpacked_native_python()
for study in Study.get_all_studies_by_name()
]
return render_template('create_study.html', studies=studies)
name = request.form.get('name', '')
encryption_key = request.form.get('encryption_key', '')
is_test = request.form.get(
'is_test', "").lower() == 'true' # 'true' -> True, 'false' -> False
duplicate_existing_study = request.form.get('copy_existing_study',
None) == 'true'
if not (len(name) <= 2**16) or escape(name) != name:
raise Exception("safety check on new study name failed")
try:
new_study = Study.create_with_object_id(name=name,
encryption_key=encryption_key,
is_test=is_test)
if duplicate_existing_study:
old_study = Study.objects.get(
pk=request.form.get('existing_study_id', None))
copy_existing_study(new_study, old_study)
flash(f'Successfully created study {name}.', 'success')
return redirect('/device_settings/{:d}'.format(new_study.pk))
except ValidationError as ve:
# display message describing failure based on the validation error (hacky, but works.)
for field, message in ve.message_dict.items():
flash(f'{field}: {message[0]}', 'danger')
return redirect('/create_study')
def migrate_studies():
d_study_list = []
for m_study in MStudySet.iterator():
with error_handler:
# Create a Django Study object modeled off the Mongolia Study
study_name = m_study['name']
d_study = DStudy(
name=study_name,
encryption_key=m_study['encryption_key'],
object_id=m_study['_id'],
deleted=m_study['deleted'],
is_test=m_study['is_test'],
)
# Validate the new Study object and add it to the bulk create list
d_study.full_clean()
d_study_list.append(d_study)
# Get lists of Mongolia Surveys, Admins and StudyDeviceSettings attached to this Study
m_survey_list = m_study['surveys']
m_admin_list = m_study['admins']
m_device_settings = m_study['device_settings']
study_referents[study_name] = {
'survey_list': m_survey_list,
'admin_list': m_admin_list,
'device_settings': m_device_settings,
}
# Bulk create the Django Studies
DStudy.objects.bulk_create(d_study_list)
# Create a reference from Mongolia Study IDs to Django Studies that doesn't require
# any future database calls.
for m_study in MStudySet.iterator():
with error_handler:
m_study_id = m_study['_id']
d_study_id = DStudy.objects.filter(name=m_study['name']).values('pk', 'deleted').get()
study_id_dict[m_study_id] = d_study_id
def choose_study():
allowed_studies = get_admins_allowed_studies_as_query_set()
# If the admin is authorized to view exactly 1 study, redirect to that study
if allowed_studies.count() == 1:
return redirect('/view_study/{:d}'.format(
allowed_studies.values_list('pk', flat=True).get()))
# Otherwise, show the "Choose Study" page
allowed_studies_json = Study.query_set_as_native_json(allowed_studies)
return render_template('choose_study.html',
studies=allowed_studies_json,
allowed_studies=allowed_studies_json,
system_admin=admin_is_system_admin())
def dashboard_page(study_id):
""" information for the general dashboard view for a study"""
study = Study.get_or_404(pk=study_id)
participants = list(Participant.objects.filter(study=study_id).values_list("patient_id", flat=True))
return render_template(
'dashboard/dashboard.html',
study=study,
participants=participants,
study_id=study_id,
data_stream_dict=COMPLETE_DATA_STREAM_DICT,
allowed_studies=get_researcher_allowed_studies(),
is_admin=researcher_is_an_admin(),
page_location='dashboard_landing',
)
def manage_researchers():
# get the study names that each user has access to, but only those that the current admin also
# has access to.
session_ids = get_session_researcher_study_ids()
researcher_list = []
for researcher in get_administerable_researchers():
allowed_studies = Study.get_all_studies_by_name().filter(
study_relations__researcher=researcher,
study_relations__study__in=session_ids,
).values_list('name', flat=True)
researcher_list.append(
(researcher.as_unpacked_native_python(), list(allowed_studies)))
return render_template('manage_researchers.html', admins=researcher_list)
def manage_researchers():
researcher_list = []
# get the study names that each user has access to
for researcher in get_administerable_researchers():
allowed_studies = list(Study.get_all_studies_by_name().filter(
study_relations__researcher=researcher).values_list('name',
flat=True))
researcher_list.append(
(researcher.as_native_python(), allowed_studies))
return render_template('manage_researchers.html',
admins=json.dumps(researcher_list),
allowed_studies=get_researcher_allowed_studies(),
is_admin=researcher_is_an_admin())
def get_admins_allowed_studies(as_json=True):
"""
Return a list of studies which the currently logged-in researcher is authorized to view and edit.
"""
researcher = Researcher.objects.get(username=session['admin_username'])
study_set = [
study for study in Study.get_all_studies_by_name().filter(
researchers=researcher).values("name", "object_id", "id",
"is_test")
]
if as_json:
return json.dumps(study_set)
else:
return study_set
def test_study_validation(self):
study_name = 'my study'
good_encryption_key = 'aabbccddeeffgghhiijjkkllmmnnoopp'
short_encryption_key = 'aabbccddeeffgghhiijjkkllmm'
long_encryption_key = 'aabbccddeeffgghhiijjkkllmmnnooppqqrrsstt'
with self.assertRaises(ValidationError):
Study.create_with_object_id(name=study_name, encryption_key=short_encryption_key)
with self.assertRaises(ValidationError):
Study.create_with_object_id(name=study_name, encryption_key=long_encryption_key)
bad_object_id = 'I am too long to be an ObjectID'
with self.assertRaises(ValidationError):
Study.objects.create(name=study_name, encryption_key=good_encryption_key, object_id=bad_object_id)
Study.create_with_object_id(name=study_name, encryption_key=good_encryption_key)
with self.assertRaises(ValidationError):
Study.create_with_object_id(name=study_name, encryption_key=good_encryption_key)
def edit_researcher(researcher_pk):
edit_researcher = Researcher.objects.get(pk=researcher_pk)
is_session_researcher = edit_researcher.username == get_session_researcher(
).username,
return render_template(
'edit_researcher.html',
admin=edit_researcher,
current_studies=Study.get_all_studies_by_name().filter(
study_relations__researcher=edit_researcher),
all_studies=get_administerable_studies(),
allowed_studies=get_researcher_allowed_studies(),
is_session_researcher=is_session_researcher,
is_admin=researcher_is_an_admin(),
redirect_url='/edit_researcher/{:s}'.format(researcher_pk),
)
def get_researcher_allowed_studies(as_json=True):
"""
Return a list of studies which the currently logged-in researcher is authorized to view and edit.
"""
session_researcher = get_session_researcher()
kwargs = {}
if not session_researcher.site_admin:
kwargs = dict(study_relations__researcher=session_researcher)
study_set = [
study for study in
Study.get_all_studies_by_name().filter(**kwargs).values("name", "object_id", "id", "is_test")
]
if as_json:
return json.dumps(study_set)
else:
return study_set
def test_all_routes(self):
"""
Tests urls
"""
app2 = subdomain("frontend")
app2.register_blueprint(admin_pages.admin_pages)
long_encryption_key = 'aabbccddefggiijjkklmnoppqqrrsstt'
researcher = Researcher.create_with_password('test_user',
'test_password')
researcher.admin = True
researcher.reset_access_credentials()
researcher.save()
study = Study.create_with_object_id(name='test_study',
encryption_key=long_encryption_key)
researcher.studies.add(study)
self.selenium.get("localhost:54321")
self.selenium.find_element_by_name('username').send_keys('test_user')
self.selenium.find_element_by_name('password').send_keys(
'test_password')
self.selenium.find_element_by_name('submit').click()
for rule in app2.url_map.iter_rules():
str_rule = str(rule)
self.assertIn(str_rule, ADMIN_PAGES)
if ADMIN_PAGES[str_rule]['method'] == 'get':
self.selenium.get("localhost:54321" + str_rule)
elif ADMIN_PAGES[str_rule]['method'] == 'post':
continue
elif ADMIN_PAGES[str_rule]['method'] == 'get_param':
str_rule_formatted = re.sub(r"<\w+:\w+>", str(study.id),
str_rule)
self.selenium.get("localhost:54321" + str_rule_formatted)
else:
continue
response = self.determine_errors()
self.assertEqual(response, '200')
def get_data_for_dashboard_datastream_display(study_id, data_stream):
""" Parses information for the data stream dashboard view GET and POST requests left the post
and get requests in the same function because the body of the get request relies on the
variables set in the post request if a post request is sent --thus if a post request is sent
we don't want all of the get request running. """
study = Study.get_or_404(pk=study_id)
if request.method == "POST":
color_low_range, color_high_range, all_flags_list = set_default_settings_post_request(
study, data_stream)
show_color = "false" if color_low_range == 0 and color_high_range == 0 else "true"
else:
color_low_range, color_high_range, show_color = extract_range_args_from_request(
)
all_flags_list = extract_flag_args_from_request()
default_filters = ""
if DashboardColorSetting.objects.filter(data_type=data_stream,
study=study).exists():
settings = DashboardColorSetting.objects.get(data_type=data_stream,
study=study)
default_filters = DashboardColorSetting.get_dashboard_color_settings(
settings)
else:
settings = None
# -------------------------------- dealing with color settings -------------------------------------------------
# test if there are default settings saved,
# and if there are, test if the default filters should be used or if the user has overridden them
if default_filters != "":
inflection_info = default_filters["inflections"]
if all_flags_list == [] and color_high_range is None and color_low_range is None:
# since none of the filters are set, parse default filters to pass in the default
# settings set the values for gradient filter
# backend: color_range_min, color_range_max --> frontend: color_low_range,
# color_high_range the above is consistent throughout the back and front ends
if settings.gradient_exists():
gradient_info = default_filters["gradient"]
color_low_range = gradient_info["color_range_min"]
color_high_range = gradient_info["color_range_max"]
show_color = "true"
else:
color_high_range = 0
color_low_range = 0
show_color = "false"
# set the values for the flag/inflection filter*s*
# the html is expecting a list of lists for the flags [[operator, value], ... ]
all_flags_list = []
for flag_info in inflection_info:
single_flag = [
flag_info["operator"], flag_info["inflection_point"]
]
all_flags_list.append(single_flag)
# change the url params from jinja t/f to python understood T/F
show_color = True if show_color == "true" else False
# ----------------------------------- general data fetching --------------------------------------------
start, end = extract_date_args_from_request()
participant_objects = Participant.objects.filter(
study=study_id).order_by("patient_id")
unique_dates = []
next_url = ""
past_url = ""
byte_streams = {}
data_exists = None
# --------------------- decide whether data is in Processed DB or Bytes DB -----------------------------
if data_stream in ALL_DATA_STREAMS:
first_day, last_day = dashboard_chunkregistry_date_query(
study_id, data_stream)
if first_day is not None:
stream_data = OrderedDict(
(participant.patient_id,
dashboard_chunkregistry_query(participant.id,
data_stream=data_stream))
for participant in participant_objects)
unique_dates, _, _ = get_unique_dates(start, end, first_day,
last_day)
next_url, past_url = create_next_past_urls(first_day,
last_day,
start=start,
end=end)
# get the byte streams per date for each patient for a specific data stream for those dates
byte_streams = OrderedDict((participant.patient_id, [
get_bytes_participant_match(
stream_data[participant.patient_id], date)
for date in unique_dates
]) for participant in participant_objects)
# check if there is data to display
data_exists = len([
data for patient in byte_streams
for data in byte_streams[patient] if data is not None
]) > 0
else:
start, end = extract_date_args_from_request()
first_day, last_day, stream_data = parse_processed_data(
study_id, participant_objects, data_stream)
if first_day is not None:
unique_dates, _, _ = get_unique_dates(start, end, first_day,
last_day)
next_url, past_url = create_next_past_urls(first_day,
last_day,
start=start,
end=end)
# get the byte streams per date for each patient for a specific data stream for those dates
byte_streams = OrderedDict((participant.patient_id, [
get_bytes_processed_data_match(
stream_data[participant.patient_id], date)
for date in unique_dates
]) for participant in participant_objects)
# check if there is data to display
data_exists = len([
data for patient in byte_streams
for data in byte_streams[patient] if data is not None
]) > 0
# ---------------------------------- base case if there is no data ------------------------------------------
if first_day is None or (not data_exists and past_url == ""):
unique_dates = []
next_url = ""
past_url = ""
byte_streams = {}
return render_template(
'dashboard/data_stream_dashboard.html',
study=study,
data_stream=COMPLETE_DATA_STREAM_DICT.get(data_stream),
times=unique_dates,
byte_streams=byte_streams,
base_next_url=next_url,
base_past_url=past_url,
study_id=study_id,
data_stream_dict=COMPLETE_DATA_STREAM_DICT,
color_low_range=color_low_range,
color_high_range=color_high_range,
first_day=first_day,
last_day=last_day,
show_color=show_color,
all_flags_list=all_flags_list,
allowed_studies=get_researcher_allowed_studies(),
is_admin=researcher_is_an_admin(),
page_location='dashboard_data',
)
def get_admins_allowed_studies_as_query_set():
researcher = Researcher.objects.get(username=session['admin_username'])
return Study.get_all_studies_by_name().filter(researchers=researcher)
def test_study_mongo_integrity(self):
django_reference_study = Study(**self.translated_reference_study)
x = compare_dictionaries(self.translated_reference_study,
django_reference_study.as_unpacked_native_python(),
ignore=['id'])
self.assertTrue(x)
