class ParticipantSurvey(AbstractSurvey):
"""
Survey can now be participant specific!
Surveys contain all information the app needs to display the survey correctly to a participant,
and when it should push the notifications to take the survey.
Surveys must have a 'survey_type', which is a string declaring the type of survey it
contains, which the app uses to display the correct interface.
Surveys contain 'content', which is a JSON blob that is unpacked on the app and displayed
to the participant in the form indicated by the survey_type.
Timings schema: a survey must indicate the day of week and time of day on which to trigger;
by default it contains no values. The timings schema mimics the Java.util.Calendar.DayOfWeek
specification: it is zero-indexed with day 0 as Sunday. 'timings' is a list of 7 lists, each
inner list containing any number of times of the day. Times of day are integer values
indicating the number of seconds past midnight.
Inherits the following fields from AbstractSurvey
content
survey_type
settings
timings
"""
# This is required for file name and path generation
object_id = models.CharField(max_length=24,
unique=True,
validators=[LengthValidator(24)])
# the study field is not inherited because we need to change its related name
study = models.ForeignKey('Study',
on_delete=models.PROTECT,
related_name='participant_surveys')
participant = models.ForeignKey('Participant',
on_delete=models.PROTECT,
related_name='participant_surveys')
@classmethod
def create_with_object_id(cls, **kwargs):
object_id = cls.generate_objectid_string("object_id")
survey = cls.objects.create(object_id=object_id, **kwargs)
return survey
@classmethod
def create_with_settings(cls, survey_type, **kwargs):
"""
Create a new Survey with the provided survey type and attached to the given Study,
as well as any other given keyword arguments. If the Survey is audio/image and no other
settings are given, give it the default audio/image survey settings.
"""
if survey_type == cls.AUDIO_SURVEY and 'settings' not in kwargs:
kwargs['settings'] = json.dumps(AUDIO_SURVEY_SETTINGS)
elif survey_type == cls.IMAGE_SURVEY and 'settings' not in kwargs:
kwargs['settings'] = json.dumps(IMAGE_SURVEY_SETTINGS)
survey = cls.create_with_object_id(survey_type=survey_type, **kwargs)
return survey
class PipelineUpload(TimestampedModel):
# no related name, this is
object_id = models.CharField(max_length=24,
unique=True,
validators=[LengthValidator(24)])
study = models.ForeignKey(Study,
related_name="pipeline_uploads",
on_delete=models.PROTECT)
file_name = models.TextField()
s3_path = models.TextField()
file_hash = models.CharField(max_length=128)
class Survey(AbstractSurvey):
"""
Surveys contain all information the app needs to display the survey correctly to a participant,
and when it should push the notifications to take the survey.
Surveys must have a 'survey_type', which is a string declaring the type of survey it
contains, which the app uses to display the correct interface.
Surveys contain 'content', which is a JSON blob that is unpacked on the app and displayed
to the participant in the form indicated by the survey_type.
Timings schema: a survey must indicate the day of week and time of day on which to trigger;
by default it contains no values. The timings schema mimics the Java.util.Calendar.DayOfWeek
specification: it is zero-indexed with day 0 as Sunday. 'timings' is a list of 7 lists, each
inner list containing any number of times of the day. Times of day are integer values
indicating the number of seconds past midnight.
"""
# Every time that a Survey object is saved (except upon creation), a SurveyArchive object
# is created. The SurveyArchive object contains all of the AbstractSurvey fields that the
# Survey object did *before* it was saved, as well as a pair of timestamps marking during
# what time period the SurveyArchive applies. The code that creates SurveyArchive objects
# is found in database.signals.create_survey_archive.
last_modified = models.DateTimeField(auto_now=True)
# This is required for file name and path generation
object_id = models.CharField(max_length=24,
unique=True,
validators=[LengthValidator(24)])
study = models.ForeignKey('Study',
on_delete=models.PROTECT,
related_name='surveys')
@classmethod
def create_with_object_id(cls, **kwargs):
object_id = cls.generate_objectid_string("object_id")
survey = cls.objects.create(object_id=object_id, **kwargs)
return survey
@classmethod
def create_with_settings(cls, survey_type, **kwargs):
"""
Create a new Survey with the provided survey type and attached to the given Study,
as well as any other given keyword arguments. If the Survey is audio and no other
settings are given, give it the default audio survey settings.
"""
if survey_type == cls.AUDIO_SURVEY and 'settings' not in kwargs:
kwargs['settings'] = json.dumps(AUDIO_SURVEY_SETTINGS)
survey = cls.create_with_object_id(survey_type=survey_type, **kwargs)
return survey
class Study(AbstractModel):
# When a Study object is created, a default DeviceSettings object is automatically
# created alongside it. If the Study is created via the researcher interface (as it
# usually is) the researcher is immediately shown the DeviceSettings to edit. The code
# to create the DeviceSettings object is in database.signals.populate_study_device_settings.
name = models.TextField(
unique=True, help_text='Name of the study; can be of any length')
encryption_key = models.CharField(
max_length=32,
validators=[LengthValidator(32)],
help_text='Key used for encrypting the study data')
object_id = models.CharField(max_length=24,
unique=True,
validators=[LengthValidator(24)],
help_text='ID used for naming S3 files')
is_test = models.BooleanField(default=True)
@classmethod
def create_with_object_id(cls, **kwargs):
"""
Creates a new study with a populated object_id field
"""
study = cls(object_id=cls.generate_objectid_string("object_id"),
**kwargs)
study.save()
return study
@classmethod
def get_all_studies_by_name(cls):
"""
Sort the un-deleted Studies a-z by name, ignoring case.
"""
return (cls.objects.filter(deleted=False).annotate(name_lower=Func(
F('name'), function='LOWER')).order_by('name_lower'))
def get_surveys_for_study(self):
survey_json_list = []
for survey in self.surveys.all():
survey_dict = survey.as_native_python()
# Make the dict look like the old Mongolia-style dict that the frontend is expecting
survey_dict.pop('id')
survey_dict.pop('deleted')
survey_dict['_id'] = survey_dict.pop('object_id')
survey_json_list.append(survey_dict)
return survey_json_list
def get_survey_ids_for_study(self, survey_type='tracking_survey'):
return self.surveys.filter(survey_type=survey_type,
deleted=False).values_list('id', flat=True)
def get_survey_ids_and_object_ids_for_study(self,
survey_type='tracking_survey'):
return self.surveys.filter(survey_type=survey_type,
deleted=False).values_list(
'id', 'object_id')
def get_study_device_settings(self):
return self.device_settings
def get_researchers(self):
return Researcher.objects.filter(studies=self)
class PipelineUpload(AbstractModel):
REQUIREDS = [
"study_id",
"tags",
"file_name",
]
# no related name, this is
object_id = models.CharField(max_length=24,
unique=True,
validators=[LengthValidator(24)])
study = models.ForeignKey(Study, related_name="pipeline_uploads")
file_name = models.TextField()
s3_path = models.TextField()
file_hash = models.CharField(max_length=128)
@classmethod
def get_creation_arguments(cls, params, file_object):
errors = []
# ensure required are present, we don't allow falsey contents.
for field in PipelineUpload.REQUIREDS:
if not params.get(field, None):
errors.append('missing required parameter: "%s"' % field)
# if we escape here early we can simplify the code that requires all parameters later
if errors:
raise InvalidUploadParameterError("\n".join(errors))
# validate study_id
study_id_object_id = params["study_id"]
if not Study.objects.get(object_id=study_id_object_id):
errors.append('encountered invalid study_id: "%s"' %
params["study_id"] if params["study_id"] else None)
study_id = Study.objects.get(object_id=study_id_object_id).id
if len(params['file_name']) > 256:
errors.append(
"encountered invalid file_name, file_names cannot be more than 256 characters"
)
if cls.objects.filter(file_name=params['file_name']).count():
errors.append('a file with the name "%s" already exists' %
params['file_name'])
try:
tags = json.loads(params["tags"])
if not isinstance(tags, list):
# must be json list, can't be json dict, number, or string.
raise ValueError()
if not tags:
errors.append(
"you must provide at least one tag for your file.")
tags = [str(_) for _ in tags]
except ValueError:
tags = None
errors.append(
"could not parse tags, ensure that your uploaded list of tags is a json compatible array."
)
if errors:
raise InvalidUploadParameterError("\n".join(errors))
created_on = timezone.now()
file_hash = chunk_hash(file_object.read())
file_object.seek(0)
s3_path = "%s/%s/%s/%s/%s" % (
PIPELINE_FOLDER,
params["study_id"],
params["file_name"],
created_on.isoformat(),
''.join(
random.choice(string.ascii_letters + string.digits)
for i in range(32)),
# todo: file_name?
)
creation_arguments = {
"created_on": created_on,
"s3_path": s3_path,
"study_id": study_id,
"file_name": params["file_name"],
"file_hash": file_hash,
}
return creation_arguments, tags
class Study(AbstractModel):
# When a Study object is created, a default DeviceSettings object is automatically
# created alongside it. If the Study is created via the researcher interface (as it
# usually is) the researcher is immediately shown the DeviceSettings to edit. The code
# to create the DeviceSettings object is in database.signals.populate_study_device_settings.
name = models.TextField(
unique=True, help_text='Name of the study; can be of any length')
encryption_key = models.CharField(
max_length=32,
validators=[LengthValidator(32)],
help_text='Key used for encrypting the study data')
object_id = models.CharField(max_length=24,
unique=True,
validators=[LengthValidator(24)],
help_text='ID used for naming S3 files')
is_test = models.BooleanField(default=True)
@classmethod
def create_with_object_id(cls, **kwargs):
"""
Creates a new study with a populated object_id field
"""
study = cls(object_id=cls.generate_objectid_string("object_id"),
**kwargs)
study.save()
return study
@classmethod
def get_all_studies_by_name(cls):
"""
Sort the un-deleted Studies a-z by name, ignoring case.
"""
return (cls.objects.filter(deleted=False).annotate(name_lower=Func(
F('name'), function='LOWER')).order_by('name_lower'))
@classmethod
def _get_administered_studies_by_name(cls, researcher):
return cls.get_all_studies_by_name().filter(
study_relations__researcher=researcher,
study_relations__relationship=ResearcherRole.study_admin,
)
def get_surveys_for_study(self, requesting_os):
survey_json_list = []
for survey in self.surveys.filter(deleted=False):
survey_dict = survey.as_native_python()
# Make the dict look like the old Mongolia-style dict that the frontend is expecting
survey_dict.pop('id')
survey_dict.pop('deleted')
survey_dict['_id'] = survey_dict.pop('object_id')
# Exclude image surveys for the Android app to avoid crashing it
if requesting_os == "ANDROID" and survey.survey_type == "image_survey":
pass
else:
survey_json_list.append(survey_dict)
return survey_json_list
def get_survey_ids_for_study(self, survey_type='tracking_survey'):
return self.surveys.filter(survey_type=survey_type,
deleted=False).values_list('id', flat=True)
def get_survey_ids_and_object_ids_for_study(self,
survey_type='tracking_survey'):
return self.surveys.filter(survey_type=survey_type,
deleted=False).values_list(
'id', 'object_id')
def get_study_device_settings(self):
return self.device_settings
def get_researchers(self):
return Researcher.objects.filter(study_relations__study=self)
@classmethod
def get_researcher_studies_by_name(cls, researcher):
return cls.get_all_studies_by_name().filter(
study_relations__researcher=researcher)
def get_researchers_by_name(self):
return (Researcher.objects.filter(
study_relations__study=self).annotate(
name_lower=Func(F('username'), function='LOWER')).order_by(
'name_lower').exclude(
username__icontains="BATCH USER").exclude(
username__icontains="AWS LAMBDA"))
# We override the as_native_python function to not include the encryption key.
def as_native_python(self,
remove_timestamps=True,
remove_encryption_key=True):
ret = super(Study,
self).as_native_python(remove_timestamps=remove_timestamps)
ret.pop("encryption_key")
return ret
class Study(TimestampedModel):
# When a Study object is created, a default DeviceSettings object is automatically
# created alongside it. If the Study is created via the researcher interface (as it
# usually is) the researcher is immediately shown the DeviceSettings to edit. The code
# to create the DeviceSettings object is in database.signals.populate_study_device_settings.
name = models.TextField(
unique=True, help_text='Name of the study; can be of any length')
encryption_key = models.CharField(
max_length=32,
validators=[LengthValidator(32)],
help_text='Key used for encrypting the study data')
object_id = models.CharField(max_length=24,
unique=True,
validators=[LengthValidator(24)],
help_text='ID used for naming S3 files')
is_test = models.BooleanField(default=True)
timezone = TimeZoneField(default="America/New_York",
help_text='Timezone of the study')
deleted = models.BooleanField(default=False)
def save(self, *args, **kwargs):
""" Ensure there is a study device settings attached to this study. """
# First we just save. This code has vacillated between throwing a validation error and not
# during study creation. Our current fix is to save, then test whether a device settings
# object exists. If not, create it.
super().save(*args, **kwargs)
try:
self.device_settings
except ObjectDoesNotExist:
settings = DeviceSettings(study=self)
self.device_settings = settings
settings.save()
# update the study object to have a device settings object (possibly unnecessary?).
super().save(*args, **kwargs)
@classmethod
def create_with_object_id(cls, **kwargs):
""" Creates a new study with a populated object_id field. """
study = cls(object_id=cls.generate_objectid_string("object_id"),
**kwargs)
study.save()
return study
@classmethod
def get_all_studies_by_name(cls):
""" Sort the un-deleted Studies a-z by name, ignoring case. """
return (cls.objects.filter(deleted=False).annotate(name_lower=Func(
F('name'), function='LOWER')).order_by('name_lower'))
@classmethod
def _get_administered_studies_by_name(cls, researcher):
return cls.get_all_studies_by_name().filter(
study_relations__researcher=researcher,
study_relations__relationship=ResearcherRole.study_admin,
)
@classmethod
def get_researcher_studies_by_name(cls, researcher):
return cls.get_all_studies_by_name().filter(
study_relations__researcher=researcher)
def get_survey_ids_and_object_ids(self, survey_type='tracking_survey'):
return self.surveys.filter(survey_type=survey_type,
deleted=False).values_list(
'id', 'object_id')
def get_researchers(self):
return Researcher.objects.filter(study_relations__study=self)
# We override the as_unpacked_native_python function to not include the encryption key.
def as_unpacked_native_python(self, remove_timestamps=True):
ret = super().as_unpacked_native_python(
remove_timestamps=remove_timestamps)
ret.pop("encryption_key")
return ret
class Survey(SurveyBase):
"""
Surveys contain all information the app needs to display the survey correctly to a participant,
and when it should push the notifications to take the survey.
Surveys must have a 'survey_type', which is a string declaring the type of survey it
contains, which the app uses to display the correct interface.
Surveys contain 'content', which is a JSON blob that is unpacked on the app and displayed
to the participant in the form indicated by the survey_type.
Timings schema: a survey must indicate the day of week and time of day on which to trigger;
by default it contains no values. The timings schema mimics the Java.util.Calendar.DayOfWeek
specification: it is zero-indexed with day 0 as Sunday. 'timings' is a list of 7 lists, each
inner list containing any number of times of the day. Times of day are integer values
indicating the number of seconds past midnight.
Inherits the following fields from SurveyBase
content
survey_type
settings
timings
"""
# This is required for file name and path generation
object_id = models.CharField(max_length=24,
unique=True,
validators=[LengthValidator(24)])
# the study field is not inherited because we need to change its related name
study = models.ForeignKey('Study',
on_delete=models.PROTECT,
related_name='surveys')
@classmethod
def create_with_object_id(cls, **kwargs):
object_id = cls.generate_objectid_string("object_id")
survey = cls.objects.create(object_id=object_id, **kwargs)
return survey
@classmethod
def create_with_settings(cls, survey_type, **kwargs):
"""
Create a new Survey with the provided survey type and attached to the given Study,
as well as any other given keyword arguments. If the Survey is audio/image and no other
settings are given, give it the default audio/image survey settings.
"""
if survey_type == cls.AUDIO_SURVEY and 'settings' not in kwargs:
kwargs['settings'] = json.dumps(AUDIO_SURVEY_SETTINGS)
elif survey_type == cls.IMAGE_SURVEY and 'settings' not in kwargs:
kwargs['settings'] = json.dumps(IMAGE_SURVEY_SETTINGS)
survey = cls.create_with_object_id(survey_type=survey_type, **kwargs)
return survey
def weekly_timings(self):
"""
Returns a json serializable object that represents the weekly schedules of this survey.
The return object is a list of 7 lists of ints
"""
from database.schedule_models import WeeklySchedule
return WeeklySchedule.export_survey_timings(self)
def relative_timings(self):
"""
Returns a json serializable object that represents the relative schedules of the survey
The return object is a list of lists
"""
schedules = []
for schedule in self.relative_schedules.all():
num_seconds = schedule.minute * 60 + schedule.hour * 3600
schedules.append(
[schedule.intervention.id, schedule.days_after, num_seconds])
return schedules
def absolute_timings(self):
"""
Returns a json serializable object that represents the absolute schedules of the survey
The return object is a list of lists
"""
schedules = []
for schedule in self.absolute_schedules.all():
num_seconds = schedule.scheduled_date.minute * 60 + schedule.scheduled_date.hour * 3600
schedules.append([
schedule.scheduled_date.year, schedule.scheduled_date.month,
schedule.scheduled_date.day, num_seconds
])
return schedules
def format_survey_for_study(self):
"""
Returns a dict with the values of the survey fields for download to the app
"""
survey_dict = self.as_unpacked_native_python()
# Make the dict look like the old Mongolia-style dict that the frontend is expecting
survey_dict.pop('id')
survey_dict.pop('deleted')
survey_dict['_id'] = survey_dict.pop('object_id')
# the old timings object does need to be provided
from database.schedule_models import WeeklySchedule
survey_dict['timings'] = WeeklySchedule.export_survey_timings(self)
return survey_dict
def create_absolute_schedules_and_events(self):
from database.schedule_models import AbsoluteSchedule, ScheduledEvent
# todo: finish writing, this doesn't work, isn't used anywhere
for schedule in AbsoluteSchedule.objects.filter(
surevy=self).values_list("scheduled_date", flat=True):
year, month, day, seconds = schedule
hour = seconds // 3600
minute = seconds % 3600 // 60
schedule_date = datetime(year, month, day, hour, minute)
absolute_schedule = AbsoluteSchedule.objects.create(
survey=self,
scheduled_date=schedule_date,
)
for participant in self.study.participants.all():
ScheduledEvent.objects.create(
survey=self,
participant=participant,
weekly_schedule=None,
relative_schedule=None,
absolute_schedule=absolute_schedule,
scheduled_time=schedule_date,
)
def save(self, *args, **kwargs):
super().save(*args, **kwargs)
self.archive()
def most_recent_archive(self):
return self.archives.latest('archive_start')
def archive(self):
""" Create an archive if there were any changes to the data since the last archive was
created, or if no archive exists. """
# get self as dictionary representation, remove fields that don't exist, extract last
# updated and the survey id.
new_data = self.as_dict()
archive_start = new_data.pop("last_updated")
survey_id = new_data.pop("id")
new_data.pop("object_id")
new_data.pop("created_on")
new_data.pop("study")
# Get the most recent archive for this Survey, to check whether the Survey has been edited
try:
prior_archive = self.most_recent_archive().as_dict()
except SurveyArchive.DoesNotExist:
prior_archive = None
# if there was a prior archive identify if there were any changes, don't create an
# archive if there were no changes.
if prior_archive is not None:
if not any(prior_archive[shared_field_name] != shared_field_value
for shared_field_name, shared_field_value in
new_data.items()):
return
SurveyArchive(
**new_data,
survey_id=survey_id,
archive_start=archive_start,
).save()