def ddb_load_interview_questions(survey_id):
ddb_client = Dynamodb(stack_name=const.STACK_NAME)
return ddb_client.query(
table_name=const.INTERVIEW_QUESTIONS_TABLE["name"],
KeyConditionExpression="survey_id = :survey_id",
ExpressionAttributeValues={
":survey_id": survey_id,
},
)
def get_unassigned_links(account_survey_id: str,
ddb_client=None) -> list[dict]:
"""
Retrieves existing personal links that have not yet been assigned to an user
"""
if ddb_client is None:
ddb_client = Dynamodb(stack_name=const.STACK_NAME)
return ddb_client.query(
table_name=const.PersonalLinksTable.NAME,
IndexName="unassigned-links",
KeyConditionExpression="account_survey_id = :account_survey_id "
"AND #status = :link_status",
ExpressionAttributeValues={
":account_survey_id": account_survey_id,
":link_status": "new",
},
ExpressionAttributeNames={
"#status": "status"
}, # needed because status is a reserved word in ddb
)
def get_appointments_by_type(type_ids, correlation_id=None):
"""
Args:
type_ids (list): Appointment type ids to query ddb
correlation_id:
Returns:
List of appointments matching any of the input type ids
"""
ddb_client = Dynamodb(stack_name=STACK_NAME, correlation_id=correlation_id)
items = list()
for i in type_ids:
result = ddb_client.query(
table_name=APPOINTMENTS_TABLE,
IndexName="project-appointments-index",
KeyConditionExpression="appointment_type_id = :type_id",
ExpressionAttributeValues={
":type_id": i,
},
)
items += result
return items
class AppointmentsCleaner:
def __init__(self, logger=None, correlation_id=None):
self.ddb_client = Dynamodb(stack_name=STACK_NAME)
self.correlation_id = correlation_id
self.target_appointment_ids = self.get_appointments_to_be_deleted()
self.logger = logger
if logger is None:
self.logger = utils.get_logger()
def get_appointments_to_be_deleted(self, now=None):
"""
Queries ddb for appointments booked for 60 days ago
"""
if now is None:
now = utils.now_with_tz()
date_format = "%Y-%m-%d"
sixty_days_ago = now - datetime.timedelta(days=60)
sixty_days_ago_string = sixty_days_ago.strftime(date_format)
result = self.ddb_client.query(
table_name=APPOINTMENTS_TABLE,
IndexName="reminders-index",
KeyConditionExpression="appointment_date = :date",
ExpressionAttributeValues={
":date": sixty_days_ago_string,
},
)
return [x["id"] for x in result]
def delete_old_appointments(self):
results = list()
for app_id in self.target_appointment_ids:
result = self.ddb_client.delete_item(
table_name=APPOINTMENTS_TABLE,
key=app_id,
)
results.append(result["ResponseMetadata"]["HTTPStatusCode"])
return results
class SurveyInitialiser:
def __init__(self, **kwargs):
self._logger = utils.get_logger()
self._correlation_id = kwargs.get("correlation_id")
self.destination_survey_id = kwargs.get("survey_id")
self.destination_response_id = kwargs.get("response_id")
err_message = "Call to initialise_survey missing mandatory data: {}"
assert self.destination_survey_id, err_message.format("survey_id")
assert self.destination_survey_id, err_message.format("response_id")
self.survey_init_config = SurveyInitConfig(
destination_survey_id=self.destination_survey_id,
correlation_id=self._correlation_id,
)
self.survey_init_config.get()
try:
self.survey_init_config.details
except AttributeError:
raise utils.ObjectDoesNotExistError(
f"Initialisation config not found for survey {self.destination_survey_id}",
details={},
)
self.destination_account = kwargs.get("account", "cambridge")
self.anon_project_specific_user_id = kwargs.get(
"anon_project_specific_user_id")
assert self.anon_project_specific_user_id, err_message.format(
"anon_project_specific_user_id")
self._core_client = CoreApiClient(correlation_id=self._correlation_id)
user = self._core_client.get_user_by_anon_project_specific_user_id(
self.anon_project_specific_user_id)
self.user_id = user["id"]
self.ddb_client = Dynamodb(stack_name=const.STACK_NAME,
correlation_id=self._correlation_id)
self.cached_responses = dict()
self.missing_responses = list()
self.target_embedded_data = dict()
self.responses_client = ResponsesClient(
survey_id=self.destination_survey_id,
qualtrics_account_name=self.destination_account,
correlation_id=self._correlation_id,
)
def query_task_response(self, survey_id):
query_result = self.ddb_client.query(
table_name=const.TASK_RESPONSES_TABLE["name"],
IndexName="user-index",
KeyConditionExpression="user_id = :user_id "
"AND survey_id = :survey_id",
ExpressionAttributeValues={
":user_id": self.user_id,
":survey_id": survey_id,
},
)
if query_result:
response = query_result[0]["participant_responses"]
self.cached_responses[survey_id] = response
return response
else:
self.missing_responses.append(survey_id)
return None
def parse_init_config(self):
"""
Populates self.target_embedded_data with the best data available
"""
for embedded_field_name, v_list in self.survey_init_config.details.items(
):
for option_dict in v_list:
previous_survey_id = option_dict["survey"]
try:
# load already queried and cached previous survey response
source_response = self.cached_responses[previous_survey_id]
except KeyError:
if previous_survey_id in self.missing_responses:
# we already know this doesn't exist in Ddb, so no point querying
source_response = None
else:
# we haven't tried fetching this previous response yet
source_response = self.query_task_response(
previous_survey_id)
if source_response:
# we have a previous response; check it includes the question data we are looking for
source_data = source_response.get(option_dict["question"])
if source_data:
if isinstance(source_data, decimal.Decimal):
source_data = str(source_data)
elif isinstance(source_data, list):
source_data = [
int(x) for x in source_data
] # Qualtrics doesn't parse list of strings properly
source_data = json.dumps(source_data)
self.target_embedded_data[
embedded_field_name] = source_data
break # no need to attempt other options; else attempt next option_dict
def init_response(self):
return self.responses_client.update_response(
response_id=self.destination_response_id,
embedded_data=self.target_embedded_data,
)
def main(self):
self.parse_init_config()
return self.init_response()
def get_response(self):
self.parse_init_config()
return self.target_embedded_data
class ConsentEmailsManager(CsvImporter):
def __init__(self, csvfile_path, consent_questions_tags, consent_info_url):
super(ConsentEmailsManager, self).__init__(csvfile_path)
self.consent_question_tags = consent_questions_tags
self.consent_info_url = consent_info_url
self.consent_statements = dict()
self.ddb_client = Dynamodb(stack_name=const.STACK_NAME)
self.anon_user_task_id_column_name = "anon_user_task_id"
self.anon_project_specific_user_id_column_name = "anon_project_specific_user_id"
self.consent_datetime_column_name = "StartDate"
self.sent_emails = list()
def get_consent_statements(self):
if not self.consent_statements:
with open(self.input_filename) as csvfile:
reader = csv.DictReader(csvfile)
question_description_row = next(reader)
for tag in self.consent_question_tags:
self.consent_statements[
tag] = f"• {question_description_row[tag].split(' - ', 1)[1]}"
return self.consent_statements
def consent_in_ddb_table(self, anon_user_task_id):
items = self.ddb_client.query(
table_name=const.CONSENT_DATA_TABLE,
IndexName="user-task-index",
KeyConditionExpression="anon_user_task_id = :anon_user_task_id",
ExpressionAttributeValues={
":anon_user_task_id": anon_user_task_id,
},
)
if items:
return True
return False
def calculate_user_consent_statements(self, user_response_row):
user_consent_statements = list()
for tag in self.consent_question_tags:
user_consent_statements.append(
{self.consent_statements[tag]: user_response_row[tag]})
return user_consent_statements
def send_email(self, user_response_row):
data = {
"first_name":
"",
"consent_datetime":
qualtrics2thiscovery_timestamp(
user_response_row[self.consent_datetime_column_name]),
"consent_statements":
json.dumps(
self.calculate_user_consent_statements(user_response_row)),
"anon_project_specific_user_id":
user_response_row[self.anon_project_specific_user_id_column_name],
"anon_user_task_id":
user_response_row[self.anon_user_task_id_column_name],
"consent_info_url":
self.consent_info_url,
}
return send_consent_email_api({"body": json.dumps(data)},
context=dict())
def parse(
self,
consent_statements_column_name="consent_statements",
):
self.get_consent_statements()
with open(self.input_filename) as csvfile:
reader = csv.DictReader(csvfile)
for row in reader:
if not row[
consent_statements_column_name]: # automated Qualtrics process failed
anon_user_task_id = row[self.anon_user_task_id_column_name]
if not self.consent_in_ddb_table(anon_user_task_id):
# this response row wasn't processed in a previous run of this script
result = self.send_email(user_response_row=row)
assert (result["statusCode"] == HTTPStatus.OK
), f"Error sending consent email: {result}"
self.sent_emails.append(anon_user_task_id)
else:
self.logger.info(
"Skipped row; consent data detected in ddb",
extra={"anon_user_task_id": anon_user_task_id},
)
class RemindersHandler:
"""
Send a reminder:
- One day before an appointment (appointment_datetime)
- Unless an email (notification or reminder) was already sent today (latest_email_datetime)
"""
def __init__(self, logger=None, correlation_id=None):
self.ddb_client = Dynamodb(stack_name=STACK_NAME)
self.correlation_id = correlation_id
self.target_appointments = self.get_appointments_to_be_reminded()
self.logger = logger
if logger is None:
self.logger = utils.get_logger()
def get_appointments_to_be_reminded(self, now=None):
if now is None:
now = utils.now_with_tz()
date_format = "%Y-%m-%d"
tomorrow = now + datetime.timedelta(days=1)
today_string = now.strftime(date_format)
tomorrow_string = tomorrow.strftime(date_format)
return self.ddb_client.query(
table_name=APPOINTMENTS_TABLE,
IndexName="reminders-index",
KeyConditionExpression="appointment_date = :date "
"AND latest_participant_notification "
"BETWEEN :t1 AND :t2",
ExpressionAttributeValues={
":date": tomorrow_string,
":t1": "2020-00-00", # excludes 0000-00-00 appointments only because those will not have received the initial booking notification yet
":t2": today_string,
},
)
def send_reminders(self, now=None):
results = list()
for appointment in self.target_appointments:
app_item_type = appointment["type"]
appointment_id = appointment["id"]
if app_item_type == "acuity-appointment":
appointment_class = AcuityAppointment
notifier_class = AppointmentNotifier
elif app_item_type == views.VIEWS_DDB_ITEM_TYPE:
appointment_class = views.ViewsAppointment
notifier_class = views.ViewsNotifier
else:
raise NotImplementedError(
f"Appointment item type {app_item_type} not supported"
)
appointment = appointment_class(
appointment_id, logger=self.logger, correlation_id=self.correlation_id
)
appointment.ddb_load()
notifier = notifier_class(
appointment=appointment,
logger=self.logger,
correlation_id=self.correlation_id,
)
try:
reminder_result = notifier.send_reminder(now=now).get("statusCode")
except:
self.logger.error(
f"{notifier_class.__name__}.send_reminder raised an exception",
extra={
"appointment": appointment.as_dict(),
"correlation_id": self.correlation_id,
"traceback": traceback.format_exc(),
},
)
reminder_result = None
results.append((reminder_result, appointment_id))
return results
class PersonalLinkManager:
def __init__(
self,
survey_id,
anon_project_specific_user_id,
account,
project_task_id,
correlation_id=None,
):
self.survey_id = survey_id
self.anon_project_specific_user_id = anon_project_specific_user_id
self.account = account
self.account_survey_id = f"{account}_{survey_id}"
self.correlation_id = correlation_id
if correlation_id is None:
self.correlation_id = utils.new_correlation_id()
self.ddb_client = Dynamodb(stack_name=const.STACK_NAME,
correlation_id=self.correlation_id)
self.project_task_id = project_task_id
def _query_user_link(self) -> list:
"""
Retrieves a personal link previously assigned to this user
"""
return self.ddb_client.query(
table_name=const.PersonalLinksTable.NAME,
IndexName="assigned-links",
KeyConditionExpression=
"anon_project_specific_user_id = :anon_project_specific_user_id "
"AND account_survey_id = :account_survey_id",
ExpressionAttributeValues={
":anon_project_specific_user_id":
self.anon_project_specific_user_id,
":account_survey_id": self.account_survey_id,
},
)
def _assign_link_to_user(self, unassigned_links: list) -> str:
"""
Assigns to user the unassigned link with the soonest expiration date.
An existing anon_project_specific_user_id is checked at assignment type to protect against
the unlikely but possible scenario where a concurrent invocation of the
lambda has assigned the same link to a different user.
Args:
unassigned_links (list): All unassigned links for this account_survey_id in PersonalLinks table
Returns:
A url representing the personal link assigned to this user
"""
# assign oldest link to user
unassigned_links.sort(key=lambda x: x["expires"])
user_id_attr_name = "anon_project_specific_user_id"
logger = utils.get_logger()
for unassigned_link in unassigned_links:
user_link = unassigned_link["url"]
try:
self.ddb_client.update_item(
table_name=const.PersonalLinksTable.NAME,
key=self.account_survey_id,
name_value_pairs={
"status": "assigned",
user_id_attr_name: self.anon_project_specific_user_id,
},
key_name="account_survey_id",
sort_key={"url": user_link},
ConditionExpression=Attr(user_id_attr_name).not_exists(),
)
except ClientError:
logger.info(
"Link assignment failed; link is already assigned to another user",
extra={
"user_link": user_link,
},
)
else:
return user_link
logger.info(
"Ran out of unassigned links; creating some more and retrying",
extra={
"unassigned_links": unassigned_links,
},
)
unassigned_links = self._create_personal_links()
return self._assign_link_to_user(unassigned_links)
def _put_create_personal_links_event(self):
eb_event = eb.ThiscoveryEvent({
"detail-type": "create_personal_links",
"detail": {
"account": self.account,
"survey_id": self.survey_id,
"project_task_id": self.project_task_id,
},
})
return eb_event.put_event()
def _create_personal_links(self) -> list[dict]:
dlg = DistributionLinksGenerator(
account=self.account,
survey_id=self.survey_id,
contact_list_id=const.DISTRIBUTION_LISTS[self.account]["id"],
correlation_id=self.correlation_id,
project_task_id=self.project_task_id,
)
return dlg.generate_links_and_upload_to_dynamodb()
def get_personal_link(self) -> str:
try:
return self._query_user_link()[0]["url"]
except IndexError: # user link not found; get unassigned links and assign one to user
unassigned_links = get_unassigned_links(
account_survey_id=self.account_survey_id,
ddb_client=self.ddb_client)
unassigned_links_len = len(unassigned_links)
if (not unassigned_links
): # unassigned links not found; create some synchronously
unassigned_links = self._create_personal_links()
elif (unassigned_links_len < const.PersonalLinksTable.BUFFER
): # create links asynchronously
self._put_create_personal_links_event()
user_link = self._assign_link_to_user(unassigned_links)
return user_link
class InterviewTask(DdbBaseItem):
"""
Represents an interview system task item in ddb table InterviewTasks
"""
def __init__(self, interview_task_id, **kwargs):
self._project_task_id = kwargs.get("project_task_id")
self._interview_task_id = interview_task_id
optional_attributes = [
"name",
"short_name",
"description",
"completion_url",
"on_demand_available",
"on_demand_questions_random",
"live_available",
"live_questions_random",
"appointment_type_id",
"modified",
]
index_key_attributes = [ # these cannot be None
"on_demand_survey_id",
"live_survey_id",
]
for oa in optional_attributes:
self.__dict__[oa] = kwargs.get(oa)
for ika in index_key_attributes:
key_attribute = kwargs.get(ika)
if key_attribute:
self.__dict__[ika] = key_attribute
self._ddb_client = Dynamodb(stack_name=const.STACK_NAME)
def as_dict(self):
return {
k: v
for k, v in self.__dict__.items()
if (k[0] != "_") and (k not in ["created"])
}
def ddb_dump(self, update_allowed=False):
return self._ddb_client.put_item(
table_name=const.INTERVIEW_TASKS_TABLE["name"],
key=str(self._project_task_id),
item_type="interview_task",
item_details=None,
item=self.as_dict(),
update_allowed=update_allowed,
key_name=const.INTERVIEW_TASKS_TABLE["partition_key"],
sort_key={const.INTERVIEW_TASKS_TABLE["sort_key"]: self._interview_task_id},
)
def ddb_load(self):
if self.modified is None:
if self._project_task_id:
item = self._ddb_client.get_item(
table_name=const.INTERVIEW_TASKS_TABLE["name"],
key=str(self._project_task_id),
key_name=const.INTERVIEW_TASKS_TABLE["partition_key"],
sort_key={
const.INTERVIEW_TASKS_TABLE["sort_key"]: self._interview_task_id
},
)
items = [item]
else:
items = self._ddb_client.query(
table_name=const.INTERVIEW_TASKS_TABLE["name"],
IndexName="interview-task-id-index",
KeyConditionExpression="interview_task_id = :interview_task_id",
ExpressionAttributeValues={
":interview_task_id": self._interview_task_id,
},
)
items_n = len(items)
assert (
items_n <= 1
), f'Found {items_n} interview_tasks in {const.INTERVIEW_TASKS_TABLE["name"]} ddb table; expected 1'
try:
self.__dict__.update(items[0])
except (IndexError, TypeError):
raise utils.ObjectDoesNotExistError(
f"InterviewTask could not be found in Dynamodb",
details={
"project_task_id": self._project_task_id,
"interview_task_id": self._interview_task_id,
"InterviewTask": self.as_dict(),
},
)