def add_template_to_ddb(template_id, template_name, template_type,
formatted_custom_properties, preview_url):
ddb_client = Dynamodb()
ddb_client.put_item(
table_name="HubspotEmailTemplates",
key=template_name,
item_type=template_type,
item_details={
"preview_url": preview_url,
},
item={
"bcc": [],
"cc": [],
"contact_properties": [],
"custom_properties": formatted_custom_properties,
"from": TRANSACTIONAL_EMAILS_FROM_ADDRESS,
"hs_template_id": template_id,
},
)
def test_deletion_of_questions_no_longer_in_survey_definition(self):
mock_deleted_question = {
"block_id": "BL_3qH1dnbq50y9V0a",
"block_name": "Your experience using thiscovery",
"question_description":
"<p>Take a moment to reflect on how much you like bread before answering this question.</p>",
"question_id": "QID12",
"question_name": "Q12",
"question_text":
"<h3>Is thiscovery the best invention since sliced bread?</h3>",
"sequence_no": "12",
"survey_id": "SV_eDrjXPqGElN0Mwm",
"survey_modified": "2021-02-16T15:59:12Z",
}
ddb_client = Dynamodb(stack_name=const.STACK_NAME)
ddb_client.put_item(
table_name=const.INTERVIEW_QUESTIONS_TABLE["name"],
key=mock_deleted_question["survey_id"],
item_type="interview_question",
item_details=None,
item=mock_deleted_question,
update_allowed=True,
key_name=const.INTERVIEW_QUESTIONS_TABLE["partition_key"],
sort_key={
const.INTERVIEW_QUESTIONS_TABLE["sort_key"]:
mock_deleted_question["question_id"]
},
)
test_event = copy.deepcopy(
td.TEST_INTERVIEW_QUESTIONS_UPDATED_EB_EVENT)
result = ep.put_interview_questions(test_event, None)
self.assertEqual(HTTPStatus.OK, result["statusCode"])
updated_question_ids, deleted_question_ids = json.loads(result["body"])
self.assertEqual(4, len(updated_question_ids))
self.assertEqual([mock_deleted_question["question_id"]],
deleted_question_ids)
def main():
calendar_name = input(
"Please input the name of the calendar you want to add to Dynamodb, as shown in Acuity's UI:"
)
if not calendar_name:
sys.exit()
acuity_client = AcuityClient()
ddb_client = Dynamodb(stack_name=STACK_NAME)
acuity_calendars = acuity_client.get_calendars()
pprint(acuity_calendars)
target_calendar = None
for c in acuity_calendars:
if c["name"] == calendar_name:
target_calendar = c
continue
if target_calendar:
response = ddb_client.put_item(
"Calendars",
target_calendar["id"],
item_type="acuity-calendar",
item_details=target_calendar,
item={
"label": target_calendar["name"],
"block_monday_morning": True,
"emails_to_notify": list(),
"myinterview_link": None,
},
)
assert (
response["ResponseMetadata"]["HTTPStatusCode"] == HTTPStatus.OK
), f"Dynamodb client put_item operation failed with response: {response}"
print(
f'Calendar "{calendar_name}" successfully added to Dynamodb table')
else:
raise utils.ObjectDoesNotExistError(
f'Calendar "{calendar_name}" not found in Acuity')
class TaskResponse(DdbBaseItem):
"""
Represents a TaskResponse Ddb item
"""
def __init__(
self,
response_id,
event_time,
anon_project_specific_user_id=None,
anon_user_task_id=None,
detail_type=None,
detail=None,
correlation_id=None,
account=None,
):
self._response_id = response_id
self._event_time = event_time
self.account = account
self.anon_project_specific_user_id = anon_project_specific_user_id
self.anon_user_task_id = anon_user_task_id
self._detail_type = detail_type
self._detail = detail
self._correlation_id = correlation_id
self._core_client = CoreApiClient(correlation_id=correlation_id)
self._ddb_client = Dynamodb(stack_name=const.STACK_NAME,
correlation_id=correlation_id)
self.project_task_id = None
split_response_id = response_id.split("-")
assert (
len(split_response_id) == 2
), f"response_id ({response_id}) not in expected format SurveyID-QualtricsResponseID"
self.survey_id = split_response_id[0]
self.qualtrics_response_id = split_response_id[1]
@classmethod
def from_eb_event(cls, event):
event_detail = event["detail"]
response_id = event_detail.pop("response_id")
event_time = event["time"]
try:
anon_project_specific_user_id = event_detail.pop(
"anon_project_specific_user_id")
anon_user_task_id = event_detail.pop("anon_user_task_id")
except KeyError as exc:
raise utils.DetailedValueError(
f"Mandatory {exc} data not found in source event",
details={
"event": event,
},
)
return cls(
response_id=response_id,
event_time=event_time,
anon_project_specific_user_id=anon_project_specific_user_id,
anon_user_task_id=anon_user_task_id,
detail_type=event["detail-type"],
detail=event_detail,
correlation_id=event["id"],
account=event_detail.pop("account", "cambridge"),
)
def get_project_task_id(self):
user_task = self._core_client.get_user_task_from_anon_user_task_id(
anon_user_task_id=self.anon_user_task_id)
self.project_task_id = user_task["project_task_id"]
def get_user_id(self):
user = self._core_client.get_user_by_anon_project_specific_user_id(
self.anon_project_specific_user_id)
return user["id"]
def ddb_dump(self, update_allowed=False, unpack_detail=False):
item = self.as_dict()
if unpack_detail:
item.update(self._detail)
self._detail = dict()
return self._ddb_client.put_item(
table_name=const.TASK_RESPONSES_TABLE["name"],
key=self._response_id,
item_type=self._detail_type,
item_details=self._detail,
item=item,
update_allowed=update_allowed,
key_name=const.TASK_RESPONSES_TABLE["partition_key"],
sort_key={
const.TASK_RESPONSES_TABLE["sort_key"]: self._event_time
},
)
class CalendarBlocker:
def __init__(self, logger, correlation_id):
self.logger = logger
self.correlation_id = correlation_id
self.calendars_table = "Calendars"
self.blocks_table = "CalendarBlocks"
self.ddb_client = Dynamodb(stack_name=STACK_NAME)
self.acuity_client = AcuityClient()
self.sns_client = SnsClient()
def notify_sns_topic(self, message, subject):
topic_arn = utils.get_secret(
"sns-topics")["interview-notifications-arn"]
self.sns_client.publish(
message=message,
topic_arn=topic_arn,
Subject=subject,
)
def get_target_calendar_ids(self):
calendars = self.ddb_client.scan(
self.calendars_table,
"block_monday_morning",
[True],
)
return [(x["id"], x["label"]) for x in calendars]
def block_upcoming_weekend(self, calendar_id):
next_monday_date = next_weekday(0)
block_end = datetime.datetime.combine(next_monday_date,
datetime.time(hour=12, minute=0))
saturday_before_next_monday = next_monday_date - datetime.timedelta(
days=2)
block_start = datetime.datetime.combine(
saturday_before_next_monday, datetime.time(hour=0, minute=0))
return self.acuity_client.post_block(calendar_id, block_start,
block_end)
def create_blocks(self):
calendars = self.get_target_calendar_ids()
self.logger.debug("Calendars to block", extra={"calendars": calendars})
created_blocks_ids = list()
affected_calendar_names = list()
for i, name in calendars:
try:
block_dict = self.block_upcoming_weekend(i)
created_blocks_ids.append(block_dict["id"])
affected_calendar_names.append(name)
response = self.ddb_client.put_item(
self.blocks_table,
block_dict["id"],
item_type="calendar-block",
item_details=block_dict,
item={
"status": "new",
"error_message": None,
},
correlation_id=self.correlation_id,
)
assert (
response["ResponseMetadata"]["HTTPStatusCode"] ==
HTTPStatus.OK
), f"Call to Dynamodb client put_item method failed with response: {response}. "
except Exception as err:
self.logger.error(
f"{repr(err)} {len(created_blocks_ids)} blocks were created before this error occurred. "
f"Created blocks ids: {created_blocks_ids}")
raise
return created_blocks_ids, affected_calendar_names
def mark_failed_block_deletion(self, item_key, exception):
error_message = f"This error happened when trying to delete Acuity calendar block {item_key}: {repr(exception)}"
self.logger.error(error_message)
self.ddb_client.update_item(
self.blocks_table,
item_key,
name_value_pairs={
"status": "error",
"error_message": error_message
},
)
def delete_blocks(self):
blocks = self.ddb_client.scan(
self.blocks_table,
filter_attr_name="status",
filter_attr_values=["new"],
)
deleted_blocks_ids = list()
affected_calendar_names = list()
for b in blocks:
item_key = b.get("id")
try:
delete_response = self.acuity_client.delete_block(item_key)
assert delete_response == HTTPStatus.NO_CONTENT, (
f"Call to Acuity client delete_block method failed with response: {delete_response}. "
f"{len(deleted_blocks_ids)} blocks were deleted before this error occurred. Deleted blocks ids: {deleted_blocks_ids}"
)
deleted_blocks_ids.append(item_key)
affected_calendar_names.append(
self.acuity_client.get_calendar_by_id(
b["details"]["calendarID"])["name"])
response = self.ddb_client.delete_item(
self.blocks_table,
item_key,
correlation_id=self.correlation_id)
assert (
response["ResponseMetadata"]["HTTPStatusCode"] ==
HTTPStatus.OK
), (f"Call to Dynamodb client delete_item method failed with response: {response}. "
f"{len(deleted_blocks_ids)} blocks were deleted before this error occurred. Deleted blocks ids: {deleted_blocks_ids}"
)
except Exception as err:
self.mark_failed_block_deletion(item_key, err)
continue
return deleted_blocks_ids, affected_calendar_names
class AppointmentType:
"""
Represents an Acuity appointment type with additional attributes
"""
def __init__(self,
ddb_client=None,
acuity_client=None,
logger=None,
correlation_id=None):
self.type_id = None
self.name = None
self.category = None
self.has_link = None
self.send_notifications = None
self.templates = None
self.modified = None # flag used in ddb_load method to check if ddb data was already fetched
self.project_task_id = None
self.system = None # Views, MyInterview, etc
self._logger = logger
self._correlation_id = correlation_id
if logger is None:
self._logger = utils.get_logger()
self._ddb_client = ddb_client
if ddb_client is None:
self._ddb_client = Dynamodb(stack_name=STACK_NAME)
self._acuity_client = acuity_client
if acuity_client is None:
self._acuity_client = AcuityClient(
correlation_id=self._correlation_id)
def as_dict(self):
return {
k: v
for k, v in self.__dict__.items()
if (k[0] != "_") and (k not in ["created", "modified"])
}
def from_dict(self, type_dict):
self.__dict__.update(type_dict)
def ddb_dump(self, update_allowed=False):
return self._ddb_client.put_item(
table_name=APPOINTMENT_TYPES_TABLE,
key=str(self.type_id),
item_type="acuity-appointment-type",
item_details=None,
item=self.as_dict(),
update_allowed=update_allowed,
)
def ddb_load(self):
if self.modified is None:
item = self._ddb_client.get_item(
table_name=APPOINTMENT_TYPES_TABLE,
key=str(self.type_id),
correlation_id=self._correlation_id,
)
try:
self.__dict__.update(item)
except TypeError:
raise utils.ObjectDoesNotExistError(
f"Appointment type {self.type_id} could not be found in Dynamodb",
details={
"appointment_type": self.as_dict(),
"correlation_id": self._correlation_id,
},
)
def get_appointment_type_id_to_info_map(self):
"""
Converts the list of appointment types returned by AcuityClient.get_appointment_types()
to a dictionary indexed by id
"""
appointment_types = self._acuity_client.get_appointment_types()
return {str(x["id"]): x for x in appointment_types}
def get_appointment_type_info_from_acuity(self):
"""
There is no direct method to get a appointment type by id (https://developers.acuityscheduling.com/reference), so
we have to fetch all appointment types and lookup
"""
if (self.name is None) or (self.category is None):
id_to_info = self.get_appointment_type_id_to_info_map()
self.name = id_to_info[str(self.type_id)]["name"]
self.category = id_to_info[str(self.type_id)]["category"]
class BaseAcuityAppointment(metaclass=ABCMeta):
"""
An abstract class representing an Acuity appointment
"""
def __init__(self, appointment_id, logger=None, correlation_id=None):
self.appointment_id = str(appointment_id)
self.acuity_info = None
self.calendar_id = None
self.calendar_name = None
self.participant_email = None
self.participant_user_id = None
self.appointment_type = AppointmentType()
self.latest_participant_notification = (
"0000-00-00 00:00:00+00:00" # used as GSI sort key, so cannot be None
)
self.appointment_date = None
self.anon_project_specific_user_id = None
self.anon_user_task_id = None
self.appointment_type_id = None
self._logger = logger
if self._logger is None:
self._logger = utils.get_logger()
self._correlation_id = correlation_id
self._ddb_client = Dynamodb(stack_name=STACK_NAME)
self._core_api_client = CoreApiClient(
correlation_id=self._correlation_id) # transactional emails
self._acuity_client = AcuityClient(correlation_id=self._correlation_id)
self.original_appointment = (
None # used to store appointment history if rescheduled
)
def __repr__(self):
return str(self.__dict__)
def from_dict(self, appointment_dict):
"""Used to quickly load appointments into Dynamodb for testing"""
self.appointment_type.from_dict(
appointment_dict.pop("appointment_type", dict()))
self.__dict__.update(appointment_dict)
def as_dict(self):
d = {
k: v
for k, v in self.__dict__.items() if (k[0] != "_") and (
k not in ["created", "modified", "appointment_type"])
}
d["appointment_type"] = self.appointment_type.as_dict()
return d
def ddb_dump(
self,
update_allowed=False,
item_type="acuity-appointment",
):
self.get_appointment_info_from_acuity(
) # populates self.appointment_type.type_id
self.appointment_type.ddb_load()
return self._ddb_client.put_item(
table_name=APPOINTMENTS_TABLE,
key=self.appointment_id,
item_type=item_type,
item_details=None,
item=self.as_dict(),
update_allowed=update_allowed,
)
def ddb_load(self):
item = self.get_appointment_item_from_ddb()
try:
item_app_type = item["appointment_type"]
except TypeError:
raise utils.ObjectDoesNotExistError(
f"Appointment {self.appointment_id} could not be found in Dynamodb",
details={
"appointment": self.as_dict(),
"correlation_id": self._correlation_id,
},
)
del item["appointment_type"]
self.__dict__.update(item)
self.appointment_type.from_dict(item_app_type)
def get_appointment_info_from_acuity(self, force_refresh=False):
if (self.acuity_info is None) or (force_refresh is True):
self.acuity_info = self._acuity_client.get_appointment_by_id(
self.appointment_id)
self.appointment_type.type_id = str(
self.acuity_info["appointmentTypeID"])
self.appointment_type_id = self.appointment_type.type_id
self.calendar_name = self.acuity_info["calendar"]
self.calendar_id = str(self.acuity_info["calendarID"])
self.participant_email = self.acuity_info["email"]
self.appointment_date = self.acuity_info["datetime"].split("T")[0]
# intake form processing
if (self.anon_project_specific_user_id is
None) or (self.anon_user_task_id is None):
for form in self.acuity_info["forms"]:
if form["id"] == ACUITY_USER_METADATA_INTAKE_FORM_ID:
intake_form_fields = {
x.get("name"): x.get("value")
for x in form["values"]
}
self.anon_project_specific_user_id = intake_form_fields.get(
"anon_project_specific_user_id")
self.anon_user_task_id = intake_form_fields.get(
"anon_user_task_id")
return self.acuity_info
def get_appointment_item_from_ddb(self):
return self._ddb_client.get_item(table_name=APPOINTMENTS_TABLE,
key=self.appointment_id)
def update_latest_participant_notification(self):
self.latest_participant_notification = str(utils.now_with_tz())
result = self._ddb_client.update_item(
table_name=APPOINTMENTS_TABLE,
key=self.appointment_id,
name_value_pairs={
"latest_participant_notification":
self.latest_participant_notification
},
)
assert (
result["ResponseMetadata"]["HTTPStatusCode"] == HTTPStatus.OK
), f"Call to ddb client update_item method failed with response {result}"
return result["ResponseMetadata"]["HTTPStatusCode"]
class SurveyDefinition:
def __init__(self,
qualtrics_account_name="cambridge",
survey_id=None,
correlation_id=None):
client = SurveyDefinitionsClient(
qualtrics_account_name=qualtrics_account_name,
survey_id=survey_id,
correlation_id=correlation_id,
)
response = client.get_survey()
assert (response["meta"]["httpStatus"] == "200 - OK"
), f"Call to Qualtrics API failed with response {response}"
self.survey_id = survey_id
self.definition = response["result"]
self.flow = self.definition["SurveyFlow"]["Flow"]
self.blocks = self.definition["Blocks"]
self.questions = self.definition["Questions"]
self.modified = self.definition["LastModified"]
self.ddb_client = Dynamodb(stack_name=const.STACK_NAME)
self.logger = utils.get_logger()
self.logger.debug(
"Initialised SurveyDefinition",
extra={
"__dict__": self.__dict__,
"correlation_id": correlation_id,
},
)
@classmethod
def from_eb_event(cls, event):
logger = utils.get_logger()
logger.debug(
"EB event",
extra={
"event": event,
},
)
event_detail = event["detail"]
try:
qualtrics_account_name = event_detail.pop("account")
survey_id = event_detail.pop("survey_id")
except KeyError as exc:
raise utils.DetailedValueError(
f"Mandatory {exc} data not found in source event",
details={
"event": event,
},
)
return cls(
qualtrics_account_name=qualtrics_account_name,
survey_id=survey_id,
correlation_id=event["id"],
)
def get_interview_question_list_from_Qualtrics(self):
def parse_question_html(s):
text_m = PROMPT_RE.search(s)
text = text_m.group(1)
description_m = DESCRIPTION_RE.search(s)
try:
description = description_m.group(1)
except AttributeError:
description = None
return text, description
interview_question_list = list()
question_counter = 1
block_ids_flow = [
x["ID"] for x in self.flow if x["Type"] not in ["Branch"]
] # flow items of Branch type represent survey branching logic
for block_id in block_ids_flow:
block = self.blocks[block_id]
block_name = block["Description"]
question_ids = [x["QuestionID"] for x in block["BlockElements"]]
for question_id in question_ids:
q = self.questions[question_id]
question_name = q["DataExportTag"]
question_text_raw = q["QuestionText"]
try:
question_text, question_description = parse_question_html(
question_text_raw)
except AttributeError: # no match found for PROMPT_RE
if SYSTEM_RE.findall(question_text_raw):
continue # this is a system config question; skip
else:
raise utils.DetailedValueError(
"Mandatory prompt div could not be found in interview question",
details={
"question": question_text_raw,
"survey_id": self.survey_id,
"question_id": question_id,
"question_export_tag": question_name,
},
)
question = InterviewQuestion(
survey_id=self.survey_id,
survey_modified=self.modified,
question_id=question_id,
question_name=question_name,
sequence_no=str(question_counter),
block_name=block_name,
block_id=block_id,
question_text=question_text,
question_description=question_description,
)
interview_question_list.append(question)
question_counter += 1
return interview_question_list
def ddb_update_interview_questions(self):
"""
Updates the list of interview questions held in Dynamodb for a particular survey.
This includes not only adding and updating questions, but also deleting questions that are no longer
present in the survey
"""
ddb_question_list = self.ddb_load_interview_questions(self.survey_id)
survey_question_list = self.get_interview_question_list_from_Qualtrics(
)
updated_question_ids = list()
deleted_question_ids = list()
for q in survey_question_list:
self.ddb_client.put_item(
table_name=const.INTERVIEW_QUESTIONS_TABLE["name"],
key=q._survey_id,
key_name=const.INTERVIEW_QUESTIONS_TABLE["partition_key"],
item_type="interview_question",
item_details=None,
item=q.as_dict(),
update_allowed=True,
sort_key={
const.INTERVIEW_QUESTIONS_TABLE["sort_key"]:
q._question_id,
},
)
updated_question_ids.append(q._question_id)
for q in ddb_question_list:
question_id = q["question_id"]
if question_id not in updated_question_ids:
self.ddb_client.delete_item(
table_name=const.INTERVIEW_QUESTIONS_TABLE["name"],
key=q["survey_id"],
key_name=const.INTERVIEW_QUESTIONS_TABLE["partition_key"],
sort_key={
const.INTERVIEW_QUESTIONS_TABLE["sort_key"]:
question_id,
},
)
deleted_question_ids.append(question_id)
return updated_question_ids, deleted_question_ids
@staticmethod
def ddb_load_interview_questions(survey_id):
ddb_client = Dynamodb(stack_name=const.STACK_NAME)
return ddb_client.query(
table_name=const.INTERVIEW_QUESTIONS_TABLE["name"],
KeyConditionExpression="survey_id = :survey_id",
ExpressionAttributeValues={
":survey_id": survey_id,
},
)
@staticmethod
def get_interview_questions(survey_id):
"""
Get interview questions for a VIEWS interview. Randomises question order
within blocks if that is specified by InterviewTask
Args:
survey_id: Qualtrics survey id of survey containing the interview questions
Returns:
"""
def randomise_questions_check():
"""
Reads configuration in InterviewTask items containing survey_id to determine whether or not
interview questions should be randomised.
Returns:
True if questions should be randomised; otherwise False
"""
bool_config_choices = {True, False}
error_message = (
f"Conflicting interview task configuration(s) found for survey {survey_id}. "
f"Random question order specified in some but not all interview tasks using that survey"
)
interview_tasks_table = const.InterviewTasksTable()
# get config for live interviews with questions coming from survey_id
live_config_matches = interview_tasks_table.query_live_survey_id_index(
survey_id=survey_id)
live_random_config_set = set()
for m in live_config_matches:
random_config = m.get("live_questions_random", False)
live_random_config_set.add(random_config)
# get config for on-demand interviews with questions coming from survey_id
on_demand_config_matches = (
interview_tasks_table.query_on_demand_survey_id_index(
survey_id=survey_id))
on_demand_random_config_set = set()
for m in on_demand_config_matches:
random_config = m.get("on_demand_questions_random", False)
on_demand_random_config_set.add(random_config)
# check for conflicts and return randomise config if there are no issues
random_config_choices = live_random_config_set.union(
on_demand_random_config_set)
random_config_choices_excluding_none = random_config_choices.intersection(
bool_config_choices)
assert len(
random_config_choices_excluding_none) <= 1, error_message
try:
return random_config_choices_excluding_none.pop()
except KeyError: # no config found; default to non-random
return False
interview_questions = SurveyDefinition.ddb_load_interview_questions(
survey_id)
try:
survey_modified = interview_questions[0]["survey_modified"]
except IndexError:
raise utils.ObjectDoesNotExistError(
f"No interview questions found for survey {survey_id}",
details={})
block_dict = dict()
for iq in interview_questions:
block_id = iq["block_id"]
try:
block = block_dict[block_id]
except KeyError:
block = {
"block_id": block_id,
"block_name": iq["block_name"],
"questions": list(),
}
question = {
"question_id": iq["question_id"],
"question_name": iq["question_name"],
"sequence_no": iq["sequence_no"],
"question_text": iq["question_text"],
"question_description": iq["question_description"],
}
block["questions"].append(question)
block_dict[block_id] = block
# order questions in each block
randomise = randomise_questions_check()
for block in block_dict.values():
if randomise:
random.shuffle(block["questions"])
else:
block["questions"] = sorted(
block["questions"], key=lambda k: int(k["sequence_no"]))
body = {
"survey_id": survey_id,
"modified": survey_modified,
"blocks": list(block_dict.values()),
"count": len(interview_questions),
}
return body
def setUpClass(cls):
ddb_client = Dynamodb(stack_name=const.STACK_NAME)
survey_init_table = const.SurveyInitTable()
# setup test data
test_task_response = {
"survey_id": "SV_cD6pZ6NoZSpnL8i",
"participant_responses": {
"Q2": 2,
"Q3": "Quality Assurance Engineer",
"Q4": ["1", "3"],
"Q2_DO": ["1", "2", "3"],
"RecipientEmail": None,
"RecipientLastName": None,
"DistributionChannel": "preview",
"Q4_DO": ["1", "2", "3"],
"Set a Value Now": None,
"RecipientFirstName": None,
"LocationLatitude": "53.3793029785",
"StartDate": "2022-01-04T14:05:59Z",
"ExternalReference": None,
"Status": 1,
"Finished": 1,
"Progress": 100,
"Q3_DO": None,
"Duration (in seconds)": 39,
"anon_project_specific_user_id":
"29aca87c-e0f9-44c2-b97e-22cbe842a908",
"EndDate": "2022-01-04T14:06:38Z",
"RecordedDate": "2022-01-04T14:06:39.384Z",
"UserLanguage": "EN-GB",
"LocationLongitude": "-1.46020507813",
"IPAddress": None,
},
"anon_project_specific_user_id":
"29aca87c-e0f9-44c2-b97e-22cbe842a908",
"anon_user_task_id": "fb92e1f0-e756-47db-ab5c-232c3618999a",
"project_task_id": "8e0fb129-f6b6-4b6b-a01a-cfdb14f8fec8",
"qualtrics_response_id": "R_1Howgi70DgOqcKG",
"user_id": "1cbe9aad-b29f-46b5-920e-b4c496d42515",
"account": "cambridge",
}
survey_init_table.put_item(
**{
"destination_survey_id": "SV_aWDwvBuOqAsxrkq",
"item_details": {
"job": [{
"question": "Q3",
"survey": "SV_cD6pZ6NoZSpnL8i"
}],
"age": [{
"question": "Q2",
"survey": "SV_cD6pZ6NoZSpnL8i"
}],
"food": [{
"question": "Q4",
"survey": "SV_cD6pZ6NoZSpnL8i"
}],
},
},
update=True,
)
ddb_client.put_item(
table_name=const.TASK_RESPONSES_TABLE["name"],
key="SV_cD6pZ6NoZSpnL8i-R_1Howgi70DgOqcKG",
key_name=const.TASK_RESPONSES_TABLE["partition_key"],
item_type="survey_response",
update_allowed=True,
item_details={},
sort_key={
const.TASK_RESPONSES_TABLE["sort_key"]: "2022-01-04T14:06:39Z"
},
item=test_task_response,
)
survey_init_table.put_item(
**{
"destination_survey_id": "SV_ezK42q9nZRtRCxU",
"item_details": {
"job": [{
"question": "Q3",
"survey": "SV_0kUfBfy4Im8nMKq"
}],
"age": [{
"question": "Q2",
"survey": "SV_0kUfBfy4Im8nMKq"
}],
"food": [{
"question": "Q4",
"survey": "SV_0kUfBfy4Im8nMKq"
}],
},
},
update=True,
)
ddb_client.put_item(
table_name=const.TASK_RESPONSES_TABLE["name"],
key="SV_0kUfBfy4Im8nMKq-R_KAd1R9N6PQNQ1SL",
key_name=const.TASK_RESPONSES_TABLE["partition_key"],
item_type="survey_response",
update_allowed=True,
item_details={},
sort_key={
const.TASK_RESPONSES_TABLE["sort_key"]: "2022-01-04T14:06:39Z"
},
item={
**test_task_response,
"survey_id": "SV_0kUfBfy4Im8nMKq",
"qualtrics_response_id": "R_KAd1R9N6PQNQ1SL",
"account": "thisinstitute",
},
)
class Consent:
"""
Represents a consent item in Dynamodb
"""
def __init__(self,
consent_id=None,
core_api_client=None,
correlation_id=None):
self.project_id = None
self.project_short_name = None
self.project_name = None
self.project_task_id = None
self.consent_id = consent_id
if consent_id is None:
self.consent_id = str(uuid.uuid4())
self.consent_datetime = None
self.anon_project_specific_user_id = None
self.anon_user_task_id = None
self.consent_statements = None
self.modified = None # flag used in ddb_load method to check if ddb data was already fetched
self._correlation_id = correlation_id
self._ddb_client = Dynamodb(stack_name=STACK_NAME)
self._core_api_client = core_api_client
if core_api_client is None:
self._core_api_client = CoreApiClient(
correlation_id=correlation_id)
def as_dict(self):
return {
k: v
for k, v in self.__dict__.items() if (k[0] != "_") and (
k not in ["created", "modified", "details", "type"])
}
def from_dict(self, consent_dict):
self.__dict__.update(consent_dict)
def ddb_dump(self, update_allowed=False):
self._get_project()
result = self._ddb_client.put_item(
table_name=CONSENT_DATA_TABLE,
key=self.project_task_id,
key_name="project_task_id",
item_type="qualtrics-consent-data",
item_details=dict(),
item=self.as_dict(),
update_allowed=update_allowed,
)["ResponseMetadata"]["HTTPStatusCode"]
assert result == HTTPStatus.OK
return result
def ddb_load(self):
if self.modified is None:
self._get_project_task_id()
item = self._ddb_client.get_item(
table_name=CONSENT_DATA_TABLE,
key=self.project_task_id,
key_name="project_task_id",
sort_key={"consent_id": self.consent_id},
correlation_id=self._correlation_id,
)
try:
self.__dict__.update(item)
except TypeError:
raise utils.ObjectDoesNotExistError(
f"Consent item {self.project_task_id}, {self.consent_id} could not be found in Dynamodb",
details={
"consent_dict": self.as_dict(),
"correlation_id": self._correlation_id,
},
)
else:
return HTTPStatus.OK
def _get_project_task_id(self):
if self.project_task_id is None:
self.project_task_id = (
self._core_api_client.get_user_task_from_anon_user_task_id(
anon_user_task_id=self.anon_user_task_id)
["project_task_id"])
assert self.project_task_id
def _get_project(self):
if self.project_id is None:
self._get_project_task_id()
project = self._core_api_client.get_project_from_project_task_id(
project_task_id=self.project_task_id)
self.project_id = project["id"]
self.project_short_name = project["name"]
self.project_name = project["name"]
class InterviewTask(DdbBaseItem):
"""
Represents an interview system task item in ddb table InterviewTasks
"""
def __init__(self, interview_task_id, **kwargs):
self._project_task_id = kwargs.get("project_task_id")
self._interview_task_id = interview_task_id
optional_attributes = [
"name",
"short_name",
"description",
"completion_url",
"on_demand_available",
"on_demand_questions_random",
"live_available",
"live_questions_random",
"appointment_type_id",
"modified",
]
index_key_attributes = [ # these cannot be None
"on_demand_survey_id",
"live_survey_id",
]
for oa in optional_attributes:
self.__dict__[oa] = kwargs.get(oa)
for ika in index_key_attributes:
key_attribute = kwargs.get(ika)
if key_attribute:
self.__dict__[ika] = key_attribute
self._ddb_client = Dynamodb(stack_name=const.STACK_NAME)
def as_dict(self):
return {
k: v
for k, v in self.__dict__.items()
if (k[0] != "_") and (k not in ["created"])
}
def ddb_dump(self, update_allowed=False):
return self._ddb_client.put_item(
table_name=const.INTERVIEW_TASKS_TABLE["name"],
key=str(self._project_task_id),
item_type="interview_task",
item_details=None,
item=self.as_dict(),
update_allowed=update_allowed,
key_name=const.INTERVIEW_TASKS_TABLE["partition_key"],
sort_key={const.INTERVIEW_TASKS_TABLE["sort_key"]: self._interview_task_id},
)
def ddb_load(self):
if self.modified is None:
if self._project_task_id:
item = self._ddb_client.get_item(
table_name=const.INTERVIEW_TASKS_TABLE["name"],
key=str(self._project_task_id),
key_name=const.INTERVIEW_TASKS_TABLE["partition_key"],
sort_key={
const.INTERVIEW_TASKS_TABLE["sort_key"]: self._interview_task_id
},
)
items = [item]
else:
items = self._ddb_client.query(
table_name=const.INTERVIEW_TASKS_TABLE["name"],
IndexName="interview-task-id-index",
KeyConditionExpression="interview_task_id = :interview_task_id",
ExpressionAttributeValues={
":interview_task_id": self._interview_task_id,
},
)
items_n = len(items)
assert (
items_n <= 1
), f'Found {items_n} interview_tasks in {const.INTERVIEW_TASKS_TABLE["name"]} ddb table; expected 1'
try:
self.__dict__.update(items[0])
except (IndexError, TypeError):
raise utils.ObjectDoesNotExistError(
f"InterviewTask could not be found in Dynamodb",
details={
"project_task_id": self._project_task_id,
"interview_task_id": self._interview_task_id,
"InterviewTask": self.as_dict(),
},
)
class SurveyResponse:
responses_table = "Responses"
def __init__(self, response_dict, correlation_id=None):
self.survey_id = response_dict.pop("survey_id", None)
self.response_id = response_dict.pop("response_id", None)
self.project_task_id = str(
utils.validate_uuid(response_dict.pop("project_task_id", None)))
self.anon_project_specific_user_id = str(
utils.validate_uuid(
response_dict.pop("anon_project_specific_user_id", None)))
self.anon_user_task_id = str(
utils.validate_uuid(response_dict.pop("anon_user_task_id", None)))
for required_parameter_name, value in [
("survey_id", self.survey_id),
("response_id", self.response_id),
]:
if not value:
raise utils.DetailedValueError(
f"Required parameter {required_parameter_name} not present in body of call",
details={
"response_dict": response_dict,
"correlation_id": correlation_id,
},
)
self.response_dict = response_dict
self.ddb_client = Dynamodb(
stack_name=const.STACK_NAME,
correlation_id=correlation_id,
)
self.correlation_id = correlation_id
def check_project_task_exists(self):
env_name = utils.get_environment_name()
if env_name == "prod":
core_api_url = "https://api.thiscovery.org/"
else:
core_api_url = f"https://{env_name}-api.thiscovery.org/"
result = utils.aws_get(
endpoint_url="v1/project",
base_url=core_api_url,
)
assert (result["statusCode"] == HTTPStatus.OK
), f"Call to core API returned error: {result}"
projects = json.loads(result["body"])
project_task_ids = list()
for p in projects:
tasks = p["tasks"]
for t in tasks:
project_task_ids.append(t["id"])
if self.project_task_id not in project_task_ids:
raise utils.ObjectDoesNotExistError(
f"Project tasks id {self.project_task_id} not found in Thiscovery database",
details={
"project_task_ids": project_task_ids,
"correlation_id": self.correlation_id,
},
)
return True
def put_item(self):
return self.ddb_client.put_item(
table_name=self.responses_table,
key=f"{self.survey_id}-{self.response_id}",
item_type="survey_response",
item_details={
**self.response_dict, "correlation_id": self.correlation_id
},
item={
"survey_id": self.survey_id,
"response_id": self.response_id,
"project_task_id": self.project_task_id,
"anon_project_specific_user_id":
self.anon_project_specific_user_id,
"anon_user_task_id": self.anon_user_task_id,
},
update_allowed=True,
)