def __init__(self, logger=None, correlation_id=None):
self.ddb_client = Dynamodb(stack_name=STACK_NAME)
self.correlation_id = correlation_id
self.target_appointment_ids = self.get_appointments_to_be_deleted()
self.logger = logger
if logger is None:
self.logger = utils.get_logger()
def __init__(self,
qualtrics_account_name="cambridge",
survey_id=None,
correlation_id=None):
client = SurveyDefinitionsClient(
qualtrics_account_name=qualtrics_account_name,
survey_id=survey_id,
correlation_id=correlation_id,
)
response = client.get_survey()
assert (response["meta"]["httpStatus"] == "200 - OK"
), f"Call to Qualtrics API failed with response {response}"
self.survey_id = survey_id
self.definition = response["result"]
self.flow = self.definition["SurveyFlow"]["Flow"]
self.blocks = self.definition["Blocks"]
self.questions = self.definition["Questions"]
self.modified = self.definition["LastModified"]
self.ddb_client = Dynamodb(stack_name=const.STACK_NAME)
self.logger = utils.get_logger()
self.logger.debug(
"Initialised SurveyDefinition",
extra={
"__dict__": self.__dict__,
"correlation_id": correlation_id,
},
)
def execute_non_query(sql, params, correlation_id=new_correlation_id()):
"""
Use this method to make changes that will be committed to the database (e.g. UPDATE, DELETE calls)
"""
logger = get_logger()
conn = _get_connection(correlation_id)
sql = minimise_white_space(sql)
param_str = str(params)
logger.info(
"postgres query",
extra={
"query": sql,
"parameters": param_str,
"correlation_id": correlation_id
},
)
with conn.cursor() as cursor:
try:
cursor.execute(sql, params)
rowcount = cursor.rowcount
conn.commit()
except psycopg2.IntegrityError as err:
errorjson = {
"error": err.args[0],
"correlation_id": str(correlation_id)
}
raise DetailedIntegrityError("Database integrity error", errorjson)
return rowcount
def __init__(self, correlation_id=None, ddb_client=None):
self.id = None
self.user_project = None
self.project_task = None
self.status = None
self.consented = None
self.progress_info = None
self.ext_user_task_id = None
self.anon_user_task_id = None
self.user_task_url = None
self._correlation_id = correlation_id
self.user_id = None
self.first_name = None
self.last_name = None
self.email = None
# this is the same as project_task above
self.project_task_id = None
self.project_task_status = None
self.project_task_url_params = None
self.anon_user_task_id = None
self.created = None
self.project_id = None
self.base_url = None
self.task_provider_name = None
self.external_task_id = None
self.user_specific_url = None
self.anonymise_url = None
# this is the same as user_project above
self.user_project_id = None
self.anon_project_specific_user_id = None
self.task_type_name = None
self._ddb_client = ddb_client
self._logger = utils.get_logger()
def __init__(self, appointment_id, logger=None, correlation_id=None):
self.appointment_id = str(appointment_id)
self.acuity_info = None
self.calendar_id = None
self.calendar_name = None
self.participant_email = None
self.participant_user_id = None
self.appointment_type = AppointmentType()
self.latest_participant_notification = (
"0000-00-00 00:00:00+00:00" # used as GSI sort key, so cannot be None
)
self.appointment_date = None
self.anon_project_specific_user_id = None
self.anon_user_task_id = None
self.appointment_type_id = None
self._logger = logger
if self._logger is None:
self._logger = utils.get_logger()
self._correlation_id = correlation_id
self._ddb_client = Dynamodb(stack_name=STACK_NAME)
self._core_api_client = CoreApiClient(
correlation_id=self._correlation_id) # transactional emails
self._acuity_client = AcuityClient(correlation_id=self._correlation_id)
self.original_appointment = (
None # used to store appointment history if rescheduled
)
def __init__(self, survey_id, account_name="cambridge"):
self.logger = get_logger()
self.survey_client = SurveyDefinitionsClient(
qualtrics_account_name=account_name, survey_id=survey_id)
self.survey_definition = self.survey_client.get_survey()["result"]
self.questions = self.survey_definition["Questions"]
self.updated_questions = dict()
def __init__(self, question_dict, question_config):
self.question_dict = question_dict
self.question_config = question_config
self.export_tag = self.question_dict["DataExportTag"]
self.question_text = self.question_dict["QuestionText"]
self.question_js = self.question_dict.get("QuestionJS")
self.logger = get_logger()
def __init__(self,
ddb_client=None,
acuity_client=None,
logger=None,
correlation_id=None):
self.type_id = None
self.name = None
self.category = None
self.has_link = None
self.send_notifications = None
self.templates = None
self.modified = None # flag used in ddb_load method to check if ddb data was already fetched
self.project_task_id = None
self.system = None # Views, MyInterview, etc
self._logger = logger
self._correlation_id = correlation_id
if logger is None:
self._logger = utils.get_logger()
self._ddb_client = ddb_client
if ddb_client is None:
self._ddb_client = Dynamodb(stack_name=STACK_NAME)
self._acuity_client = acuity_client
if acuity_client is None:
self._acuity_client = AcuityClient(
correlation_id=self._correlation_id)
def __init__(self, survey_consent_event):
self.logger = survey_consent_event.get("logger", utils.get_logger())
self.correlation_id = survey_consent_event.get(
"correlation_id", utils.new_correlation_id())
self.consent_dict = json.loads(survey_consent_event["body"])
self.consent_info_url = self.consent_dict["consent_info_url"]
del self.consent_dict["consent_info_url"]
consent_embedded_data_fieldname = "consent_statements"
self.consent_dict[consent_embedded_data_fieldname] = json.loads(
self.consent_dict[consent_embedded_data_fieldname])
self.to_recipient_email = self.consent_dict.get("to_recipient_email")
try:
self.template_name = self.consent_dict["template_name"]
except KeyError:
self.template_name = DEFAULT_CONSENT_EMAIL_TEMPLATE
else:
del self.consent_dict["template_name"]
try:
self.consent_dict[
"consent_datetime"] = qualtrics2thiscovery_timestamp(
self.consent_dict["consent_datetime"])
except KeyError:
self.consent_dict["consent_datetime"] = str(utils.now_with_tz())
self.core_api_client = CoreApiClient(
correlation_id=self.correlation_id)
self.consent = Consent(core_api_client=self.core_api_client,
correlation_id=self.correlation_id)
self.consent.from_dict(consent_dict=self.consent_dict)
def __init__(
self,
response_id,
event_time=None,
anon_project_specific_user_id=None,
anon_user_task_id=None,
detail_type=None,
detail=None,
correlation_id=None,
interview_task_id=None,
account=None,
):
super().__init__(
response_id=response_id,
event_time=event_time,
anon_project_specific_user_id=anon_project_specific_user_id,
anon_user_task_id=anon_user_task_id,
detail_type=detail_type,
detail=detail,
correlation_id=correlation_id,
account=account,
)
del self.account
try:
self._detail["event_type"] = "user_interview_task"
except TypeError:
self._detail = {"event_type": "user_interview_task"}
self.interview_task_id = interview_task_id
self.interview_task = None
self._logger = utils.get_logger()
def main(dry_run):
dry_run_postfix = ""
if dry_run:
dry_run_postfix = (
"(this was a dry run, so no changes were actually made to the survey)"
)
logger = get_logger()
survey_client = SurveyDefinitionsClient(survey_id=conf.CURRENT_SURVEY_ID)
survey = survey_client.get_survey()["result"]
questions = survey["Questions"]
updated_questions = list()
for k, v in questions.items():
updated_question = False
export_tag = v["DataExportTag"]
question_text = v["QuestionText"]
# add t1 question scores and graphs
if export_tag in conf.current_delphi_round_question_config:
survey_question = SurveyQuestion(
v, conf.current_delphi_round_question_config[export_tag]
)
survey_question.inject_previous_round_delphi_score()
# graphs
# add question JS
existing_question_js = v.get("QuestionJS")
if not existing_question_js:
existing_question_js = q_js.EMPTY_JS
split_js = q_js.js_add_on_ready_re.split(existing_question_js, maxsplit=1)
assert (
n := len(split_js)
) == 2, f"Unexpected split of existing_question_js ({n}): {split_js}"
t1_tag = conf.this_task_to_previous_task_mapping[export_tag]
v["QuestionJS"] = (
split_js[0]
+ "Qualtrics.SurveyEngine.addOnReady(function()\n{"
+ get_graph_js_for_qualtrics(t1_tag)
+ split_js[1]
)
# add graph container
container_id = f"highcharts-{t1_tag}"
replacements_1 = conf.chart_placeholder_re.subn(
HIGHCHARTS_CONTAINER.format(container_id), question_text
)
total_replacements = replacements_1[1]
assert (
total_replacements == 1
), f"Unexpected number of replacements ({total_replacements}) in question_text: {question_text}"
replacements = replacements_1
v["QuestionText"] = replacements[0]
updated_question = True
if updated_question:
updated_questions.append(export_tag)
if not dry_run:
survey_client.update_question(question_id=k, data=v)
print(
f"Updated {len(updated_questions)} questions: {updated_questions} {dry_run_postfix}"
)
def __init__(self, message_id, correlation_id=None):
self.message_id = message_id
self.correlation_id = correlation_id
self.s3_client = S3Client()
self.bucket = utils.get_secret("incoming-email-bucket")['name']
self.message = None
self.message_obj_http_path = None
self.logger = utils.get_logger()
def list_users_by_project(project_id, logger=None, correlation_id=None):
if logger is None:
logger = utils.get_logger()
users = execute_query(
base_sql=sql_q.LIST_USERS_BY_PROJECT_SQL,
params=(project_id, ),
correlation_id=correlation_id,
)
return users
def main():
logger = utils.get_logger()
ac = AcuityClient()
for e in EVENTS:
try:
response = ac.post_webhooks(e)
logger.info("Created webhook", extra={"response": response})
except utils.DetailedValueError:
pass
def execute_non_query_multiple(sql_iterable,
params_iterable,
correlation_id=new_correlation_id()):
"""
Args:
sql_iterable (tuple, list, etc): iterable containing sql queries to be executed
params_iterable (tuple, list, etc): iterable containing params for sql queries in sql_iterable
correlation_id:
Returns:
List of number of rows affected by each of the input sql queries
"""
logger = get_logger()
conn = _get_connection(correlation_id)
results = []
with conn.cursor() as cursor:
for (sql, params) in zip(sql_iterable, params_iterable):
sql = minimise_white_space(sql)
param_str = str(params)
logger.info(
"postgres query",
extra={
"query": sql,
"parameters": param_str,
"correlation_id": correlation_id,
},
)
try:
cursor.execute(sql, params)
except psycopg2.IntegrityError as err:
errorjson = {
"error": err.args[0],
"correlation_id": str(correlation_id),
}
raise DetailedIntegrityError("Database integrity error",
errorjson)
except Exception as ex:
raise ex
rowcount = cursor.rowcount
logger.info(
f"postgres query updated {rowcount} rows",
extra={
"query": sql,
"parameters": param_str,
"correlation_id": correlation_id,
},
)
results.append(rowcount)
conn.commit()
return results
def __init__(self, correlation_id=None):
acuity_credentials = utils.get_secret("acuity-connection")
self.session = requests.Session()
self.session.auth = (
acuity_credentials["user-id"],
acuity_credentials["api-key"],
)
self.logger = utils.get_logger()
self.calendars = None
self.correlation_id = correlation_id
self.app_endpoint = self.base_url + "appointments"
def execute_query(
base_sql,
params=None,
correlation_id=new_correlation_id(),
return_json=True,
jsonize_sql=True,
):
"""
Use this method to query the database (e.g. using SELECT). Changes will not be committed to the database, so don't use this method for UPDATE and DELETE
calls.
Args:
base_sql:
params (tuple or list): http://initd.org/psycopg/docs/usage.html#passing-parameters-to-sql-queries
correlation_id:
return_json:
jsonize_sql:
Returns:
"""
logger = get_logger()
# tell sql to create json if that's what's wanted
if return_json and jsonize_sql:
sql = _jsonize_sql(base_sql)
else:
sql = base_sql
sql = minimise_white_space(sql)
param_str = str(params)
logger.info(
"postgres query",
extra={
"query": sql,
"parameters": param_str,
"correlation_id": correlation_id
},
)
conn = _get_connection(correlation_id)
with conn.cursor() as cursor:
cursor.execute(sql, params)
records = cursor.fetchall()
logger.info(
"postgres result",
extra={
"rows returned": str(len(records)),
"correlation_id": correlation_id
},
)
if return_json:
return _get_json_from_tuples(records)
else:
return records
def cochrane_get(url):
full_url = utils.get_secret("cochrane-connection")["base_url"] + url
headers = dict()
headers["Content-Type"] = "application/json"
logger = utils.get_logger()
response = requests.get(full_url, headers=headers)
if response.ok:
data = response.json()
logger.info("API response", extra={"body": data})
return data
else:
raise utils.DetailedValueError("Cochrane API call failed",
details={"response": response.content})
def execute_query_multiple(
base_sql_tuple,
params_tuple=None,
correlation_id=new_correlation_id(),
return_json=True,
jsonize_sql=True,
):
"""
Use this method to query the database (e.g. using SELECT). Changes will not be committed to the database, so don't use this method for UPDATE and DELETE
calls.
"""
logger = get_logger()
conn = _get_connection(correlation_id)
if params_tuple is None:
params_tuple = tuple([None] * len(base_sql_tuple))
results = []
with conn.cursor() as cursor:
for (base_sql, params) in zip(base_sql_tuple, params_tuple):
# tell sql to create json if that's what's wanted
if return_json and jsonize_sql:
sql = _jsonize_sql(base_sql)
else:
sql = base_sql
sql = minimise_white_space(sql)
param_str = str(params)
logger.info(
"postgres query",
extra={
"query": sql,
"parameters": param_str,
"correlation_id": correlation_id,
},
)
cursor.execute(sql, params)
records = cursor.fetchall()
logger.info(
"postgres result",
extra={
"rows returned": str(len(records)),
"correlation_id": correlation_id,
},
)
if return_json:
results.append(_get_json_from_tuples(records))
else:
results.append(records)
logger.info("Returning multiple results", extra={"results": results})
return results
def _assign_link_to_user(self, unassigned_links: list) -> str:
"""
Assigns to user the unassigned link with the soonest expiration date.
An existing anon_project_specific_user_id is checked at assignment type to protect against
the unlikely but possible scenario where a concurrent invocation of the
lambda has assigned the same link to a different user.
Args:
unassigned_links (list): All unassigned links for this account_survey_id in PersonalLinks table
Returns:
A url representing the personal link assigned to this user
"""
# assign oldest link to user
unassigned_links.sort(key=lambda x: x["expires"])
user_id_attr_name = "anon_project_specific_user_id"
logger = utils.get_logger()
for unassigned_link in unassigned_links:
user_link = unassigned_link["url"]
try:
self.ddb_client.update_item(
table_name=const.PersonalLinksTable.NAME,
key=self.account_survey_id,
name_value_pairs={
"status": "assigned",
user_id_attr_name: self.anon_project_specific_user_id,
},
key_name="account_survey_id",
sort_key={"url": user_link},
ConditionExpression=Attr(user_id_attr_name).not_exists(),
)
except ClientError:
logger.info(
"Link assignment failed; link is already assigned to another user",
extra={
"user_link": user_link,
},
)
else:
return user_link
logger.info(
"Ran out of unassigned links; creating some more and retrying",
extra={
"unassigned_links": unassigned_links,
},
)
unassigned_links = self._create_personal_links()
return self._assign_link_to_user(unassigned_links)
def __init__(self, logger=None, correlation_id=None, test_event=False):
"""
Args:
logger:
correlation_id:
test_event: set to True when the event originates from unittests
"""
self.logger = logger
if logger is None:
self.logger = utils.get_logger()
self.core_api_client = CoreApiClient(correlation_id=correlation_id)
self.correlation_id = correlation_id
self.test_mode = test_event
self.appointment = None
self.original_booking = None # used for determining if interviewer has changed when processing rescheduling events
def main(webhook_ids):
logger = utils.get_logger()
ac = AcuityClient()
webhooks = ac.get_webhooks()
env_name = utils.get_environment_name()
for wh in webhooks:
if wh["id"] in webhook_ids:
print("Acuity webhook details:")
pprint(wh)
confirmation = input(
"Are you sure you want to delete the webhook above? (y/n)")
if confirmation.lower() == "y":
ac.delete_webhooks(wh["id"])
logger.info("Deleted Acuity webhook", extra={"webhook": wh})
print("")
def wrapper(*args, **kwargs):
response = func(*args, **kwargs)
if response.ok:
try:
return response.json()
except JSONDecodeError:
return response
else:
logger = utils.get_logger()
logger.error(
f"Acuity API call failed with response: {response}",
extra={"response.content": response.content},
)
raise utils.DetailedValueError(
f"Acuity API call failed with response: {response}",
details={"response": response.content},
)
def _get_connection(correlation_id=None):
logger = get_logger()
if config.conn is None:
env_dict = get_secret("database-connection")
if env_dict is None:
raise ConnectionError(
"Could not get database-connection secret from AWS")
config.conn = psycopg2.connect(**env_dict)
# using dsn obscures password
logger.info(
"created database connection",
extra={
"conn_string": config.conn.dsn,
"correlation_id": correlation_id
},
)
return config.conn
def __init__(self, **kwargs):
self._logger = utils.get_logger()
self._correlation_id = kwargs.get("correlation_id")
self.destination_survey_id = kwargs.get("survey_id")
self.destination_response_id = kwargs.get("response_id")
err_message = "Call to initialise_survey missing mandatory data: {}"
assert self.destination_survey_id, err_message.format("survey_id")
assert self.destination_survey_id, err_message.format("response_id")
self.survey_init_config = SurveyInitConfig(
destination_survey_id=self.destination_survey_id,
correlation_id=self._correlation_id,
)
self.survey_init_config.get()
try:
self.survey_init_config.details
except AttributeError:
raise utils.ObjectDoesNotExistError(
f"Initialisation config not found for survey {self.destination_survey_id}",
details={},
)
self.destination_account = kwargs.get("account", "cambridge")
self.anon_project_specific_user_id = kwargs.get(
"anon_project_specific_user_id")
assert self.anon_project_specific_user_id, err_message.format(
"anon_project_specific_user_id")
self._core_client = CoreApiClient(correlation_id=self._correlation_id)
user = self._core_client.get_user_by_anon_project_specific_user_id(
self.anon_project_specific_user_id)
self.user_id = user["id"]
self.ddb_client = Dynamodb(stack_name=const.STACK_NAME,
correlation_id=self._correlation_id)
self.cached_responses = dict()
self.missing_responses = list()
self.target_embedded_data = dict()
self.responses_client = ResponsesClient(
survey_id=self.destination_survey_id,
qualtrics_account_name=self.destination_account,
correlation_id=self._correlation_id,
)
def set_interview_url(appointment_id,
interview_url,
event_type,
logger=None,
correlation_id=None):
"""
Args:
appointment_id:
interview_url:
event_type (str): passed on to AppointmentNotifier ('booking', 'rescheduling' or 'cancellation')
logger:
correlation_id:
Returns:
update_result:
notification_results (dict): Dictionary containing participant and researchers notification results
"""
if logger is None:
logger = utils.get_logger()
appointment = AcuityAppointment(
appointment_id=appointment_id,
logger=logger,
correlation_id=correlation_id,
)
appointment.ddb_load()
update_result = appointment.update_link(link=interview_url)
notification_results = {
"participant": None,
"researchers": list(),
}
if appointment.appointment_type.send_notifications is True:
notifier = AppointmentNotifier(
appointment=appointment,
logger=logger,
correlation_id=correlation_id,
)
notification_results = notifier.send_notifications(
event_type=event_type)
return update_result, notification_results
def process_notifications(event, context):
logger = get_logger()
notifications = c_notif.get_notifications_to_process(
stack_name=const.STACK_NAME)
logger.info("process_notifications",
extra={"count": str(len(notifications))})
# note that we need to process all registrations first, then do task signups (otherwise we might try to process a signup for someone not yet registered)
signup_notifications = list()
login_notifications = list()
transactional_emails = list()
for notification in notifications:
notification_type = notification["type"]
if notification_type == NotificationType.USER_REGISTRATION.value:
pass
# process_user_registration(notification)
elif notification_type == NotificationType.TASK_SIGNUP.value:
# add to list for later processing
signup_notifications.append(notification)
elif notification_type == NotificationType.USER_LOGIN.value:
# add to list for later processing
login_notifications.append(notification)
elif notification_type == NotificationType.TRANSACTIONAL_EMAIL.value:
transactional_emails.append(notification)
else:
error_message = (
f"Processing of {notification_type} notifications not implemented yet"
)
logger.error(error_message)
raise NotImplementedError(error_message)
for signup_notification in signup_notifications:
pass
# process_task_signup(signup_notification)
for login_notification in login_notifications:
process_user_login(login_notification)
for email in transactional_emails:
pass
def main():
logger = get_logger()
survey_client = SurveyDefinitionsClient(survey_id=SURVEY_ID)
survey = survey_client.get_survey()["result"]
questions = survey["Questions"]
t2_to_t1_extended_mapping = get_extended_mapping()
for k, v in questions.items():
export_tag = v["DataExportTag"]
question_text = v["QuestionText"]
if export_tag in t2_to_t1_extended_mapping:
if (m := PROMPT_QUESTION_RE.search(question_text)) or (
m := SLIDER_QUESTION_RE.search(question_text)):
logger.debug(
"Expected substitution",
extra={
"before": m.group("question"),
"after": t2_to_t1_extended_mapping[export_tag],
},
)
v["QuestionText"] = question_text.replace(
m.group("question"), t2_to_t1_extended_mapping[export_tag])
survey_client.update_question(question_id=k, data=v)
def translate_ids(col_to_fetch):
logger = utils.get_logger()
user_input = input(
"Please paste list of anon_project_specific_user_ids separated by commas:"
)
anon_ids = split_string_into_list_of_strings(user_input)
user_ids = list()
unresolved_ids = list()
for anon_project_specific_user_id in anon_ids:
result = pg_utils.execute_query(
sql_q.GET_USER_BY_ANON_PROJECT_SPECIFIC_USER_ID_SQL,
[str(anon_project_specific_user_id)],
)
try:
data_as_list = []
for col in col_to_fetch:
data_as_list.append(str(result[0][col]))
data_as_str = ", ".join(data_as_list)
user_ids.append(data_as_str)
except IndexError:
logger.warning(
f"anon_project_specific_user_id {anon_project_specific_user_id} could not be found"
)
unresolved_ids.append(anon_project_specific_user_id)
pg_utils.close_connection()
if unresolved_ids:
print("\nThe following anon_project_specific_user_ids could not be translated (%s):" % len(unresolved_ids))
print(";\n".join(unresolved_ids))
if user_ids:
print("\nTranslated ids (%s):" % len(user_ids))
print(";\n".join(user_ids))
def main(response_dataset, account, survey_id):
logger = utils.get_logger()
with open(response_dataset) as csvfile:
reader = csv.DictReader(csvfile)
row_counter = 0
for row in reader:
row_counter += 1
logger.info(f"Working on row {row_counter}")
try:
response_id = row.pop("ResponseId")
event_time = row.pop("RecordedDate")
anon_project_specific_user_id = row.pop(
"anon_project_specific_user_id")
anon_user_task_id = row.pop("anon_user_task_id")
except KeyError as exc:
raise utils.DetailedValueError(
f"Mandatory {exc} data not found in source event",
details={
"row": row,
},
)
try:
utils.validate_uuid(anon_project_specific_user_id)
except utils.DetailedValueError:
logger.warning(
f"Could not process row because anon_project_specific_user_id is not valid: {anon_project_specific_user_id}"
)
continue
emulated_event = user_task_completed_event_template.copy()
emulated_event["time"] = event_time
emulated_event["detail"] = {
"anon_project_specific_user_id": anon_project_specific_user_id,
"anon_user_task_id": anon_user_task_id,
"response_id": f"{survey_id}-{response_id}",
"account": account,
}
resp.put_task_response(emulated_event, {})
