def remove_replies_to_treatments(self):
comments = self.get_comment_objects_for_experiment_comment_replies(
self.get_all_experiment_comment_replies())
removed_comment_ids = []
parent_submission_ids = set()
for comment in comments:
if (comment.banned_by is None):
comment.remove()
removed_comment_ids.append(comment.id)
parent_submission_ids.add(comment.link_id)
experiment_action = ExperimentAction(experiment_id=self.experiment.id,
action="RemoveRepliesToTreatment",
metadata_json=json.dumps({
"parent_submission_ids":
list(parent_submission_ids),
"removed_comment_ids":
removed_comment_ids
}))
self.db_session.add(experiment_action)
self.db_session.commit()
self.log.info(
"{controller}: Experiment {experiment}: found {replies} replies to {treatments} treatment comments. Removed {removed} comments."
.format(controller=self.__class__.__name__,
experiment=self.experiment.id,
replies=len(comments),
treatments=len(parent_submission_ids),
removed=len(removed_comment_ids)))
return len(removed_comment_ids)
def make_control_nonaction(self,
experiment_thing,
submission,
group="control"):
if (self.submission_acceptable(submission) == False):
return None
metadata = json.loads(experiment_thing.metadata_json)
treatment_arm = int(metadata['randomization']['treatment'])
condition = metadata['condition']
experiment_action = ExperimentAction(
experiment_id=self.experiment.id,
praw_key_id=PrawKey.get_praw_id(ENV, self.experiment_name),
action="Intervention",
action_object_type=ThingType.SUBMISSION.value,
action_object_id=submission.id,
metadata_json=json.dumps({
"group": group,
"condition": condition,
"arm": "arm_" + str(treatment_arm)
}))
self.db_session.add(experiment_action)
self.db_session.commit()
self.log.info(
"{0}: Experiment {1} applied arm {2} to post {3} (condition = {4})"
.format(self.__class__.__name__, self.experiment_name,
treatment_arm, submission.id, condition))
return experiment_action.id
def send_surveys(self):
experiment_id = self.experiment_controller.experiment.id
experiment_name = self.experiment_controller.experiment.name
try:
with self.db_session.cooplock(self.survey_task_id, experiment_id):
user_things = self.experiment_controller.get_surveyable_user_things(
)
self.log.info(
"Experiment {0}: identified {1} surveyable users.".format(
experiment_name, len(user_things)))
self.log.info("Experiment {0}: messaging users: {1}".format(
experiment_name,
[user_thing.thing_id for user_thing in user_things]))
message_dicts = self.get_unique_recipient_message_dicts(
user_things)
responses = self.messaging_controller.send_messages(
message_dicts, self.survey_task_id)
updates = []
for user_thing in user_things:
if user_thing.thing_id in responses.keys():
response = responses[user_thing.thing_id]
self.append_surveyed_user_log(user_thing.thing_id,
response)
if len(response.get("errors", {})) > 0:
self.log.info(
"Experiment {0}: failed to survey user {1}: {2}"
.format(experiment_name, user_thing.thing_id,
str(response)))
self._update_metadata(user_thing, "survey_status",
"failed")
updates.append(user_thing)
else:
self.log.info(
"Experiment {0}: successfully surveyed user {1}: {2}"
.format(experiment_name, user_thing.thing_id,
str(response)))
survey_action = ExperimentAction(
experiment_id=experiment_id,
action="SendSurvey",
action_object_type=ThingType.USER.value,
action_object_id=user_thing.thing_id)
self._update_metadata(survey_action,
"survey_status", "sent")
self._update_metadata(user_thing, "survey_status",
"sent")
updates.append(survey_action)
updates.append(user_thing)
if updates:
self.db_session.add_retryable(updates)
except Exception as e:
self.log.exception(
"Experiment {0}: error occurred while sending surveys: ".
format(experiment_name))
def submission_acceptable(self, submission):
if (submission is None):
## TODO: Determine what to do if you can't find the post
self.log.error("{0}: Can't find experiment {1} post {2}".format(
self.__class__.__name__, self.subreddit, submission.id))
return False
## Avoid Acting if the Intervention has already been recorded
if (self.db_session.query(ExperimentAction).filter(
and_(
ExperimentAction.experiment_id == self.experiment.id,
ExperimentAction.action_object_type
== ThingType.SUBMISSION.value,
ExperimentAction.action_object_id == submission.id,
ExperimentAction.action == "Intervention")).count() > 0):
self.log.info(
"{0}: Experiment {1} post {2} already has an Intervention recorded"
.format(self.__class__.__name__, self.experiment_name,
submission.id))
return False
## possible comment texts to avoid
all_experiment_messages = []
for label, condition in self.experiment_settings['conditions'].items():
all_experiment_messages = all_experiment_messages + list(
condition['arms'].values())
## Avoid Acting if an identical sticky comment already exists
for comment in submission.comments:
if (hasattr(comment, "stickied") and comment.stickied
and (comment.body in all_experiment_messages)):
self.log.info(
"{0}: Experiment {1} post {2} already has a sticky comment {2}"
.format(self.__class__.__name__, self.experiment_name,
submission.id, comment.id))
return False
## Avoid Acting if the submission is not recent enough
curtime = time.time()
# if((curtime - submission.created_utc) > 10800):
if ((curtime - submission.created_utc) > self.max_eligibility_age):
self.log.info(
"{0}: Submission created_utc {1} is {2} seconds greater than current time {3}, exceeding the max eligibility age of {4}. Declining to Add to the Experiment"
.format(self.__class__.__name__, submission.created_utc,
curtime - submission.created_utc, curtime,
self.max_eligibility_age))
experiment_action = ExperimentAction(
experiment_id=self.experiment.id,
praw_key_id=PrawKey.get_praw_id(ENV, self.experiment_name),
action="NonIntervention:MaxAgeExceeded",
action_object_type=ThingType.SUBMISSION.value,
action_object_id=submission.id)
return False
return True
def make_sticky_post(self, submission):
if(self.submission_acceptable(submission) == False):
return None
## FIRST STEP: DETERMINE AMA STATUS
## TODO: refactor to check this only once
is_ama = self.is_ama(submission)
if(is_ama):
comment_text = self.ama_comment_text
else:
comment_text = self.nonama_comment_text
comment = submission.add_comment(comment_text)
distinguish_results = comment.distinguish(sticky=True)
self.log.info("Experiment {0} applied treatment to post {1} (AMA = {2}). Result: {3}".format(
self.experiment_name,
submission.id,
is_ama,
str(distinguish_results)
))
experiment_action = ExperimentAction(
experiment_id = self.experiment.id,
praw_key_id = PrawKey.get_praw_id(ENV, self.experiment_name),
action_subject_type = ThingType.COMMENT.value,
action_subject_id = comment.id,
action = "Intervention",
action_object_type = ThingType.SUBMISSION.value,
action_object_id = submission.id,
metadata_json = json.dumps({"group":"treatment",
"action_object_created_utc":comment.created_utc})
)
comment_thing = ExperimentThing(
experiment_id = self.experiment.id,
object_created = datetime.datetime.fromtimestamp(comment.created_utc),
object_type = ThingType.COMMENT.value,
id = comment.id,
metadata_json = json.dumps({"group":"treatment","submission_id":submission.id})
)
self.db_session.add(comment_thing)
self.db_session.add(experiment_action)
self.db_session.commit()
return distinguish_results
def make_control_nonaction(self, submission):
if(self.submission_acceptable(submission) == False):
return None
experiment_action = ExperimentAction(
experiment_id = self.experiment.id,
praw_key_id = PrawKey.get_praw_id(ENV, self.experiment_name),
action = "Intervention",
action_object_type = ThingType.SUBMISSION.value,
action_object_id = submission.id,
metadata_json = json.dumps({"group":"control"})
)
self.db_session.add(experiment_action)
self.db_session.commit()
self.log.info("Experiment {0} applied control condition to post {1}".format(
self.experiment_name,
submission.id
))
return experiment_action.id
def submission_acceptable(self, submission):
if(submission is None):
## TODO: Determine what to do if you can't find the post
self.log.error("Can't find experiment {0} post {1}".format(self.subreddit, submission.id))
return False
## Avoid Acting if the Intervention has already been recorded
if(self.db_session.query(ExperimentAction).filter(and_(
ExperimentAction.experiment_id == self.experiment.id,
ExperimentAction.action_object_type == ThingType.SUBMISSION.value,
ExperimentAction.action_object_id == submission.id,
ExperimentAction.action == "Intervention")).count() > 0):
self.log.info("Experiment {0} post {1} already has an Intervention recorded".format(
self.experiment_name,
submission.id))
return False
## Avoid Acting if an identical sticky comment already exists
for comment in submission.comments:
if(comment.stickied and (comment.body == self.ama_comment_text or comment.body == self.nonama_comment_text)):
self.log.info("Experiment {0} post {1} already has a sticky comment {2}".format(
self.experiment_name,
submission.id,
comment.id))
return False
## Avoid Acting if the submission is not recent enough
curtime = time.time()
# if((curtime - submission.created_utc) > 10800):
if((curtime - submission.created_utc) > self.max_eligibility_age):
self.log.info("Submission created_utc {0} is {1} seconds greater than current time {2}, exceeding the max eligibility age of {3}. Declining to Add to the Experiment".format(
submission.created_utc,
curtime - submission.created_utc,
curtime,
self.max_eligibility_age))
experiment_action = ExperimentAction(
experiment_id = self.experiment.id,
praw_key_id = PrawKey.get_praw_id(ENV, self.experiment_name),
action = "NonIntervention:MaxAgeExceeded",
action_object_type = ThingType.SUBMISSION.value,
action_object_id = submission.id
)
return False
return True
def send_messages(self, experiment_things):
self.db_session.execute("Lock Tables experiment_actions WRITE, experiment_things WRITE, message_logs WRITE")
message_results = []
try:
mc = MessagingController(self.db_session, self.r, self.log)
action = "SendMessage"
messages_to_send = []
for experiment_thing in experiment_things:
message = self.format_message(experiment_thing)
## if it's a control group, log inaction
## and do nothing, otherwise add to messages_to_send
if message is None:
metadata = json.loads(experiment_thing.metadata_json)
metadata['message_status'] = "sent"
ea = ExperimentAction(
experiment_id = self.experiment.id,
action = action,
action_object_type = ThingType.USER.value,
action_object_id = experiment_thing.id,
metadata_json = json.dumps(metadata)
)
experiment_thing.query_index = "Intervention Complete"
experiment_thing.metadata_json = json.dumps(metadata)
self.db_session.add(ea)
else:
message['account'] = experiment_thing.thing_id
messages_to_send.append(message)
# send messages_to_send
message_results = mc.send_messages(messages_to_send,
"NewcomerMessagingExperiment({0})::send_messages".format(
self.experiment_name))
# iterate through message_result, linked with experiment_things
for experiment_thing in experiment_things:
if(experiment_thing.thing_id in message_results.keys()):
message_result = message_results[experiment_thing.thing_id]
metadata = json.loads(experiment_thing.metadata_json)
update_records = False
message_errors = 0
if 'errors' in message_result:
message_errors = len(message_result['errors'])
## TAKE ACTION WITH INVALID USERNAME
## (add an action and UPDATE THE EXPERIMENT_THING)
## TO INDICATE THAT THE ACCOUNT DOESN'T EXIST
## NOTE: THE MESSAGE ATTEMPT WILL BE LOGGED
## SO YOU DON'T HAVE TO LOG AN ExperimentAction
## Ignore other errors
## (since you will want to retry in those cases)
if(message_errors > 0):
for error in message_result['errors']:
invalid_username = False
if error['error'] == 'invalid username':
invalid_username = True
if(invalid_username):
metadata['message_status'] = "nonexistent"
metadata['survey_status'] = "nonexistent"
experiment_thing.query_index = "Intervention Impossible"
update_records = True
## if there are no errors
## add an ExperimentAction and
## update the experiment_thing metadata
else:
metadata['message_status'] = "sent"
experiment_thing.query_index = "Intervention Complete"
update_records = True
if(update_records):
metadata_json = json.dumps(metadata)
ea = ExperimentAction(
experiment_id = self.experiment.id,
action = action,
action_object_type = ThingType.USER.value,
action_object_id = experiment_thing.id,
metadata_json = metadata_json
)
self.db_session.add(ea)
experiment_thing.metadata_json = metadata_json
self.db_session.commit()
except(Exception) as e:
self.db_session.execute("UNLOCK TABLES")
self.log.error("Error in NewcomerMessagingExperimentController::send_messages", extra=sys.exc_info()[0])
return []
self.db_session.execute("UNLOCK TABLES")
return message_results
def set_stylesheet(self, condition, arm):
arms = self.experiment_settings['conditions'][condition]['arms']
intervention_line = arms[arm]
found_code = False
stylesheet_data = self.r.get_stylesheet(self.subreddit)
if "stylesheet" in stylesheet_data.keys():
line_list = []
for line in stylesheet_data['stylesheet'].split("\n"):
## IF A LINE FROM THE STUDY IS FOUND,
## REPLACE IT WITH THE INTERVENTION
if line in arms.values():
line = intervention_line
found_code = True
line_list.append(line)
## IF THE CODE IS NOT FOUND, ADD IT TO THE END
if (found_code != True):
line_list.append("/* CivilServantBot Experiment CSS */")
line_list.append(intervention_line)
line_list.append("")
new_stylesheet = "\n".join(line_list)
result = self.r.set_stylesheet(self.subreddit, new_stylesheet)
if ('errors' in result.keys() and len(result['errors']) == 0):
self.log.info(
"{0}: Experiment {1}: Applied Arm {3} of Condition {4} in {2}".
format(self.__class__.__name__, self.experiment.id,
self.subreddit, arm, condition))
experiment_action = ExperimentAction(
experiment_id=self.experiment.id,
praw_key_id=PrawKey.get_praw_id(ENV, self.experiment_name),
action="Intervention",
action_object_type=ThingType.STYLESHEET.value,
action_object_id=None,
metadata_json=json.dumps({
"arm": arm,
"condition": condition
}))
self.db_session.add(experiment_action)
self.db_session.commit()
else:
self.log.error(
"{0}: Experiment {1}: Failed to apply Arm {2} of Condition {3}. Reddit errors: {4}"
.format(self.__class__.__name__, self.experiment.id,
self.subreddit, arm, condition,
", ".join(result['errors'])))
# experiment_action = ExperimentAction(
# experiment_id = self.experiment.id,
# praw_key_id = PrawKey.get_praw_id(ENV, self.experiment_name),
# action = "NonIntervention:PrawError.{0}.{1}".format(condition,arm),
# action_object_type = ThingType.STYLESHEET.value,
# action_object_id = None
# )
# self.db_session.add(experiment_action)
# self.db_session.commit()
## IF WE FAILED TO APPLY THE INTERVENTION, ROLL BACK THAT RANDOMIZATION
self.experiment_settings['conditions'][condname][
'next_randomization'] -= 1
self.experiment.settings_json = json.dumps(
self.experiment_settings)
self.db_session.commit()
## TO HELP WITH TESTING, RETURN THE FULL TEXT OF THE STYLESHEET
return new_stylesheet
def make_sticky_post(self, experiment_thing, submission):
if (self.submission_acceptable(submission) == False):
return None
metadata = json.loads(experiment_thing.metadata_json)
treatment_arm = int(metadata['randomization']['treatment'])
condition = metadata['condition']
comment_text = self.experiment_settings['conditions'][condition][
'arms']["arm_" + str(treatment_arm)]
## THIS METHOD IS FOR USE WHEN INTENDING TO TEST AN EXPERIMENT WITHOUT
## MAKING ANY ACTUAL COMMENTS ON REDDIT
#def _TEST_add_comment():
# import collections, random, string
# return collections.namedtuple("Comment", ["id", "created_utc"])(
# "_" + "".join(random.choice(string.ascii_lowercase) for i in range(6)),
# datetime.datetime.utcnow().timestamp())
## TO USE THIS DRY RUN TEST CODE, UNCOMMENT THE FOLLOWING TWO LINES
## AND COMMENT OUT THE TWO LINES AFTER IT
#comment = _TEST_add_comment() #submission.add_comment(comment_text)
## distinguish_results = "DUMMY DISTINGUISH: Assume successful"
comment = submission.add_comment(comment_text)
distinguish_results = comment.distinguish(sticky=True)
self.log.info(
"{0}: Experiment {1} applied arm {2} to post {3} (condition = {4}). Result: {5}"
.format(self.__class__.__name__, self.experiment_name,
treatment_arm, submission.id, condition,
str(distinguish_results)))
experiment_action = ExperimentAction(
experiment_id=self.experiment.id,
praw_key_id=PrawKey.get_praw_id(ENV, self.experiment_name),
action_subject_type=ThingType.COMMENT.value,
action_subject_id=comment.id,
action="Intervention",
action_object_type=ThingType.SUBMISSION.value,
action_object_id=submission.id,
metadata_json=json.dumps({
"group":
"treatment",
"condition":
condition,
"arm":
"arm_" + str(treatment_arm),
"randomization":
metadata['randomization'],
"action_object_created_utc":
comment.created_utc
}))
comment_thing = ExperimentThing(
experiment_id=self.experiment.id,
object_created=datetime.datetime.fromtimestamp(
comment.created_utc),
object_type=ThingType.COMMENT.value,
id=comment.id,
metadata_json=json.dumps({
"group": "treatment",
"arm": "arm_" + str(treatment_arm),
"condition": condition,
"randomization": metadata['randomization'],
"submission_id": submission.id
}))
self.db_session.add(comment_thing)
self.db_session.add(experiment_action)
self.db_session.commit()
return distinguish_results