def test_identify_condition(mock_subreddit, mock_reddit):
r = mock_reddit.return_value
experiment_name_to_controller = {
"sticky_comment_0": AMAStickyCommentExperimentController,
"sticky_comment_frontpage_test":
FrontPageStickyCommentExperimentController
}
for experiment_name in experiment_name_to_controller:
with open(
os.path.join(BASE_DIR, "config", "experiments") + "/" +
experiment_name + ".yml", "r") as f:
experiment_settings = yaml.load(f.read())['test']
sub_data = []
with open("{script_dir}/fixture_data/subreddit_posts_0.json".format(
script_dir=TEST_DIR)) as f:
fixture = [
x['data'] for x in json.loads(f.read())['data']['children']
]
for post in fixture:
json_dump = json.dumps(post)
postobj = json2obj(json_dump)
sub_data.append(postobj)
mock_subreddit.get_new.return_value = sub_data
mock_subreddit.display_name = experiment_settings['subreddit']
mock_subreddit.name = experiment_settings['subreddit']
mock_subreddit.id = experiment_settings['subreddit_id']
r.get_subreddit.return_value = mock_subreddit
patch('praw.')
## TEST THE BASE CASE OF RANDOMIZATION
controller = experiment_name_to_controller[experiment_name]
controller_instance = controller(experiment_name, db_session, r, log)
# "mock" FrontPageController.posts
if controller_instance.__class__ is FrontPageStickyCommentExperimentController:
instance = FrontPageController(db_session, r, log)
instance.posts = sub_data
if controller_instance.__class__ is FrontPageStickyCommentExperimentController:
objs = controller_instance.set_eligible_objects(instance)
elif controller_instance.__class__ is AMAStickyCommentExperimentController:
objs = controller_instance.set_eligible_objects()
eligible_objects = controller_instance.get_eligible_objects(objs)
condition_list = []
for obj in eligible_objects:
condition_list.append(controller_instance.identify_condition(obj))
if controller_instance.__class__ is FrontPageStickyCommentExperimentController:
assert Counter(condition_list)['frontpage_post'] == 100
elif controller_instance.__class__ is AMAStickyCommentExperimentController:
assert Counter(condition_list)['nonama'] == 98
assert Counter(condition_list)['ama'] == 2
clear_all_tables()
def test_identify_condition(mock_subreddit, mock_reddit):
r = mock_reddit.return_value
experiment_name_to_controller = {
"sticky_comment_0": AMAStickyCommentExperimentController,
"sticky_comment_frontpage_test": FrontPageStickyCommentExperimentController
}
for experiment_name in experiment_name_to_controller:
with open(os.path.join(BASE_DIR, "config", "experiments") + "/"+ experiment_name + ".yml", "r") as f:
experiment_settings = yaml.load(f.read())['test']
sub_data = []
with open("{script_dir}/fixture_data/subreddit_posts_0.json".format(script_dir=TEST_DIR)) as f:
fixture = [x['data'] for x in json.loads(f.read())['data']['children']]
for post in fixture:
json_dump = json.dumps(post)
postobj = json2obj(json_dump)
sub_data.append(postobj)
mock_subreddit.get_new.return_value = sub_data
mock_subreddit.display_name = experiment_settings['subreddit']
mock_subreddit.name = experiment_settings['subreddit']
mock_subreddit.id = experiment_settings['subreddit_id']
r.get_subreddit.return_value = mock_subreddit
patch('praw.')
## TEST THE BASE CASE OF RANDOMIZATION
controller = experiment_name_to_controller[experiment_name]
controller_instance = controller(experiment_name, db_session, r, log)
# "mock" FrontPageController.posts
if controller_instance.__class__ is FrontPageStickyCommentExperimentController:
instance = FrontPageController(db_session, r, log)
instance.posts = sub_data
if controller_instance.__class__ is FrontPageStickyCommentExperimentController:
objs = controller_instance.set_eligible_objects(instance)
elif controller_instance.__class__ is AMAStickyCommentExperimentController:
objs = controller_instance.set_eligible_objects()
eligible_objects = controller_instance.get_eligible_objects(objs)
condition_list = []
for obj in eligible_objects:
condition_list.append(controller_instance.identify_condition(obj))
if controller_instance.__class__ is FrontPageStickyCommentExperimentController:
assert Counter(condition_list)['frontpage_post'] == 100
elif controller_instance.__class__ is AMAStickyCommentExperimentController:
assert Counter(condition_list)['nonama'] == 98
assert Counter(condition_list)['ama'] == 2
clear_all_tables()
def test_update_experiment(intervene_ama_arm_1, intervene_ama_arm_0,
intervene_nonama_arm_1, intervene_nonama_arm_0,
intervene_frontpage_post_arm_1,
intervene_frontpage_post_arm_0, mock_subreddit,
mock_reddit):
r = mock_reddit.return_value
experiment_name_to_controller = {
"sticky_comment_0": AMAStickyCommentExperimentController,
"sticky_comment_frontpage_test":
FrontPageStickyCommentExperimentController
}
for experiment_name in experiment_name_to_controller:
with open(
os.path.join(BASE_DIR, "config", "experiments") + "/" +
experiment_name + ".yml", "r") as f:
experiment_settings = yaml.load(f.read())['test']
sub_data = []
with open("{script_dir}/fixture_data/subreddit_posts_0.json".format(
script_dir=TEST_DIR)) as f:
fixture = [
x['data'] for x in json.loads(f.read())['data']['children']
]
for post in fixture:
json_dump = json.dumps(post)
postobj = json2obj(json_dump)
sub_data.append(postobj)
mock_subreddit.get_new.return_value = sub_data
mock_subreddit.display_name = experiment_settings['subreddit']
mock_subreddit.name = experiment_settings['subreddit']
mock_subreddit.id = experiment_settings['subreddit_id']
r.get_subreddit.return_value = mock_subreddit
patch('praw.')
controller = experiment_name_to_controller[experiment_name]
controller_instance = controller(experiment_name, db_session, r, log)
# "mock" FrontPageController.posts
if controller_instance.__class__ is FrontPageStickyCommentExperimentController:
instance = FrontPageController(db_session, r, log)
instance.posts = sub_data
## GET RANDOMIZATIONS
#objs = scec.set_eligible_objects()
#eligible_objects = scec.get_eligible_objects(objs)
#experiment_things = scec.assign_randomized_conditions(eligible_objects)
assert db_session.query(ExperimentThing).count() == 0
## MOCK RETURN VALUES FROM ACTION METHODS
## This will never be saved. Just creating an object
comment_thing = ExperimentThing(
experiment_id=controller_instance.experiment.id,
object_created=None,
object_type=ThingType.COMMENT.value,
id="12345",
metadata_json=json.dumps({
"group": None,
"submission_id": None
}))
# for AMAStickyCommentExperimentController
intervene_nonama_arm_0.return_value = comment_thing
intervene_nonama_arm_1.return_value = comment_thing
intervene_ama_arm_0.return_value = comment_thing
intervene_ama_arm_1.return_value = comment_thing
# for FrontPageStickyCommentExperimentController
intervene_frontpage_post_arm_0.return_value = comment_thing
intervene_frontpage_post_arm_1.return_value = comment_thing
if controller_instance.__class__ is FrontPageStickyCommentExperimentController:
experiment_return = controller_instance.callback_update_experiment(
instance)
elif controller_instance.__class__ is AMAStickyCommentExperimentController:
experiment_return = controller_instance.update_experiment()
## ASSERT THAT THE METHODS FOR TAKING ACTIONS ARE CALLED
if controller_instance.__class__ is FrontPageStickyCommentExperimentController:
assert intervene_frontpage_post_arm_0.called == True
assert intervene_frontpage_post_arm_1.called == True
elif controller_instance.__class__ is AMAStickyCommentExperimentController:
assert intervene_nonama_arm_0.called == True
assert intervene_nonama_arm_1.called == True
assert intervene_ama_arm_0.called == True
assert intervene_ama_arm_1.called == True
assert len(experiment_return) == 100
clear_all_tables()
def test_assign_randomized_conditions(mock_subreddit, mock_reddit):
r = mock_reddit.return_value
experiment_name_to_controller = {
"sticky_comment_0": AMAStickyCommentExperimentController,
"sticky_comment_frontpage_test":
FrontPageStickyCommentExperimentController
}
for experiment_name in experiment_name_to_controller:
with open(
os.path.join(BASE_DIR, "config", "experiments") + "/" +
experiment_name + ".yml", "r") as f:
experiment_settings = yaml.load(f.read())['test']
sub_data = []
with open("{script_dir}/fixture_data/subreddit_posts_0.json".format(
script_dir=TEST_DIR)) as f:
fixture = [
x['data'] for x in json.loads(f.read())['data']['children']
]
for post in fixture:
json_dump = json.dumps(post)
postobj = json2obj(json_dump)
sub_data.append(postobj)
mock_subreddit.get_new.return_value = sub_data
mock_subreddit.display_name = experiment_settings['subreddit']
mock_subreddit.name = experiment_settings['subreddit']
mock_subreddit.id = experiment_settings['subreddit_id']
r.get_subreddit.return_value = mock_subreddit
patch('praw.')
controller = experiment_name_to_controller[experiment_name]
controller_instance = controller(experiment_name, db_session, r, log)
# "mock" FrontPageController.posts
if controller_instance.__class__ is FrontPageStickyCommentExperimentController:
instance = FrontPageController(db_session, r, log)
instance.posts = sub_data
## TEST THE BASE CASE OF RANDOMIZATION
if controller_instance.__class__ is FrontPageStickyCommentExperimentController:
objs = controller_instance.set_eligible_objects(instance)
elif controller_instance.__class__ is AMAStickyCommentExperimentController:
objs = controller_instance.set_eligible_objects()
eligible_objects = controller_instance.get_eligible_objects(objs)
assert len(eligible_objects) == 100 ####
experiment_action_count = db_session.query(ExperimentAction).count()
experiment_settings = json.loads(
controller_instance.experiment.settings_json)
for condition_name in experiment_settings['conditions']:
assert experiment_settings['conditions'][condition_name][
'next_randomization'] == 0
assert db_session.query(ExperimentThing).count() == 0
controller_instance.assign_randomized_conditions(eligible_objects)
assert db_session.query(ExperimentThing).count() == 100
assert len(db_session.query(Experiment).all()) == 1
experiment = db_session.query(Experiment).first()
experiment_settings = json.loads(experiment.settings_json)
if controller_instance.__class__ is AMAStickyCommentExperimentController:
assert experiment_settings['conditions']['ama'][
'next_randomization'] == 2
assert experiment_settings['conditions']['nonama'][
'next_randomization'] == 98
assert sum([
x['next_randomization']
for x in list(experiment_settings['conditions'].values())
]) == 100
elif controller_instance.__class__ is FrontPageStickyCommentExperimentController:
assert experiment_settings['conditions']['frontpage_post'][
'next_randomization'] == 100
for experiment_thing in db_session.query(ExperimentThing).all():
assert experiment_thing.id != None
assert experiment_thing.object_type == ThingType.SUBMISSION.value
assert experiment_thing.experiment_id == controller_instance.experiment.id
assert "randomization" in json.loads(
experiment_thing.metadata_json).keys()
assert "condition" in json.loads(
experiment_thing.metadata_json).keys()
## TEST THE CASE WHERE THE AMA EXPERIMENT HAS CONCLUDED
### first step: set the condition counts to have just one remaining condition left
for condition_name in experiment_settings['conditions']:
experiment_settings['conditions'][condition_name][
'next_randomization'] = len(
experiment_settings['conditions'][condition_name]
['randomizations']) - 1
controller_instance.experiment_settings['conditions'][
condition_name]['next_randomization'] = experiment_settings[
'conditions'][condition_name]['next_randomization']
experiment_settings_json = json.dumps(experiment_settings)
db_session.commit()
posts = []
if controller_instance.__class__ is AMAStickyCommentExperimentController:
posts += [x for x in sub_data
if "ama" in x.link_flair_css_class][0:2]
posts += [
x for x in sub_data if "ama" not in x.link_flair_css_class
][0:2]
elif controller_instance.__class__ is FrontPageStickyCommentExperimentController:
posts += sub_data[0:4]
## generate new fake ids for these fixture posts,
## which would otherwise be duplicates
new_posts = []
for post in posts:
post = post.json_dict
post['id'] = ''.join(
random.SystemRandom().choice(string.ascii_uppercase +
string.digits) for _ in range(7))
new_posts.append(json2obj(json.dumps(post)))
# Only 1 randomization left for each condition, while there are >1 new_posts
assert len(new_posts) == 4
experiment_things = controller_instance.assign_randomized_conditions(
new_posts)
## assert that only 1 item from each condition went through
assert len(experiment_things) == len(experiment_settings['conditions'])
for thing in experiment_things:
assert thing.id in [x.id for x in new_posts]
## CHECK THE EMPTY CASE
## make sure that no actions are taken if the list is empty
experiment_action_count = db_session.query(ExperimentAction).count()
controller_instance.assign_randomized_conditions([])
assert db_session.query(
ExperimentAction).count() == experiment_action_count
clear_all_tables()
def test_get_eligible_objects(mock_subreddit, mock_reddit):
r = mock_reddit.return_value
experiment_name_to_controller = {
"sticky_comment_0": AMAStickyCommentExperimentController,
"sticky_comment_frontpage_test":
FrontPageStickyCommentExperimentController
}
for experiment_name in experiment_name_to_controller:
with open(
os.path.join(BASE_DIR, "config", "experiments") + "/" +
experiment_name + ".yml", "r") as f:
experiment_settings = yaml.load(f.read())['test']
sub_data = []
with open("{script_dir}/fixture_data/subreddit_posts_0.json".format(
script_dir=TEST_DIR)) as f:
fixture = [
x['data'] for x in json.loads(f.read())['data']['children']
]
for post in fixture:
json_dump = json.dumps(post)
postobj = json2obj(json_dump)
sub_data.append(postobj)
mock_subreddit.get_new.return_value = sub_data
mock_subreddit.display_name = experiment_settings['subreddit']
mock_subreddit.name = experiment_settings['subreddit']
mock_subreddit.id = experiment_settings['subreddit_id']
r.get_subreddit.return_value = mock_subreddit
patch('praw.')
assert (len(db_session.query(Experiment).all()) == 0)
controller = experiment_name_to_controller[experiment_name]
controller_instance = controller(experiment_name, db_session, r, log)
assert (len(db_session.query(Experiment).all()) == 1)
# "mock" FrontPageController.posts
if controller_instance.__class__ is FrontPageStickyCommentExperimentController:
instance = FrontPageController(db_session, r, log)
instance.posts = sub_data
### TEST THE METHOD FOR FETCHING ELIGIBLE OBJECTS
### FIRST TIME AROUND
assert len(db_session.query(Post).all()) == 0
if controller_instance.__class__ is FrontPageStickyCommentExperimentController:
objs = controller_instance.set_eligible_objects(instance)
elif controller_instance.__class__ is AMAStickyCommentExperimentController:
objs = controller_instance.set_eligible_objects()
eligible_objects = controller_instance.get_eligible_objects(objs)
assert len(eligible_objects) == 100
if controller_instance.__class__ is FrontPageStickyCommentExperimentController:
eligible_submissions = {sub.id: sub for sub in eligible_objects}
controller_instance.archive_eligible_submissions(
eligible_submissions)
assert len(db_session.query(Post).all()) == 100
### TEST THE METHOD FOR FETCHING ELIGIBLE OBJECTS
### SECOND TIME AROUND, WITH SOME ExperimentThing objects stored
limit = 50
#for post in db_session.query(Post).all():
for post in eligible_objects:
limit -= 1
experiment_thing = ExperimentThing(
id=post.id,
object_type=int(ThingType.SUBMISSION.value),
experiment_id=controller_instance.experiment.id)
db_session.add(experiment_thing)
if (limit <= 0):
break
db_session.commit()
if controller_instance.__class__ is FrontPageStickyCommentExperimentController:
objs = controller_instance.set_eligible_objects(instance)
elif controller_instance.__class__ is AMAStickyCommentExperimentController:
objs = controller_instance.set_eligible_objects()
eligible_objects = controller_instance.get_eligible_objects(objs)
assert len(eligible_objects) == 50
clear_all_tables()
def test_identify_condition(mock_subreddit, mock_reddit):
r = mock_reddit.return_value
experiment_name_to_controller = {
"sticky_comment_0": AMAStickyCommentExperimentController,
"sticky_comment_frontpage_test":
FrontPageStickyCommentExperimentController #,
#"mod_user_test": ModUserExperimentController
}
sub_data = []
with open("{script_dir}/fixture_data/subreddit_posts_0.json".format(
script_dir=TEST_DIR)) as f:
fixture = [x['data'] for x in json.loads(f.read())['data']['children']]
for post in fixture:
json_dump = json.dumps(post)
postobj = json2obj(json_dump)
sub_data.append(postobj)
mock_subreddit.get_new.return_value = sub_data
############
for experiment_name in experiment_name_to_controller:
with open(
os.path.join(BASE_DIR, "config", "experiments") + "/" +
experiment_name + ".yml", "r") as f:
experiment_settings = yaml.load(f.read())['test']
mock_subreddit.display_name = experiment_settings['subreddit']
mock_subreddit.name = experiment_settings['subreddit']
mock_subreddit.id = experiment_settings['subreddit_id']
r.get_subreddit.return_value = mock_subreddit
patch('praw.')
################################################
## TEST THE BASE CASE OF RANDOMIZATION
controller = experiment_name_to_controller[experiment_name]
controller_instance = controller(experiment_name, db_session, r, log)
# "mock" FrontPageController.posts, ModeratorController.mod_actions
if controller_instance.__class__ is FrontPageStickyCommentExperimentController:
instance = FrontPageController(db_session, r, log)
instance.posts = sub_data
#elif controller_instance.__class__ is ModUserExperimentController:
# modaction_pages = []
# modaction_data = []
# for filename in sorted(glob.glob("{script_dir}/fixture_data/mod_action*".format(script_dir=TEST_DIR))):
# with open(filename, "r") as f:
# mod_action_list = []
# for mod_action in json.loads(f.read()):
# mod_action['sr_id36'] = controller_instance.subreddit_id
# mod_action_list.append(json2obj(json.dumps(mod_action)))
# modaction_pages.append(mod_action_list)
# modaction_data = modaction_pages[0]
# log.info("************** {0}".format(len(modaction_data)))
# #r.get_mod_log.return_value = modaction_data
# instance = ModeratorController(experiment_settings['subreddit'], db_session, r, log)
# instance.mod_actions = modaction_data
if controller_instance.__class__ is FrontPageStickyCommentExperimentController:
objs = controller_instance.set_eligible_objects(instance)
eligible_objects = controller_instance.get_eligible_objects(objs)
elif controller_instance.__class__ is AMAStickyCommentExperimentController:
objs = controller_instance.set_eligible_objects()
eligible_objects = controller_instance.get_eligible_objects(objs)
#elif controller_instance.__class__ is ModUserExperimentController:
# eligible_objects = controller_instance.get_eligible_users_and_archive_mod_actions(instance)
condition_list = []
for obj in eligible_objects:
condition_list.append(controller_instance.identify_condition(obj))
if controller_instance.__class__ is FrontPageStickyCommentExperimentController:
assert Counter(condition_list)['frontpage_post'] == 100
elif controller_instance.__class__ is AMAStickyCommentExperimentController:
assert Counter(condition_list)['nonama'] == 98
assert Counter(condition_list)['ama'] == 2
#elif controller_instance.__class__ is ModUserExperimentController:
# assert Counter(condition_list)['main'] == 100 ###
clear_all_tables()
def test_update_experiment(intervene_ama_arm_1, intervene_ama_arm_0,
intervene_nonama_arm_1, intervene_nonama_arm_0,
intervene_frontpage_post_arm_1, intervene_frontpage_post_arm_0,
mock_subreddit, mock_reddit):
r = mock_reddit.return_value
experiment_name_to_controller = {
"sticky_comment_0": AMAStickyCommentExperimentController,
"sticky_comment_frontpage_test": FrontPageStickyCommentExperimentController
}
for experiment_name in experiment_name_to_controller:
with open(os.path.join(BASE_DIR, "config", "experiments") + "/"+ experiment_name + ".yml", "r") as f:
experiment_settings = yaml.load(f.read())['test']
sub_data = []
with open("{script_dir}/fixture_data/subreddit_posts_0.json".format(script_dir=TEST_DIR)) as f:
fixture = [x['data'] for x in json.loads(f.read())['data']['children']]
for post in fixture:
json_dump = json.dumps(post)
postobj = json2obj(json_dump)
sub_data.append(postobj)
mock_subreddit.get_new.return_value = sub_data
mock_subreddit.display_name = experiment_settings['subreddit']
mock_subreddit.name = experiment_settings['subreddit']
mock_subreddit.id = experiment_settings['subreddit_id']
r.get_subreddit.return_value = mock_subreddit
patch('praw.')
controller = experiment_name_to_controller[experiment_name]
controller_instance = controller(experiment_name, db_session, r, log)
# "mock" FrontPageController.posts
if controller_instance.__class__ is FrontPageStickyCommentExperimentController:
instance = FrontPageController(db_session, r, log)
instance.posts = sub_data
## GET RANDOMIZATIONS
#objs = scec.set_eligible_objects()
#eligible_objects = scec.get_eligible_objects(objs)
#experiment_things = scec.assign_randomized_conditions(eligible_objects)
assert db_session.query(ExperimentThing).count() == 0
## MOCK RETURN VALUES FROM ACTION METHODS
## This will never be saved. Just creating an object
comment_thing = ExperimentThing(
experiment_id = controller_instance.experiment.id,
object_created = None,
object_type = ThingType.COMMENT.value,
id = "12345",
metadata_json = json.dumps({"group":None,"submission_id":None})
)
# for AMAStickyCommentExperimentController
intervene_nonama_arm_0.return_value = comment_thing
intervene_nonama_arm_1.return_value = comment_thing
intervene_ama_arm_0.return_value = comment_thing
intervene_ama_arm_1.return_value = comment_thing
# for FrontPageStickyCommentExperimentController
intervene_frontpage_post_arm_0.return_value = comment_thing
intervene_frontpage_post_arm_1.return_value = comment_thing
if controller_instance.__class__ is FrontPageStickyCommentExperimentController:
experiment_return = controller_instance.callback_update_experiment(instance)
elif controller_instance.__class__ is AMAStickyCommentExperimentController:
experiment_return = controller_instance.update_experiment()
## ASSERT THAT THE METHODS FOR TAKING ACTIONS ARE CALLED
if controller_instance.__class__ is FrontPageStickyCommentExperimentController:
assert intervene_frontpage_post_arm_0.called == True
assert intervene_frontpage_post_arm_1.called == True
elif controller_instance.__class__ is AMAStickyCommentExperimentController:
assert intervene_nonama_arm_0.called == True
assert intervene_nonama_arm_1.called == True
assert intervene_ama_arm_0.called == True
assert intervene_ama_arm_1.called == True
assert len(experiment_return) == 100
clear_all_tables()
def test_assign_randomized_conditions(mock_subreddit, mock_reddit):
r = mock_reddit.return_value
experiment_name_to_controller = {
"sticky_comment_0": AMAStickyCommentExperimentController,
"sticky_comment_frontpage_test": FrontPageStickyCommentExperimentController
}
for experiment_name in experiment_name_to_controller:
with open(os.path.join(BASE_DIR, "config", "experiments") + "/"+ experiment_name + ".yml", "r") as f:
experiment_settings = yaml.load(f.read())['test']
sub_data = []
with open("{script_dir}/fixture_data/subreddit_posts_0.json".format(script_dir=TEST_DIR)) as f:
fixture = [x['data'] for x in json.loads(f.read())['data']['children']]
for post in fixture:
json_dump = json.dumps(post)
postobj = json2obj(json_dump)
sub_data.append(postobj)
mock_subreddit.get_new.return_value = sub_data
mock_subreddit.display_name = experiment_settings['subreddit']
mock_subreddit.name = experiment_settings['subreddit']
mock_subreddit.id = experiment_settings['subreddit_id']
r.get_subreddit.return_value = mock_subreddit
patch('praw.')
controller = experiment_name_to_controller[experiment_name]
controller_instance = controller(experiment_name, db_session, r, log)
# "mock" FrontPageController.posts
if controller_instance.__class__ is FrontPageStickyCommentExperimentController:
instance = FrontPageController(db_session, r, log)
instance.posts = sub_data
## TEST THE BASE CASE OF RANDOMIZATION
if controller_instance.__class__ is FrontPageStickyCommentExperimentController:
objs = controller_instance.set_eligible_objects(instance)
elif controller_instance.__class__ is AMAStickyCommentExperimentController:
objs = controller_instance.set_eligible_objects()
eligible_objects = controller_instance.get_eligible_objects(objs)
assert len(eligible_objects) == 100 ####
experiment_action_count = db_session.query(ExperimentAction).count()
experiment_settings = json.loads(controller_instance.experiment.settings_json)
for condition_name in experiment_settings['conditions']:
assert experiment_settings['conditions'][condition_name]['next_randomization'] == 0
assert db_session.query(ExperimentThing).count() == 0
controller_instance.assign_randomized_conditions(eligible_objects)
assert db_session.query(ExperimentThing).count() == 100
assert len(db_session.query(Experiment).all()) == 1
experiment = db_session.query(Experiment).first()
experiment_settings = json.loads(experiment.settings_json)
if controller_instance.__class__ is AMAStickyCommentExperimentController:
assert experiment_settings['conditions']['ama']['next_randomization'] == 2
assert experiment_settings['conditions']['nonama']['next_randomization'] == 98
assert sum([x['next_randomization'] for x in list(experiment_settings['conditions'].values())]) == 100
elif controller_instance.__class__ is FrontPageStickyCommentExperimentController:
assert experiment_settings['conditions']['frontpage_post']['next_randomization'] == 100
for experiment_thing in db_session.query(ExperimentThing).all():
assert experiment_thing.id != None
assert experiment_thing.object_type == ThingType.SUBMISSION.value
assert experiment_thing.experiment_id == controller_instance.experiment.id
assert "randomization" in json.loads(experiment_thing.metadata_json).keys()
assert "condition" in json.loads(experiment_thing.metadata_json).keys()
## TEST THE CASE WHERE THE AMA EXPERIMENT HAS CONCLUDED
### first step: set the condition counts to have just one remaining condition left
for condition_name in experiment_settings['conditions']:
experiment_settings['conditions'][condition_name]['next_randomization'] = len(experiment_settings['conditions'][condition_name]['randomizations']) - 1
controller_instance.experiment_settings['conditions'][condition_name]['next_randomization'] = experiment_settings['conditions'][condition_name]['next_randomization']
experiment_settings_json = json.dumps(experiment_settings)
db_session.commit()
posts = []
if controller_instance.__class__ is AMAStickyCommentExperimentController:
posts += [x for x in sub_data if "ama" in x.link_flair_css_class][0:2]
posts += [x for x in sub_data if "ama" not in x.link_flair_css_class][0:2]
elif controller_instance.__class__ is FrontPageStickyCommentExperimentController:
posts += sub_data[0:4]
## generate new fake ids for these fixture posts,
## which would otherwise be duplicates
new_posts = []
for post in posts:
post = post.json_dict
post['id'] = ''.join(random.SystemRandom().choice(string.ascii_uppercase + string.digits) for _ in range(7))
new_posts.append(json2obj(json.dumps(post)))
# Only 1 randomization left for each condition, while there are >1 new_posts
assert len(new_posts) == 4
experiment_things = controller_instance.assign_randomized_conditions(new_posts)
## assert that only 1 item from each condition went through
assert len(experiment_things) == len(experiment_settings['conditions'])
for thing in experiment_things:
assert thing.id in [x.id for x in new_posts]
## CHECK THE EMPTY CASE
## make sure that no actions are taken if the list is empty
experiment_action_count = db_session.query(ExperimentAction).count()
controller_instance.assign_randomized_conditions([])
assert db_session.query(ExperimentAction).count() == experiment_action_count
clear_all_tables()
def test_get_eligible_objects(mock_subreddit, mock_reddit):
r = mock_reddit.return_value
experiment_name_to_controller = {
"sticky_comment_0": AMAStickyCommentExperimentController,
"sticky_comment_frontpage_test": FrontPageStickyCommentExperimentController
}
for experiment_name in experiment_name_to_controller:
with open(os.path.join(BASE_DIR, "config", "experiments") + "/"+ experiment_name + ".yml", "r") as f:
experiment_settings = yaml.load(f.read())['test']
sub_data = []
with open("{script_dir}/fixture_data/subreddit_posts_0.json".format(script_dir=TEST_DIR)) as f:
fixture = [x['data'] for x in json.loads(f.read())['data']['children']]
for post in fixture:
json_dump = json.dumps(post)
postobj = json2obj(json_dump)
sub_data.append(postobj)
mock_subreddit.get_new.return_value = sub_data
mock_subreddit.display_name = experiment_settings['subreddit']
mock_subreddit.name = experiment_settings['subreddit']
mock_subreddit.id = experiment_settings['subreddit_id']
r.get_subreddit.return_value = mock_subreddit
patch('praw.')
assert(len(db_session.query(Experiment).all()) == 0)
controller = experiment_name_to_controller[experiment_name]
controller_instance = controller(experiment_name, db_session, r, log)
assert(len(db_session.query(Experiment).all()) == 1)
# "mock" FrontPageController.posts
if controller_instance.__class__ is FrontPageStickyCommentExperimentController:
instance = FrontPageController(db_session, r, log)
instance.posts = sub_data
### TEST THE METHOD FOR FETCHING ELIGIBLE OBJECTS
### FIRST TIME AROUND
assert len(db_session.query(Post).all()) == 0
if controller_instance.__class__ is FrontPageStickyCommentExperimentController:
objs = controller_instance.set_eligible_objects(instance)
elif controller_instance.__class__ is AMAStickyCommentExperimentController:
objs = controller_instance.set_eligible_objects()
eligible_objects = controller_instance.get_eligible_objects(objs)
assert len(eligible_objects) == 100
if controller_instance.__class__ is FrontPageStickyCommentExperimentController:
eligible_submissions = {sub.id: sub for sub in eligible_objects}
controller_instance.archive_eligible_submissions(eligible_submissions)
assert len(db_session.query(Post).all()) == 100
### TEST THE METHOD FOR FETCHING ELIGIBLE OBJECTS
### SECOND TIME AROUND, WITH SOME ExperimentThing objects stored
limit = 50
#for post in db_session.query(Post).all():
for post in eligible_objects:
limit -= 1
experiment_thing = ExperimentThing(
id = post.id,
object_type=int(ThingType.SUBMISSION.value),
experiment_id = controller_instance.experiment.id)
db_session.add(experiment_thing)
if(limit <= 0):
break
db_session.commit()
if controller_instance.__class__ is FrontPageStickyCommentExperimentController:
objs = controller_instance.set_eligible_objects(instance)
elif controller_instance.__class__ is AMAStickyCommentExperimentController:
objs = controller_instance.set_eligible_objects()
eligible_objects = controller_instance.get_eligible_objects(objs)
assert len(eligible_objects) == 50
clear_all_tables()