def test_identify_condition(mock_subreddit, mock_reddit): r = mock_reddit.return_value experiment_name_to_controller = { "sticky_comment_0": AMAStickyCommentExperimentController, "sticky_comment_frontpage_test": FrontPageStickyCommentExperimentController } for experiment_name in experiment_name_to_controller: with open( os.path.join(BASE_DIR, "config", "experiments") + "/" + experiment_name + ".yml", "r") as f: experiment_settings = yaml.load(f.read())['test'] sub_data = [] with open("{script_dir}/fixture_data/subreddit_posts_0.json".format( script_dir=TEST_DIR)) as f: fixture = [ x['data'] for x in json.loads(f.read())['data']['children'] ] for post in fixture: json_dump = json.dumps(post) postobj = json2obj(json_dump) sub_data.append(postobj) mock_subreddit.get_new.return_value = sub_data mock_subreddit.display_name = experiment_settings['subreddit'] mock_subreddit.name = experiment_settings['subreddit'] mock_subreddit.id = experiment_settings['subreddit_id'] r.get_subreddit.return_value = mock_subreddit patch('praw.') ## TEST THE BASE CASE OF RANDOMIZATION controller = experiment_name_to_controller[experiment_name] controller_instance = controller(experiment_name, db_session, r, log) # "mock" FrontPageController.posts if controller_instance.__class__ is FrontPageStickyCommentExperimentController: instance = FrontPageController(db_session, r, log) instance.posts = sub_data if controller_instance.__class__ is FrontPageStickyCommentExperimentController: objs = controller_instance.set_eligible_objects(instance) elif controller_instance.__class__ is AMAStickyCommentExperimentController: objs = controller_instance.set_eligible_objects() eligible_objects = controller_instance.get_eligible_objects(objs) condition_list = [] for obj in eligible_objects: condition_list.append(controller_instance.identify_condition(obj)) if controller_instance.__class__ is FrontPageStickyCommentExperimentController: assert Counter(condition_list)['frontpage_post'] == 100 elif controller_instance.__class__ is AMAStickyCommentExperimentController: assert Counter(condition_list)['nonama'] == 98 assert Counter(condition_list)['ama'] == 2 clear_all_tables()
def test_identify_condition(mock_subreddit, mock_reddit): r = mock_reddit.return_value experiment_name_to_controller = { "sticky_comment_0": AMAStickyCommentExperimentController, "sticky_comment_frontpage_test": FrontPageStickyCommentExperimentController } for experiment_name in experiment_name_to_controller: with open(os.path.join(BASE_DIR, "config", "experiments") + "/"+ experiment_name + ".yml", "r") as f: experiment_settings = yaml.load(f.read())['test'] sub_data = [] with open("{script_dir}/fixture_data/subreddit_posts_0.json".format(script_dir=TEST_DIR)) as f: fixture = [x['data'] for x in json.loads(f.read())['data']['children']] for post in fixture: json_dump = json.dumps(post) postobj = json2obj(json_dump) sub_data.append(postobj) mock_subreddit.get_new.return_value = sub_data mock_subreddit.display_name = experiment_settings['subreddit'] mock_subreddit.name = experiment_settings['subreddit'] mock_subreddit.id = experiment_settings['subreddit_id'] r.get_subreddit.return_value = mock_subreddit patch('praw.') ## TEST THE BASE CASE OF RANDOMIZATION controller = experiment_name_to_controller[experiment_name] controller_instance = controller(experiment_name, db_session, r, log) # "mock" FrontPageController.posts if controller_instance.__class__ is FrontPageStickyCommentExperimentController: instance = FrontPageController(db_session, r, log) instance.posts = sub_data if controller_instance.__class__ is FrontPageStickyCommentExperimentController: objs = controller_instance.set_eligible_objects(instance) elif controller_instance.__class__ is AMAStickyCommentExperimentController: objs = controller_instance.set_eligible_objects() eligible_objects = controller_instance.get_eligible_objects(objs) condition_list = [] for obj in eligible_objects: condition_list.append(controller_instance.identify_condition(obj)) if controller_instance.__class__ is FrontPageStickyCommentExperimentController: assert Counter(condition_list)['frontpage_post'] == 100 elif controller_instance.__class__ is AMAStickyCommentExperimentController: assert Counter(condition_list)['nonama'] == 98 assert Counter(condition_list)['ama'] == 2 clear_all_tables()
def test_update_experiment(intervene_ama_arm_1, intervene_ama_arm_0, intervene_nonama_arm_1, intervene_nonama_arm_0, intervene_frontpage_post_arm_1, intervene_frontpage_post_arm_0, mock_subreddit, mock_reddit): r = mock_reddit.return_value experiment_name_to_controller = { "sticky_comment_0": AMAStickyCommentExperimentController, "sticky_comment_frontpage_test": FrontPageStickyCommentExperimentController } for experiment_name in experiment_name_to_controller: with open( os.path.join(BASE_DIR, "config", "experiments") + "/" + experiment_name + ".yml", "r") as f: experiment_settings = yaml.load(f.read())['test'] sub_data = [] with open("{script_dir}/fixture_data/subreddit_posts_0.json".format( script_dir=TEST_DIR)) as f: fixture = [ x['data'] for x in json.loads(f.read())['data']['children'] ] for post in fixture: json_dump = json.dumps(post) postobj = json2obj(json_dump) sub_data.append(postobj) mock_subreddit.get_new.return_value = sub_data mock_subreddit.display_name = experiment_settings['subreddit'] mock_subreddit.name = experiment_settings['subreddit'] mock_subreddit.id = experiment_settings['subreddit_id'] r.get_subreddit.return_value = mock_subreddit patch('praw.') controller = experiment_name_to_controller[experiment_name] controller_instance = controller(experiment_name, db_session, r, log) # "mock" FrontPageController.posts if controller_instance.__class__ is FrontPageStickyCommentExperimentController: instance = FrontPageController(db_session, r, log) instance.posts = sub_data ## GET RANDOMIZATIONS #objs = scec.set_eligible_objects() #eligible_objects = scec.get_eligible_objects(objs) #experiment_things = scec.assign_randomized_conditions(eligible_objects) assert db_session.query(ExperimentThing).count() == 0 ## MOCK RETURN VALUES FROM ACTION METHODS ## This will never be saved. Just creating an object comment_thing = ExperimentThing( experiment_id=controller_instance.experiment.id, object_created=None, object_type=ThingType.COMMENT.value, id="12345", metadata_json=json.dumps({ "group": None, "submission_id": None })) # for AMAStickyCommentExperimentController intervene_nonama_arm_0.return_value = comment_thing intervene_nonama_arm_1.return_value = comment_thing intervene_ama_arm_0.return_value = comment_thing intervene_ama_arm_1.return_value = comment_thing # for FrontPageStickyCommentExperimentController intervene_frontpage_post_arm_0.return_value = comment_thing intervene_frontpage_post_arm_1.return_value = comment_thing if controller_instance.__class__ is FrontPageStickyCommentExperimentController: experiment_return = controller_instance.callback_update_experiment( instance) elif controller_instance.__class__ is AMAStickyCommentExperimentController: experiment_return = controller_instance.update_experiment() ## ASSERT THAT THE METHODS FOR TAKING ACTIONS ARE CALLED if controller_instance.__class__ is FrontPageStickyCommentExperimentController: assert intervene_frontpage_post_arm_0.called == True assert intervene_frontpage_post_arm_1.called == True elif controller_instance.__class__ is AMAStickyCommentExperimentController: assert intervene_nonama_arm_0.called == True assert intervene_nonama_arm_1.called == True assert intervene_ama_arm_0.called == True assert intervene_ama_arm_1.called == True assert len(experiment_return) == 100 clear_all_tables()
def test_assign_randomized_conditions(mock_subreddit, mock_reddit): r = mock_reddit.return_value experiment_name_to_controller = { "sticky_comment_0": AMAStickyCommentExperimentController, "sticky_comment_frontpage_test": FrontPageStickyCommentExperimentController } for experiment_name in experiment_name_to_controller: with open( os.path.join(BASE_DIR, "config", "experiments") + "/" + experiment_name + ".yml", "r") as f: experiment_settings = yaml.load(f.read())['test'] sub_data = [] with open("{script_dir}/fixture_data/subreddit_posts_0.json".format( script_dir=TEST_DIR)) as f: fixture = [ x['data'] for x in json.loads(f.read())['data']['children'] ] for post in fixture: json_dump = json.dumps(post) postobj = json2obj(json_dump) sub_data.append(postobj) mock_subreddit.get_new.return_value = sub_data mock_subreddit.display_name = experiment_settings['subreddit'] mock_subreddit.name = experiment_settings['subreddit'] mock_subreddit.id = experiment_settings['subreddit_id'] r.get_subreddit.return_value = mock_subreddit patch('praw.') controller = experiment_name_to_controller[experiment_name] controller_instance = controller(experiment_name, db_session, r, log) # "mock" FrontPageController.posts if controller_instance.__class__ is FrontPageStickyCommentExperimentController: instance = FrontPageController(db_session, r, log) instance.posts = sub_data ## TEST THE BASE CASE OF RANDOMIZATION if controller_instance.__class__ is FrontPageStickyCommentExperimentController: objs = controller_instance.set_eligible_objects(instance) elif controller_instance.__class__ is AMAStickyCommentExperimentController: objs = controller_instance.set_eligible_objects() eligible_objects = controller_instance.get_eligible_objects(objs) assert len(eligible_objects) == 100 #### experiment_action_count = db_session.query(ExperimentAction).count() experiment_settings = json.loads( controller_instance.experiment.settings_json) for condition_name in experiment_settings['conditions']: assert experiment_settings['conditions'][condition_name][ 'next_randomization'] == 0 assert db_session.query(ExperimentThing).count() == 0 controller_instance.assign_randomized_conditions(eligible_objects) assert db_session.query(ExperimentThing).count() == 100 assert len(db_session.query(Experiment).all()) == 1 experiment = db_session.query(Experiment).first() experiment_settings = json.loads(experiment.settings_json) if controller_instance.__class__ is AMAStickyCommentExperimentController: assert experiment_settings['conditions']['ama'][ 'next_randomization'] == 2 assert experiment_settings['conditions']['nonama'][ 'next_randomization'] == 98 assert sum([ x['next_randomization'] for x in list(experiment_settings['conditions'].values()) ]) == 100 elif controller_instance.__class__ is FrontPageStickyCommentExperimentController: assert experiment_settings['conditions']['frontpage_post'][ 'next_randomization'] == 100 for experiment_thing in db_session.query(ExperimentThing).all(): assert experiment_thing.id != None assert experiment_thing.object_type == ThingType.SUBMISSION.value assert experiment_thing.experiment_id == controller_instance.experiment.id assert "randomization" in json.loads( experiment_thing.metadata_json).keys() assert "condition" in json.loads( experiment_thing.metadata_json).keys() ## TEST THE CASE WHERE THE AMA EXPERIMENT HAS CONCLUDED ### first step: set the condition counts to have just one remaining condition left for condition_name in experiment_settings['conditions']: experiment_settings['conditions'][condition_name][ 'next_randomization'] = len( experiment_settings['conditions'][condition_name] ['randomizations']) - 1 controller_instance.experiment_settings['conditions'][ condition_name]['next_randomization'] = experiment_settings[ 'conditions'][condition_name]['next_randomization'] experiment_settings_json = json.dumps(experiment_settings) db_session.commit() posts = [] if controller_instance.__class__ is AMAStickyCommentExperimentController: posts += [x for x in sub_data if "ama" in x.link_flair_css_class][0:2] posts += [ x for x in sub_data if "ama" not in x.link_flair_css_class ][0:2] elif controller_instance.__class__ is FrontPageStickyCommentExperimentController: posts += sub_data[0:4] ## generate new fake ids for these fixture posts, ## which would otherwise be duplicates new_posts = [] for post in posts: post = post.json_dict post['id'] = ''.join( random.SystemRandom().choice(string.ascii_uppercase + string.digits) for _ in range(7)) new_posts.append(json2obj(json.dumps(post))) # Only 1 randomization left for each condition, while there are >1 new_posts assert len(new_posts) == 4 experiment_things = controller_instance.assign_randomized_conditions( new_posts) ## assert that only 1 item from each condition went through assert len(experiment_things) == len(experiment_settings['conditions']) for thing in experiment_things: assert thing.id in [x.id for x in new_posts] ## CHECK THE EMPTY CASE ## make sure that no actions are taken if the list is empty experiment_action_count = db_session.query(ExperimentAction).count() controller_instance.assign_randomized_conditions([]) assert db_session.query( ExperimentAction).count() == experiment_action_count clear_all_tables()
def test_get_eligible_objects(mock_subreddit, mock_reddit): r = mock_reddit.return_value experiment_name_to_controller = { "sticky_comment_0": AMAStickyCommentExperimentController, "sticky_comment_frontpage_test": FrontPageStickyCommentExperimentController } for experiment_name in experiment_name_to_controller: with open( os.path.join(BASE_DIR, "config", "experiments") + "/" + experiment_name + ".yml", "r") as f: experiment_settings = yaml.load(f.read())['test'] sub_data = [] with open("{script_dir}/fixture_data/subreddit_posts_0.json".format( script_dir=TEST_DIR)) as f: fixture = [ x['data'] for x in json.loads(f.read())['data']['children'] ] for post in fixture: json_dump = json.dumps(post) postobj = json2obj(json_dump) sub_data.append(postobj) mock_subreddit.get_new.return_value = sub_data mock_subreddit.display_name = experiment_settings['subreddit'] mock_subreddit.name = experiment_settings['subreddit'] mock_subreddit.id = experiment_settings['subreddit_id'] r.get_subreddit.return_value = mock_subreddit patch('praw.') assert (len(db_session.query(Experiment).all()) == 0) controller = experiment_name_to_controller[experiment_name] controller_instance = controller(experiment_name, db_session, r, log) assert (len(db_session.query(Experiment).all()) == 1) # "mock" FrontPageController.posts if controller_instance.__class__ is FrontPageStickyCommentExperimentController: instance = FrontPageController(db_session, r, log) instance.posts = sub_data ### TEST THE METHOD FOR FETCHING ELIGIBLE OBJECTS ### FIRST TIME AROUND assert len(db_session.query(Post).all()) == 0 if controller_instance.__class__ is FrontPageStickyCommentExperimentController: objs = controller_instance.set_eligible_objects(instance) elif controller_instance.__class__ is AMAStickyCommentExperimentController: objs = controller_instance.set_eligible_objects() eligible_objects = controller_instance.get_eligible_objects(objs) assert len(eligible_objects) == 100 if controller_instance.__class__ is FrontPageStickyCommentExperimentController: eligible_submissions = {sub.id: sub for sub in eligible_objects} controller_instance.archive_eligible_submissions( eligible_submissions) assert len(db_session.query(Post).all()) == 100 ### TEST THE METHOD FOR FETCHING ELIGIBLE OBJECTS ### SECOND TIME AROUND, WITH SOME ExperimentThing objects stored limit = 50 #for post in db_session.query(Post).all(): for post in eligible_objects: limit -= 1 experiment_thing = ExperimentThing( id=post.id, object_type=int(ThingType.SUBMISSION.value), experiment_id=controller_instance.experiment.id) db_session.add(experiment_thing) if (limit <= 0): break db_session.commit() if controller_instance.__class__ is FrontPageStickyCommentExperimentController: objs = controller_instance.set_eligible_objects(instance) elif controller_instance.__class__ is AMAStickyCommentExperimentController: objs = controller_instance.set_eligible_objects() eligible_objects = controller_instance.get_eligible_objects(objs) assert len(eligible_objects) == 50 clear_all_tables()
def test_identify_condition(mock_subreddit, mock_reddit): r = mock_reddit.return_value experiment_name_to_controller = { "sticky_comment_0": AMAStickyCommentExperimentController, "sticky_comment_frontpage_test": FrontPageStickyCommentExperimentController #, #"mod_user_test": ModUserExperimentController } sub_data = [] with open("{script_dir}/fixture_data/subreddit_posts_0.json".format( script_dir=TEST_DIR)) as f: fixture = [x['data'] for x in json.loads(f.read())['data']['children']] for post in fixture: json_dump = json.dumps(post) postobj = json2obj(json_dump) sub_data.append(postobj) mock_subreddit.get_new.return_value = sub_data ############ for experiment_name in experiment_name_to_controller: with open( os.path.join(BASE_DIR, "config", "experiments") + "/" + experiment_name + ".yml", "r") as f: experiment_settings = yaml.load(f.read())['test'] mock_subreddit.display_name = experiment_settings['subreddit'] mock_subreddit.name = experiment_settings['subreddit'] mock_subreddit.id = experiment_settings['subreddit_id'] r.get_subreddit.return_value = mock_subreddit patch('praw.') ################################################ ## TEST THE BASE CASE OF RANDOMIZATION controller = experiment_name_to_controller[experiment_name] controller_instance = controller(experiment_name, db_session, r, log) # "mock" FrontPageController.posts, ModeratorController.mod_actions if controller_instance.__class__ is FrontPageStickyCommentExperimentController: instance = FrontPageController(db_session, r, log) instance.posts = sub_data #elif controller_instance.__class__ is ModUserExperimentController: # modaction_pages = [] # modaction_data = [] # for filename in sorted(glob.glob("{script_dir}/fixture_data/mod_action*".format(script_dir=TEST_DIR))): # with open(filename, "r") as f: # mod_action_list = [] # for mod_action in json.loads(f.read()): # mod_action['sr_id36'] = controller_instance.subreddit_id # mod_action_list.append(json2obj(json.dumps(mod_action))) # modaction_pages.append(mod_action_list) # modaction_data = modaction_pages[0] # log.info("************** {0}".format(len(modaction_data))) # #r.get_mod_log.return_value = modaction_data # instance = ModeratorController(experiment_settings['subreddit'], db_session, r, log) # instance.mod_actions = modaction_data if controller_instance.__class__ is FrontPageStickyCommentExperimentController: objs = controller_instance.set_eligible_objects(instance) eligible_objects = controller_instance.get_eligible_objects(objs) elif controller_instance.__class__ is AMAStickyCommentExperimentController: objs = controller_instance.set_eligible_objects() eligible_objects = controller_instance.get_eligible_objects(objs) #elif controller_instance.__class__ is ModUserExperimentController: # eligible_objects = controller_instance.get_eligible_users_and_archive_mod_actions(instance) condition_list = [] for obj in eligible_objects: condition_list.append(controller_instance.identify_condition(obj)) if controller_instance.__class__ is FrontPageStickyCommentExperimentController: assert Counter(condition_list)['frontpage_post'] == 100 elif controller_instance.__class__ is AMAStickyCommentExperimentController: assert Counter(condition_list)['nonama'] == 98 assert Counter(condition_list)['ama'] == 2 #elif controller_instance.__class__ is ModUserExperimentController: # assert Counter(condition_list)['main'] == 100 ### clear_all_tables()
def test_update_experiment(intervene_ama_arm_1, intervene_ama_arm_0, intervene_nonama_arm_1, intervene_nonama_arm_0, intervene_frontpage_post_arm_1, intervene_frontpage_post_arm_0, mock_subreddit, mock_reddit): r = mock_reddit.return_value experiment_name_to_controller = { "sticky_comment_0": AMAStickyCommentExperimentController, "sticky_comment_frontpage_test": FrontPageStickyCommentExperimentController } for experiment_name in experiment_name_to_controller: with open(os.path.join(BASE_DIR, "config", "experiments") + "/"+ experiment_name + ".yml", "r") as f: experiment_settings = yaml.load(f.read())['test'] sub_data = [] with open("{script_dir}/fixture_data/subreddit_posts_0.json".format(script_dir=TEST_DIR)) as f: fixture = [x['data'] for x in json.loads(f.read())['data']['children']] for post in fixture: json_dump = json.dumps(post) postobj = json2obj(json_dump) sub_data.append(postobj) mock_subreddit.get_new.return_value = sub_data mock_subreddit.display_name = experiment_settings['subreddit'] mock_subreddit.name = experiment_settings['subreddit'] mock_subreddit.id = experiment_settings['subreddit_id'] r.get_subreddit.return_value = mock_subreddit patch('praw.') controller = experiment_name_to_controller[experiment_name] controller_instance = controller(experiment_name, db_session, r, log) # "mock" FrontPageController.posts if controller_instance.__class__ is FrontPageStickyCommentExperimentController: instance = FrontPageController(db_session, r, log) instance.posts = sub_data ## GET RANDOMIZATIONS #objs = scec.set_eligible_objects() #eligible_objects = scec.get_eligible_objects(objs) #experiment_things = scec.assign_randomized_conditions(eligible_objects) assert db_session.query(ExperimentThing).count() == 0 ## MOCK RETURN VALUES FROM ACTION METHODS ## This will never be saved. Just creating an object comment_thing = ExperimentThing( experiment_id = controller_instance.experiment.id, object_created = None, object_type = ThingType.COMMENT.value, id = "12345", metadata_json = json.dumps({"group":None,"submission_id":None}) ) # for AMAStickyCommentExperimentController intervene_nonama_arm_0.return_value = comment_thing intervene_nonama_arm_1.return_value = comment_thing intervene_ama_arm_0.return_value = comment_thing intervene_ama_arm_1.return_value = comment_thing # for FrontPageStickyCommentExperimentController intervene_frontpage_post_arm_0.return_value = comment_thing intervene_frontpage_post_arm_1.return_value = comment_thing if controller_instance.__class__ is FrontPageStickyCommentExperimentController: experiment_return = controller_instance.callback_update_experiment(instance) elif controller_instance.__class__ is AMAStickyCommentExperimentController: experiment_return = controller_instance.update_experiment() ## ASSERT THAT THE METHODS FOR TAKING ACTIONS ARE CALLED if controller_instance.__class__ is FrontPageStickyCommentExperimentController: assert intervene_frontpage_post_arm_0.called == True assert intervene_frontpage_post_arm_1.called == True elif controller_instance.__class__ is AMAStickyCommentExperimentController: assert intervene_nonama_arm_0.called == True assert intervene_nonama_arm_1.called == True assert intervene_ama_arm_0.called == True assert intervene_ama_arm_1.called == True assert len(experiment_return) == 100 clear_all_tables()
def test_assign_randomized_conditions(mock_subreddit, mock_reddit): r = mock_reddit.return_value experiment_name_to_controller = { "sticky_comment_0": AMAStickyCommentExperimentController, "sticky_comment_frontpage_test": FrontPageStickyCommentExperimentController } for experiment_name in experiment_name_to_controller: with open(os.path.join(BASE_DIR, "config", "experiments") + "/"+ experiment_name + ".yml", "r") as f: experiment_settings = yaml.load(f.read())['test'] sub_data = [] with open("{script_dir}/fixture_data/subreddit_posts_0.json".format(script_dir=TEST_DIR)) as f: fixture = [x['data'] for x in json.loads(f.read())['data']['children']] for post in fixture: json_dump = json.dumps(post) postobj = json2obj(json_dump) sub_data.append(postobj) mock_subreddit.get_new.return_value = sub_data mock_subreddit.display_name = experiment_settings['subreddit'] mock_subreddit.name = experiment_settings['subreddit'] mock_subreddit.id = experiment_settings['subreddit_id'] r.get_subreddit.return_value = mock_subreddit patch('praw.') controller = experiment_name_to_controller[experiment_name] controller_instance = controller(experiment_name, db_session, r, log) # "mock" FrontPageController.posts if controller_instance.__class__ is FrontPageStickyCommentExperimentController: instance = FrontPageController(db_session, r, log) instance.posts = sub_data ## TEST THE BASE CASE OF RANDOMIZATION if controller_instance.__class__ is FrontPageStickyCommentExperimentController: objs = controller_instance.set_eligible_objects(instance) elif controller_instance.__class__ is AMAStickyCommentExperimentController: objs = controller_instance.set_eligible_objects() eligible_objects = controller_instance.get_eligible_objects(objs) assert len(eligible_objects) == 100 #### experiment_action_count = db_session.query(ExperimentAction).count() experiment_settings = json.loads(controller_instance.experiment.settings_json) for condition_name in experiment_settings['conditions']: assert experiment_settings['conditions'][condition_name]['next_randomization'] == 0 assert db_session.query(ExperimentThing).count() == 0 controller_instance.assign_randomized_conditions(eligible_objects) assert db_session.query(ExperimentThing).count() == 100 assert len(db_session.query(Experiment).all()) == 1 experiment = db_session.query(Experiment).first() experiment_settings = json.loads(experiment.settings_json) if controller_instance.__class__ is AMAStickyCommentExperimentController: assert experiment_settings['conditions']['ama']['next_randomization'] == 2 assert experiment_settings['conditions']['nonama']['next_randomization'] == 98 assert sum([x['next_randomization'] for x in list(experiment_settings['conditions'].values())]) == 100 elif controller_instance.__class__ is FrontPageStickyCommentExperimentController: assert experiment_settings['conditions']['frontpage_post']['next_randomization'] == 100 for experiment_thing in db_session.query(ExperimentThing).all(): assert experiment_thing.id != None assert experiment_thing.object_type == ThingType.SUBMISSION.value assert experiment_thing.experiment_id == controller_instance.experiment.id assert "randomization" in json.loads(experiment_thing.metadata_json).keys() assert "condition" in json.loads(experiment_thing.metadata_json).keys() ## TEST THE CASE WHERE THE AMA EXPERIMENT HAS CONCLUDED ### first step: set the condition counts to have just one remaining condition left for condition_name in experiment_settings['conditions']: experiment_settings['conditions'][condition_name]['next_randomization'] = len(experiment_settings['conditions'][condition_name]['randomizations']) - 1 controller_instance.experiment_settings['conditions'][condition_name]['next_randomization'] = experiment_settings['conditions'][condition_name]['next_randomization'] experiment_settings_json = json.dumps(experiment_settings) db_session.commit() posts = [] if controller_instance.__class__ is AMAStickyCommentExperimentController: posts += [x for x in sub_data if "ama" in x.link_flair_css_class][0:2] posts += [x for x in sub_data if "ama" not in x.link_flair_css_class][0:2] elif controller_instance.__class__ is FrontPageStickyCommentExperimentController: posts += sub_data[0:4] ## generate new fake ids for these fixture posts, ## which would otherwise be duplicates new_posts = [] for post in posts: post = post.json_dict post['id'] = ''.join(random.SystemRandom().choice(string.ascii_uppercase + string.digits) for _ in range(7)) new_posts.append(json2obj(json.dumps(post))) # Only 1 randomization left for each condition, while there are >1 new_posts assert len(new_posts) == 4 experiment_things = controller_instance.assign_randomized_conditions(new_posts) ## assert that only 1 item from each condition went through assert len(experiment_things) == len(experiment_settings['conditions']) for thing in experiment_things: assert thing.id in [x.id for x in new_posts] ## CHECK THE EMPTY CASE ## make sure that no actions are taken if the list is empty experiment_action_count = db_session.query(ExperimentAction).count() controller_instance.assign_randomized_conditions([]) assert db_session.query(ExperimentAction).count() == experiment_action_count clear_all_tables()
def test_get_eligible_objects(mock_subreddit, mock_reddit): r = mock_reddit.return_value experiment_name_to_controller = { "sticky_comment_0": AMAStickyCommentExperimentController, "sticky_comment_frontpage_test": FrontPageStickyCommentExperimentController } for experiment_name in experiment_name_to_controller: with open(os.path.join(BASE_DIR, "config", "experiments") + "/"+ experiment_name + ".yml", "r") as f: experiment_settings = yaml.load(f.read())['test'] sub_data = [] with open("{script_dir}/fixture_data/subreddit_posts_0.json".format(script_dir=TEST_DIR)) as f: fixture = [x['data'] for x in json.loads(f.read())['data']['children']] for post in fixture: json_dump = json.dumps(post) postobj = json2obj(json_dump) sub_data.append(postobj) mock_subreddit.get_new.return_value = sub_data mock_subreddit.display_name = experiment_settings['subreddit'] mock_subreddit.name = experiment_settings['subreddit'] mock_subreddit.id = experiment_settings['subreddit_id'] r.get_subreddit.return_value = mock_subreddit patch('praw.') assert(len(db_session.query(Experiment).all()) == 0) controller = experiment_name_to_controller[experiment_name] controller_instance = controller(experiment_name, db_session, r, log) assert(len(db_session.query(Experiment).all()) == 1) # "mock" FrontPageController.posts if controller_instance.__class__ is FrontPageStickyCommentExperimentController: instance = FrontPageController(db_session, r, log) instance.posts = sub_data ### TEST THE METHOD FOR FETCHING ELIGIBLE OBJECTS ### FIRST TIME AROUND assert len(db_session.query(Post).all()) == 0 if controller_instance.__class__ is FrontPageStickyCommentExperimentController: objs = controller_instance.set_eligible_objects(instance) elif controller_instance.__class__ is AMAStickyCommentExperimentController: objs = controller_instance.set_eligible_objects() eligible_objects = controller_instance.get_eligible_objects(objs) assert len(eligible_objects) == 100 if controller_instance.__class__ is FrontPageStickyCommentExperimentController: eligible_submissions = {sub.id: sub for sub in eligible_objects} controller_instance.archive_eligible_submissions(eligible_submissions) assert len(db_session.query(Post).all()) == 100 ### TEST THE METHOD FOR FETCHING ELIGIBLE OBJECTS ### SECOND TIME AROUND, WITH SOME ExperimentThing objects stored limit = 50 #for post in db_session.query(Post).all(): for post in eligible_objects: limit -= 1 experiment_thing = ExperimentThing( id = post.id, object_type=int(ThingType.SUBMISSION.value), experiment_id = controller_instance.experiment.id) db_session.add(experiment_thing) if(limit <= 0): break db_session.commit() if controller_instance.__class__ is FrontPageStickyCommentExperimentController: objs = controller_instance.set_eligible_objects(instance) elif controller_instance.__class__ is AMAStickyCommentExperimentController: objs = controller_instance.set_eligible_objects() eligible_objects = controller_instance.get_eligible_objects(objs) assert len(eligible_objects) == 50 clear_all_tables()