class BotTestCase(TestCase): def setUp(self): self.experiment = Experiment(name="backgroundcolor", state=ENABLED_STATE) self.experiment.save() self.request = request_factory.get("/", HTTP_USER_AGENT="GoogleBot/2.1") self.experiment_counter = ExperimentCounter() def test_user_does_not_enroll(self): experiment_user = participant(self.request) experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) self.assertEqual( self.experiment_counter.participant_count(self.experiment, TEST_ALTERNATIVE), 0, "Bot counted towards results", ) def test_bot_in_control_group(self): experiment_user = participant(self.request) experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME), "control", "Bot enrolled in a group") self.assertEqual( experiment_user.is_enrolled(self.experiment.name, TEST_ALTERNATIVE, self.request), False, "Bot in test alternative", ) self.assertEqual( experiment_user.is_enrolled(self.experiment.name, CONTROL_GROUP, self.request), True, "Bot not in control group", ) def tearDown(self): self.experiment_counter.delete(self.experiment)
class BotTestCase(TestCase): def setUp(self): self.experiment = Experiment(name='backgroundcolor', state=ENABLED_STATE) self.experiment.save() self.request = request_factory.get('/', HTTP_USER_AGENT='GoogleBot/2.1') def test_user_does_not_enroll(self): experiment_user = create_user(self.request) experiment_user.set_enrollment(self.experiment, TEST_ALTERNATIVE) self.assertEqual(self.experiment.participant_count(TEST_ALTERNATIVE), 0, "Bot counted towards results") def test_bot_in_control_group(self): experiment_user = create_user(self.request) experiment_user.set_enrollment(self.experiment, TEST_ALTERNATIVE) self.assertEqual(experiment_user.get_enrollment(self.experiment), CONTROL_GROUP, "Bot alternative is not control") self.assertEqual( experiment_user.is_enrolled(self.experiment.name, TEST_ALTERNATIVE, self.request), False, "Bot in test alternative") self.assertEqual( experiment_user.is_enrolled(self.experiment.name, CONTROL_GROUP, self.request), True, "Bot not in control group") def tearDown(self): self.experiment.delete()
def test_experiment_is_related_to_owner(self): owner = User.objects.first() experiment = Experiment(nes_id=17, owner=owner, version=1, sent_date=datetime.utcnow()) experiment.save() self.assertIn(experiment, owner.experiment_set.all())
class WebUserTests: def setUp(self): self.experiment = Experiment(name='backgroundcolor', state=ENABLED_STATE) self.experiment.save() self.request = request_factory.get('/') self.request.session = DatabaseSession() def tearDown(self): self.experiment.delete() def confirm_human(self, experiment_user): pass def participants(self, alternative): return self.experiment.participant_count(alternative) def enrollment_initially_none(self, ): experiment_user = create_user(self.request) self.assertEqual(experiment_user.get_enrollment(self.experiment), None) def test_user_enrolls(self): experiment_user = create_user(self.request) experiment_user.set_enrollment(self.experiment, TEST_ALTERNATIVE) self.assertEqual(experiment_user.get_enrollment(self.experiment), TEST_ALTERNATIVE) def test_record_goal_increments_counts(self): experiment_user = create_user(self.request) self.confirm_human(experiment_user) experiment_user.set_enrollment(self.experiment, TEST_ALTERNATIVE) self.assertEqual( self.experiment.goal_count(TEST_ALTERNATIVE, TEST_GOAL), 0) experiment_user.record_goal(TEST_GOAL) self.assertEqual( self.experiment.goal_count(TEST_ALTERNATIVE, TEST_GOAL), 1) def test_can_record_goal_multiple_times(self): experiment_user = create_user(self.request) self.confirm_human(experiment_user) experiment_user.set_enrollment(self.experiment, TEST_ALTERNATIVE) experiment_user.record_goal(TEST_GOAL) experiment_user.record_goal(TEST_GOAL) experiment_user.record_goal(TEST_GOAL) self.assertEqual( self.experiment.goal_count(TEST_ALTERNATIVE, TEST_GOAL), 1) def test_counts_increment_immediately_once_confirmed_human(self): experiment_user = create_user(self.request) self.confirm_human(experiment_user) experiment_user.set_enrollment(self.experiment, TEST_ALTERNATIVE) self.assertEqual(self.participants(TEST_ALTERNATIVE), 1, "Did not count participant after confirm human")
def approve_experiment(experiment: Experiment, request) -> None: """Approves an experiment, and informs users of the approval :param experiment: :class:`Experiment` the experiment to approve :param request: Django request. Used to create an absolute URL in the mail """ experiment.approved = True experiment.save() sent_confirmation_email(experiment, request)
class WebUserTests: def setUp(self): self.experiment = Experiment(name='backgroundcolor', state=ENABLED_STATE) self.experiment.save() self.request = request_factory.get('/') self.request.session = DatabaseSession() def tearDown(self): self.experiment.delete() def confirm_human(self, experiment_user): pass def participants(self, alternative): return self.experiment.participant_count(alternative) def enrollment_initially_none(self,): experiment_user = create_user(self.request) self.assertEqual(experiment_user.get_enrollment(self.experiment), None) def test_user_enrolls(self): experiment_user = create_user(self.request) experiment_user.set_enrollment(self.experiment, TEST_ALTERNATIVE) self.assertEqual(experiment_user.get_enrollment(self.experiment), TEST_ALTERNATIVE) def test_record_goal_increments_counts(self): experiment_user = create_user(self.request) self.confirm_human(experiment_user) experiment_user.set_enrollment(self.experiment, TEST_ALTERNATIVE) self.assertEqual(self.experiment.goal_count(TEST_ALTERNATIVE, TEST_GOAL), 0) experiment_user.record_goal(TEST_GOAL) self.assertEqual(self.experiment.goal_count(TEST_ALTERNATIVE, TEST_GOAL), 1) def test_can_record_goal_multiple_times(self): experiment_user = create_user(self.request) self.confirm_human(experiment_user) experiment_user.set_enrollment(self.experiment, TEST_ALTERNATIVE) experiment_user.record_goal(TEST_GOAL) experiment_user.record_goal(TEST_GOAL) experiment_user.record_goal(TEST_GOAL) self.assertEqual(self.experiment.goal_count(TEST_ALTERNATIVE, TEST_GOAL), 1) def test_counts_increment_immediately_once_confirmed_human(self): experiment_user = create_user(self.request) self.confirm_human(experiment_user) experiment_user.set_enrollment(self.experiment, TEST_ALTERNATIVE) self.assertEqual(self.participants(TEST_ALTERNATIVE), 1, "Did not count participant after confirm human")
def test_cannot_save_empty_attributes(self): owner = User.objects.first() # version=17: large number to avoid conflicts with global setup experiment = Experiment(nes_id=1, title='', description='', owner=owner, version=17, slug='', sent_date=datetime.utcnow()) # TODO: slug='' does not raises ValidationError with self.assertRaises(ValidationError): experiment.save() experiment.full_clean()
class BotTestCase(TestCase): def setUp(self): self.experiment = Experiment(name='backgroundcolor', state=ENABLED_STATE) self.experiment.save() self.request = request_factory.get('/', HTTP_USER_AGENT='GoogleBot/2.1') def test_user_does_not_enroll(self): experiment_user = create_user(self.request) experiment_user.set_enrollment(self.experiment, TEST_ALTERNATIVE) self.assertEqual(self.experiment.participant_count(TEST_ALTERNATIVE), 0, "Bot counted towards results") def test_bot_in_control_group(self): experiment_user = create_user(self.request) experiment_user.set_enrollment(self.experiment, TEST_ALTERNATIVE) self.assertEqual(experiment_user.get_enrollment(self.experiment), CONTROL_GROUP, "Bot alternative is not control") self.assertEqual(experiment_user.is_enrolled(self.experiment.name, TEST_ALTERNATIVE, self.request), False, "Bot in test alternative") self.assertEqual(experiment_user.is_enrolled(self.experiment.name, CONTROL_GROUP, self.request), True, "Bot not in control group")
def addexperiment(request, checksum): dir_path = 'media/datasets/' + checksum + "/" dump_file = dir_path + "dataset.csv" meta_file = dir_path + "metadata.json" if request.method == "POST": auth = request.META['HTTP_AUTHORIZATION'].split() username, password = base64.b64decode( auth[1]).decode("utf-8").split(':') user = authenticate(username=username, password=password) if user is None or user.is_active == False: return HttpResponse('AUTHENTICATION_FAILED') if Experiment.objects.filter(checksum=checksum).exists(): return HttpResponse('DATASET_ALREADY_IN_DB') with open(meta_file, "r") as f: experiment_meta = json.load(f) sensors_abbrs = list(pd.read_csv(dump_file, nrows=1).columns) sensors_abbrs.remove('time') sensors = { sensor['abbreviation']: sensor['id'] for sensor in list( Sensor.objects.filter(abbreviation__in=sensors_abbrs).values( 'abbreviation', 'id')) } with open(dump_file, "r") as f: num_lines = sum(1 for line in f) - 1 experiment_start_unix = int( time.mktime( time.strptime(experiment_meta['start_time'], '%Y-%m-%d %H:%M:%S'))) * (10**6) rock = Rock.objects.get(id=experiment_meta['rock_id']) experiment = Experiment(start_time=experiment_meta['start_time'], description=experiment_meta['description'], rock_id=rock, checksum=checksum, nr_data_points=num_lines * len(sensors_abbrs)) experiment.sensors = [id for id in sensors.values()] experiment.save() add_experiment_to_db.delay(checksum) return HttpResponse('EXPERIMENT_BEING_ADDED_TO_THE_DB')
def handle(self, *args, **options): try: user_data = {} if hasattr(self.stdin, 'isatty') and not self.stdin.isatty(): raise NotRunningInTTYException("Not running in a TTY") for field_name in Experiment.REQUIRED_FIELDS: field = Experiment._meta.get_field(field_name) user_data[field_name] = None while user_data[field_name] is None: message = field_name.upper() + ': ' user_data[field_name] = self.get_input_data(field, message) except KeyboardInterrupt: self.stderr.write("\nOperation cancelled.") sys.exit(1) except NotRunningInTTYException: self.stdout.write( "Experiment creation skipped due to not running in a TTY. " "You can run `manage.py create_exp` in your project " "to create one manually.") app_name = user_data['name'] if app_name not in settings.INSTALLED_APPS: try: src = 'exp0' dest = app_name shutil.copytree(src, dest) except Exception as e: self.stderr.write("Directory not copied. Error: %s" % str(e)) try: template_dir = app_name + '/templates/' + app_name os.makedirs(template_dir) with open("vlab/settings.py", "a") as myFile: myFile.write("INSTALLED_APPS += ('%s',)\n" % app_name) myFile.close() experiment = Experiment(**user_data) experiment.save() except Exception as e: self.stderr.write("Directory not copied. Error 1: %s" % str(e)) shutil.rmtree(app_name)
def get_experiment(name, description, category, publishable, user, project, publication_list): #check if experiment exist, if so return, if not create new try: year_later = datetime.now() + timedelta(days=(1*365)) experiment = Experiment(name=name, description=description, category=category, publishable=publishable, date_created=datetime.now(), date_embargo=year_later) experiment.validate_unique() experiment.save() experiment.projects.add(project) for publication in publication_list: experiment.publications.add(publication) experiment_contributor = ExperimentContributor.experiment_contributor_objects.create_experiment_as_superuser(experiment, user) experiment_contributor.validate_unique() experiment_contributor.save() except ValidationError as e: experiment = Experiment.objects.get(name=name) return experiment
class BotTests(object): def setUp(self): self.experiment = Experiment(name='backgroundcolor', state=ENABLED_STATE) self.experiment.save() self.experiment_counter = ExperimentCounter() def test_user_does_not_enroll(self): self.experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) self.assertEqual(self.experiment_counter.participant_count(self.experiment, TEST_ALTERNATIVE), 0, "Bot counted towards results") def test_user_does_not_fire_goals(self): self.experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) self.experiment_user.goal(TEST_GOAL) self.assertEqual(self.experiment_counter.participant_count(self.experiment, TEST_ALTERNATIVE), 0, "Bot counted towards results") def test_bot_in_control_group(self): self.experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) self.assertEqual(self.experiment_user.get_alternative(EXPERIMENT_NAME), 'control', "Bot enrolled in a group") self.assertEqual(self.experiment_user.is_enrolled(self.experiment.name, TEST_ALTERNATIVE), False, "Bot in test alternative") self.assertEqual(self.experiment_user.is_enrolled(self.experiment.name, CONTROL_GROUP), True, "Bot not in control group") def tearDown(self): self.experiment_counter.delete(self.experiment)
class WebUserTests(object): def setUp(self): self.experiment = Experiment(name=EXPERIMENT_NAME, state=ENABLED_STATE) self.experiment.save() self.request = request_factory.get('/') self.request.session = DatabaseSession() self.experiment_counter = ExperimentCounter() def tearDown(self): self.experiment_counter.delete(self.experiment) def test_enrollment_initially_control(self): experiment_user = participant(self.request) self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME), 'control', "Default Enrollment wasn't control") def test_user_enrolls(self): experiment_user = participant(self.request) experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME), TEST_ALTERNATIVE, "Wrong Alternative Set") def test_record_goal_increments_counts(self): experiment_user = participant(self.request) experiment_user.confirm_human() experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) self.assertEqual(self.experiment_counter.goal_count(self.experiment, TEST_ALTERNATIVE, TEST_GOAL), 0) experiment_user.goal(TEST_GOAL) self.assertEqual(self.experiment_counter.goal_count(self.experiment, TEST_ALTERNATIVE, TEST_GOAL), 1, "Did not increment Goal count") def test_can_record_goal_multiple_times(self): experiment_user = participant(self.request) experiment_user.confirm_human() experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) experiment_user.goal(TEST_GOAL) experiment_user.goal(TEST_GOAL) experiment_user.goal(TEST_GOAL) self.assertEqual(self.experiment_counter.goal_count(self.experiment, TEST_ALTERNATIVE, TEST_GOAL), 1, "Did not increment goal count correctly") self.assertEqual(self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, TEST_GOAL), {3: 1}, "Incorrect goal count distribution") def test_counts_increment_immediately_once_confirmed_human(self): experiment_user = participant(self.request) experiment_user.confirm_human() experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) self.assertEqual(self.experiment_counter.participant_count(self.experiment, TEST_ALTERNATIVE), 1, "Did not count participant after confirm human") def test_visit_increases_goal(self): thetime = timezone.now() with patch('experiments.utils.now', return_value=thetime): experiment_user = participant(self.request) experiment_user.confirm_human() experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) experiment_user.visit() self.assertEqual(self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, VISIT_NOT_PRESENT_COUNT_GOAL), {1: 1}, "Not Present Visit was not correctly counted") self.assertEqual(self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, VISIT_PRESENT_COUNT_GOAL), {}, "Present Visit was not correctly counted") with patch('experiments.utils.now', return_value=thetime + timedelta(hours=7)): experiment_user.visit() self.assertEqual(self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, VISIT_NOT_PRESENT_COUNT_GOAL), {2: 1}, "No Present Visit was not correctly counted") self.assertEqual(self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, VISIT_PRESENT_COUNT_GOAL), {1: 1}, "Present Visit was not correctly counted") def test_visit_twice_increases_once(self): experiment_user = participant(self.request) experiment_user.confirm_human() experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) experiment_user.visit() experiment_user.visit() self.assertEqual(self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, VISIT_NOT_PRESENT_COUNT_GOAL), {1: 1}, "Visit was not correctly counted") self.assertEqual(self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, VISIT_PRESENT_COUNT_GOAL), {}, "Present Visit was not correctly counted") def test_user_force_enrolls(self): experiment_user = participant(self.request) experiment_user.enroll(EXPERIMENT_NAME, ['control', 'alternative1', 'alternative2'], force_alternative='alternative2') self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME), 'alternative2') def test_user_does_not_force_enroll_to_new_alternative(self): alternatives = ['control', 'alternative1', 'alternative2'] experiment_user = participant(self.request) experiment_user.enroll(EXPERIMENT_NAME, alternatives) alternative = experiment_user.get_alternative(EXPERIMENT_NAME) self.assertIsNotNone(alternative) other_alternative = random.choice(list(set(alternatives) - set(alternative))) experiment_user.enroll(EXPERIMENT_NAME, alternatives, force_alternative=other_alternative) self.assertEqual(alternative, experiment_user.get_alternative(EXPERIMENT_NAME)) def test_second_force_enroll_does_not_change_alternative(self): alternatives = ['control', 'alternative1', 'alternative2'] experiment_user = participant(self.request) experiment_user.enroll(EXPERIMENT_NAME, alternatives, force_alternative='alternative1') alternative = experiment_user.get_alternative(EXPERIMENT_NAME) self.assertIsNotNone(alternative) other_alternative = random.choice(list(set(alternatives) - set(alternative))) experiment_user.enroll(EXPERIMENT_NAME, alternatives, force_alternative=other_alternative) self.assertEqual(alternative, experiment_user.get_alternative(EXPERIMENT_NAME))
def scheduleHeuristicSignatureRecording(self, job): # Retrieve the heuristic version try: heuristic_version = HeuristicVersion.objects.get(id=job.command.parameters[0]) except: alert = Alert() alert.message = 'Unknown heuristic version ID: %d' % job.command.parameters[0] job.outStream.write("ERROR - %s\n" % alert.message) job.markAsFailed(alert=alert) return job.markAsRunning() # If the heuristic already has a signature, delete it try: signature = heuristic_version.signature[0] except: signature = None if signature is not None: if signature.experiment.isRunning() or signature.experiment.isScheduled: self.channel.sendMessage(Message('CANCEL_EXPERIMENT', [signature.experiment.id])) job.markAsDone() return signature.experiment.configuration.delete() signature.delete() # Retrieve the recording configuration try: configuration_template = Configuration.objects.get(experiment_type=Configuration.SIGNATURE, name__startswith='template/') except: alert = Alert() alert.message = 'No signature recording configuration found' job.outStream.write("ERROR - %s\n" % alert.message) job.markAsFailed(alert=alert) return # Create the configuration configuration = Configuration() configuration.name = configuration_template.name.replace('template/', '%s/' % heuristic_version.fullname()) configuration.heuristics = configuration_template.heuristics configuration.experiment_type = configuration_template.experiment_type configuration.task = configuration_template.task configuration.save() # Add the heuristic version configuration.heuristics_set.add(heuristic_version) # Save the configuration configuration.save() # Add the settings for setting in configuration_template.settings.all(): configuration.addSetting(setting.name, setting.value) # Create the experiment experiment = Experiment() experiment.name = configuration.name experiment.configuration = configuration experiment.save() # Create the signature signature = HeuristicSignature() signature.heuristic_version = heuristic_version signature.experiment = experiment signature.save() self.channel.sendMessage(Message('RUN_EXPERIMENT', [experiment.id])) job.markAsDone()
def scheduleHeuristicEvaluation(self, job): # Retrieve the heuristic version try: heuristic_version = HeuristicVersion.objects.get(id=job.command.parameters[0]) except: alert = Alert() alert.message = 'Unknown heuristic version ID: %d' % job.command.parameters[0] job.outStream.write("ERROR - %s\n" % alert.message) job.markAsFailed(alert=alert) return job.markAsRunning() # If the heuristic is already evaluated, remove the results from the database if heuristic_version.evaluated: configurations = map(lambda x: x.experiment.configuration, heuristic_version.evaluation_results.all()) heuristic_version.evaluation_results.all().delete() heuristic_version.evaluated = False heuristic_version.rank = None heuristic_version.save() for configuration in configurations: configuration.delete() # If the heuristic is currently evaluated, remove the existing results from # the database and cancel the running experiments elif heuristic_version.evaluation_results.count() > 0: for evaluation_results in heuristic_version.evaluation_results.filter(experiment__status=Experiment.STATUS_RUNNING): self.channel.sendMessage(Message('CANCEL_EXPERIMENT', [evaluation_results.experiment.id])) configurations = map(lambda x: x.experiment.configuration, heuristic_version.evaluation_results.exclude(experiment__status=Experiment.STATUS_RUNNING)) heuristic_version.evaluation_results.exclude(experiment__status=Experiment.STATUS_RUNNING).delete() for configuration in configurations: configuration.delete() job.markAsDone() return # Retrieve all the evaluation configurations evaluation_configurations = Configuration.objects.filter(experiment_type=Configuration.EVALUATION, name__startswith='template/') if evaluation_configurations.count() == 0: alert = Alert() alert.message = 'No evaluation configuration found' job.outStream.write("ERROR - %s\n" % alert.message) job.markAsFailed(alert=alert) return # Schedule the evaluation experiments for evaluation_configuration in evaluation_configurations: # Create the configuration configuration = Configuration() configuration.name = evaluation_configuration.name.replace('template/', '%s/' % heuristic_version.fullname()) configuration.heuristics = evaluation_configuration.heuristics configuration.experiment_type = evaluation_configuration.experiment_type configuration.task = evaluation_configuration.task configuration.save() # Add the list of heuristic versions for version in evaluation_configuration.heuristics_set.all(): configuration.heuristics_set.add(version) if heuristic_version not in evaluation_configuration.heuristics_set.all(): configuration.heuristics_set.add(heuristic_version) else: configuration.addSetting('PREDICTOR_SETUP/ADDITIONAL_HEURISTICS', heuristic_version.absolutename()) # Add the list of instruments for instrument in evaluation_configuration.instruments_set.all(): configuration.instruments_set.add(instrument) # Save the configuration configuration.save() # Add the settings for setting in evaluation_configuration.settings.all(): configuration.addSetting(setting.name, setting.value) # Create the experiment experiment = Experiment() experiment.name = configuration.name experiment.configuration = configuration experiment.save() # Create the evaluation results entry evaluation_results = HeuristicEvaluationResults() evaluation_results.heuristic_version = heuristic_version evaluation_results.evaluation_config = evaluation_configuration evaluation_results.experiment = experiment evaluation_results.save() self.channel.sendMessage(Message('RUN_EXPERIMENT', [experiment.id])) job.markAsDone()
def schedulePublicExperiments(self, job): job.markAsRunning() # Retrieve the configuration templates for template in Configuration.objects.filter(experiment_type=Configuration.PUBLIC, name__startswith='template/'): # Retrieve the list of heuristic versions already used in a public experiment # based on this template already_used_heuristic_versions = [] previous = None try: previous = Configuration.objects.filter(experiment_type=Configuration.PUBLIC, name__startswith=template.name.replace('template/', '')).order_by('-experiment__creation_date')[0] if previous.experiment.status == Experiment.STATUS_RUNNING: continue already_used_heuristic_versions = list(previous.heuristics_set.all()) except: pass # Ensure that some new heuristics exist since the last public experiment # based on this template current_heuristic_versions = map(lambda x: x.latest_public_version, list(Heuristic.objects.filter(latest_public_version__isnull=False, simple=False))) new_heuristic_versions = filter(lambda x: x not in already_used_heuristic_versions, current_heuristic_versions) if len(new_heuristic_versions) == 0: continue # Cancel the previous experiment if needed if (previous is not None) and (previous.experiment.status == Experiment.STATUS_SCHEDULED): self.channel.sendMessage(Message('CANCEL_EXPERIMENT', [previous.experiment.id])) continue # Create the configuration configuration = Configuration() configuration.name = template.name.replace('template/', '') + '/' + datetime.now().strftime('%Y-%m-%d %H:%M:%S') configuration.heuristics = template.heuristics configuration.experiment_type = template.experiment_type configuration.task = template.task configuration.save() # Add the list of heuristic versions for heuristic_version in current_heuristic_versions: configuration.heuristics_set.add(heuristic_version) # Add the list of instruments for instrument in template.instruments_set.all(): configuration.instruments_set.add(instrument) # Save the configuration configuration.save() # Add the settings for setting in template.settings.all(): configuration.addSetting(setting.name, setting.value) # Create the experiment experiment = Experiment() experiment.name = configuration.name experiment.configuration = configuration experiment.save() self.channel.sendMessage(Message('RUN_EXPERIMENT', [experiment.id])) job.markAsDone()
class WebUserTests: def setUp(self): self.experiment = Experiment(name=EXPERIMENT_NAME, state=ENABLED_STATE) self.experiment.save() self.request = request_factory.get('/') self.request.session = DatabaseSession() def tearDown(self): self.experiment.delete() def confirm_human(self, experiment_user): pass def participants(self, alternative): return self.experiment.participant_count(alternative) def enrollment_initially_none(self, ): experiment_user = participant(self.request) self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME), None) def test_user_enrolls(self): experiment_user = participant(self.request) experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME), TEST_ALTERNATIVE) def test_record_goal_increments_counts(self): experiment_user = participant(self.request) self.confirm_human(experiment_user) experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) self.assertEqual( self.experiment.goal_count(TEST_ALTERNATIVE, TEST_GOAL), 0) experiment_user.goal(TEST_GOAL) self.assertEqual( self.experiment.goal_count(TEST_ALTERNATIVE, TEST_GOAL), 1) def test_can_record_goal_multiple_times(self): experiment_user = participant(self.request) self.confirm_human(experiment_user) experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) experiment_user.goal(TEST_GOAL) experiment_user.goal(TEST_GOAL) experiment_user.goal(TEST_GOAL) self.assertEqual( self.experiment.goal_count(TEST_ALTERNATIVE, TEST_GOAL), 1) def test_counts_increment_immediately_once_confirmed_human(self): experiment_user = participant(self.request) self.confirm_human(experiment_user) experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) self.assertEqual(self.participants(TEST_ALTERNATIVE), 1, "Did not count participant after confirm human") def test_visit_increases_goal(self): experiment_user = participant(self.request) self.confirm_human(experiment_user) experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) experiment_user.visit() self.assertEqual( self.experiment.goal_distribution(TEST_ALTERNATIVE, VISIT_COUNT_GOAL), {1: 1}) def test_visit_twice_increases_once(self): experiment_user = participant(self.request) self.confirm_human(experiment_user) experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) experiment_user.visit() experiment_user.visit() self.assertEqual( self.experiment.goal_distribution(TEST_ALTERNATIVE, VISIT_COUNT_GOAL), {1: 1})
class WebUserTests(object): def setUp(self): self.experiment = Experiment(name=EXPERIMENT_NAME, state=ENABLED_STATE) self.experiment.save() self.request = request_factory.get('/') self.request.session = DatabaseSession() self.experiment_counter = ExperimentCounter() def tearDown(self): self.experiment_counter.delete(self.experiment) def test_enrollment_initially_control(self): experiment_user = participant(self.request) self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME), 'control', "Default Enrollment wasn't control") def test_user_enrolls(self): experiment_user = participant(self.request) experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME), TEST_ALTERNATIVE, "Wrong Alternative Set") def test_record_goal_increments_counts(self): experiment_user = participant(self.request) experiment_user.confirm_human() experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) self.assertEqual( self.experiment_counter.goal_count(self.experiment, TEST_ALTERNATIVE, TEST_GOAL), 0) experiment_user.goal(TEST_GOAL) self.assertEqual( self.experiment_counter.goal_count(self.experiment, TEST_ALTERNATIVE, TEST_GOAL), 1, "Did not increment Goal count") def test_can_record_goal_multiple_times(self): experiment_user = participant(self.request) experiment_user.confirm_human() experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) experiment_user.goal(TEST_GOAL) experiment_user.goal(TEST_GOAL) experiment_user.goal(TEST_GOAL) self.assertEqual( self.experiment_counter.goal_count(self.experiment, TEST_ALTERNATIVE, TEST_GOAL), 1, "Did not increment goal count correctly") self.assertEqual( self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, TEST_GOAL), {3: 1}, "Incorrect goal count distribution") def test_counts_increment_immediately_once_confirmed_human(self): experiment_user = participant(self.request) experiment_user.confirm_human() experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) self.assertEqual( self.experiment_counter.participant_count(self.experiment, TEST_ALTERNATIVE), 1, "Did not count participant after confirm human") def test_visit_increases_goal(self): thetime = timezone.now() with mock.patch('experiments.utils.now', return_value=thetime): experiment_user = participant(self.request) experiment_user.confirm_human() experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) experiment_user.visit() self.assertEqual( self.experiment_counter.goal_distribution( self.experiment, TEST_ALTERNATIVE, VISIT_NOT_PRESENT_COUNT_GOAL), {1: 1}, "Not Present Visit was not correctly counted") self.assertEqual( self.experiment_counter.goal_distribution( self.experiment, TEST_ALTERNATIVE, VISIT_PRESENT_COUNT_GOAL), {}, "Present Visit was not correctly counted") with mock.patch('experiments.utils.now', return_value=thetime + timedelta(hours=7)): experiment_user.visit() self.assertEqual( self.experiment_counter.goal_distribution( self.experiment, TEST_ALTERNATIVE, VISIT_NOT_PRESENT_COUNT_GOAL), {2: 1}, "No Present Visit was not correctly counted") self.assertEqual( self.experiment_counter.goal_distribution( self.experiment, TEST_ALTERNATIVE, VISIT_PRESENT_COUNT_GOAL), {1: 1}, "Present Visit was not correctly counted") def test_visit_twice_increases_once(self): experiment_user = participant(self.request) experiment_user.confirm_human() experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) experiment_user.visit() experiment_user.visit() self.assertEqual( self.experiment_counter.goal_distribution( self.experiment, TEST_ALTERNATIVE, VISIT_NOT_PRESENT_COUNT_GOAL), {1: 1}, "Visit was not correctly counted") self.assertEqual( self.experiment_counter.goal_distribution( self.experiment, TEST_ALTERNATIVE, VISIT_PRESENT_COUNT_GOAL), {}, "Present Visit was not correctly counted") def test_user_force_enrolls(self): experiment_user = participant(self.request) experiment_user.enroll(EXPERIMENT_NAME, ['control', 'alternative1', 'alternative2'], force_alternative='alternative2') self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME), 'alternative2') def test_user_does_not_force_enroll_to_new_alternative(self): alternatives = ['control', 'alternative1', 'alternative2'] experiment_user = participant(self.request) experiment_user.enroll(EXPERIMENT_NAME, alternatives) alternative = experiment_user.get_alternative(EXPERIMENT_NAME) self.assertIsNotNone(alternative) other_alternative = random.choice( list(set(alternatives) - set(alternative))) experiment_user.enroll(EXPERIMENT_NAME, alternatives, force_alternative=other_alternative) self.assertEqual(alternative, experiment_user.get_alternative(EXPERIMENT_NAME)) def test_second_force_enroll_does_not_change_alternative(self): alternatives = ['control', 'alternative1', 'alternative2'] experiment_user = participant(self.request) experiment_user.enroll(EXPERIMENT_NAME, alternatives, force_alternative='alternative1') alternative = experiment_user.get_alternative(EXPERIMENT_NAME) self.assertIsNotNone(alternative) other_alternative = random.choice( list(set(alternatives) - set(alternative))) experiment_user.enroll(EXPERIMENT_NAME, alternatives, force_alternative=other_alternative) self.assertEqual(alternative, experiment_user.get_alternative(EXPERIMENT_NAME))
class WebUserTests(object): def setUp(self): self.experiment = Experiment(name=EXPERIMENT_NAME, state=ENABLED_STATE) self.experiment.save() self.request = request_factory.get("/") self.request.session = DatabaseSession() self.experiment_counter = ExperimentCounter() def tearDown(self): self.experiment_counter.delete(self.experiment) def confirm_human(self, experiment_user): pass def participants(self, alternative): return self.experiment_counter.participant_count(self.experiment, alternative) def enrollment_initially_none(self,): experiment_user = participant(self.request) self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME), None) def test_user_enrolls(self): experiment_user = participant(self.request) experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME), TEST_ALTERNATIVE) def test_record_goal_increments_counts(self): experiment_user = participant(self.request) self.confirm_human(experiment_user) experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) self.assertEqual(self.experiment_counter.goal_count(self.experiment, TEST_ALTERNATIVE, TEST_GOAL), 0) experiment_user.goal(TEST_GOAL) self.assertEqual(self.experiment_counter.goal_count(self.experiment, TEST_ALTERNATIVE, TEST_GOAL), 1) def test_can_record_goal_multiple_times(self): experiment_user = participant(self.request) self.confirm_human(experiment_user) experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) experiment_user.goal(TEST_GOAL) experiment_user.goal(TEST_GOAL) experiment_user.goal(TEST_GOAL) self.assertEqual(self.experiment_counter.goal_count(self.experiment, TEST_ALTERNATIVE, TEST_GOAL), 1) def test_counts_increment_immediately_once_confirmed_human(self): experiment_user = participant(self.request) self.confirm_human(experiment_user) experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) self.assertEqual(self.participants(TEST_ALTERNATIVE), 1, "Did not count participant after confirm human") def test_visit_increases_goal(self): experiment_user = participant(self.request) self.confirm_human(experiment_user) experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) experiment_user.visit() self.assertEqual( self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, VISIT_COUNT_GOAL), {1: 1} ) def test_visit_twice_increases_once(self): experiment_user = participant(self.request) self.confirm_human(experiment_user) experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) experiment_user.visit() experiment_user.visit() self.assertEqual( self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, VISIT_COUNT_GOAL), {1: 1} )