class BotTestCase(TestCase):
def setUp(self):
self.experiment = Experiment(name="backgroundcolor", state=ENABLED_STATE)
self.experiment.save()
self.request = request_factory.get("/", HTTP_USER_AGENT="GoogleBot/2.1")
self.experiment_counter = ExperimentCounter()
def test_user_does_not_enroll(self):
experiment_user = participant(self.request)
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
self.assertEqual(
self.experiment_counter.participant_count(self.experiment, TEST_ALTERNATIVE),
0,
"Bot counted towards results",
)
def test_bot_in_control_group(self):
experiment_user = participant(self.request)
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME), "control", "Bot enrolled in a group")
self.assertEqual(
experiment_user.is_enrolled(self.experiment.name, TEST_ALTERNATIVE, self.request),
False,
"Bot in test alternative",
)
self.assertEqual(
experiment_user.is_enrolled(self.experiment.name, CONTROL_GROUP, self.request),
True,
"Bot not in control group",
)
def tearDown(self):
self.experiment_counter.delete(self.experiment)
class BotTestCase(TestCase):
def setUp(self):
self.experiment = Experiment(name='backgroundcolor',
state=ENABLED_STATE)
self.experiment.save()
self.request = request_factory.get('/',
HTTP_USER_AGENT='GoogleBot/2.1')
def test_user_does_not_enroll(self):
experiment_user = create_user(self.request)
experiment_user.set_enrollment(self.experiment, TEST_ALTERNATIVE)
self.assertEqual(self.experiment.participant_count(TEST_ALTERNATIVE),
0, "Bot counted towards results")
def test_bot_in_control_group(self):
experiment_user = create_user(self.request)
experiment_user.set_enrollment(self.experiment, TEST_ALTERNATIVE)
self.assertEqual(experiment_user.get_enrollment(self.experiment),
CONTROL_GROUP, "Bot alternative is not control")
self.assertEqual(
experiment_user.is_enrolled(self.experiment.name, TEST_ALTERNATIVE,
self.request), False,
"Bot in test alternative")
self.assertEqual(
experiment_user.is_enrolled(self.experiment.name, CONTROL_GROUP,
self.request), True,
"Bot not in control group")
def tearDown(self):
self.experiment.delete()
def test_experiment_is_related_to_owner(self):
owner = User.objects.first()
experiment = Experiment(nes_id=17,
owner=owner,
version=1,
sent_date=datetime.utcnow())
experiment.save()
self.assertIn(experiment, owner.experiment_set.all())
class WebUserTests:
def setUp(self):
self.experiment = Experiment(name='backgroundcolor',
state=ENABLED_STATE)
self.experiment.save()
self.request = request_factory.get('/')
self.request.session = DatabaseSession()
def tearDown(self):
self.experiment.delete()
def confirm_human(self, experiment_user):
pass
def participants(self, alternative):
return self.experiment.participant_count(alternative)
def enrollment_initially_none(self, ):
experiment_user = create_user(self.request)
self.assertEqual(experiment_user.get_enrollment(self.experiment), None)
def test_user_enrolls(self):
experiment_user = create_user(self.request)
experiment_user.set_enrollment(self.experiment, TEST_ALTERNATIVE)
self.assertEqual(experiment_user.get_enrollment(self.experiment),
TEST_ALTERNATIVE)
def test_record_goal_increments_counts(self):
experiment_user = create_user(self.request)
self.confirm_human(experiment_user)
experiment_user.set_enrollment(self.experiment, TEST_ALTERNATIVE)
self.assertEqual(
self.experiment.goal_count(TEST_ALTERNATIVE, TEST_GOAL), 0)
experiment_user.record_goal(TEST_GOAL)
self.assertEqual(
self.experiment.goal_count(TEST_ALTERNATIVE, TEST_GOAL), 1)
def test_can_record_goal_multiple_times(self):
experiment_user = create_user(self.request)
self.confirm_human(experiment_user)
experiment_user.set_enrollment(self.experiment, TEST_ALTERNATIVE)
experiment_user.record_goal(TEST_GOAL)
experiment_user.record_goal(TEST_GOAL)
experiment_user.record_goal(TEST_GOAL)
self.assertEqual(
self.experiment.goal_count(TEST_ALTERNATIVE, TEST_GOAL), 1)
def test_counts_increment_immediately_once_confirmed_human(self):
experiment_user = create_user(self.request)
self.confirm_human(experiment_user)
experiment_user.set_enrollment(self.experiment, TEST_ALTERNATIVE)
self.assertEqual(self.participants(TEST_ALTERNATIVE), 1,
"Did not count participant after confirm human")
def approve_experiment(experiment: Experiment, request) -> None:
"""Approves an experiment, and informs users of the approval
:param experiment: :class:`Experiment` the experiment to approve
:param request: Django request. Used to create an absolute URL in the mail
"""
experiment.approved = True
experiment.save()
sent_confirmation_email(experiment, request)
class WebUserTests:
def setUp(self):
self.experiment = Experiment(name='backgroundcolor', state=ENABLED_STATE)
self.experiment.save()
self.request = request_factory.get('/')
self.request.session = DatabaseSession()
def tearDown(self):
self.experiment.delete()
def confirm_human(self, experiment_user):
pass
def participants(self, alternative):
return self.experiment.participant_count(alternative)
def enrollment_initially_none(self,):
experiment_user = create_user(self.request)
self.assertEqual(experiment_user.get_enrollment(self.experiment), None)
def test_user_enrolls(self):
experiment_user = create_user(self.request)
experiment_user.set_enrollment(self.experiment, TEST_ALTERNATIVE)
self.assertEqual(experiment_user.get_enrollment(self.experiment), TEST_ALTERNATIVE)
def test_record_goal_increments_counts(self):
experiment_user = create_user(self.request)
self.confirm_human(experiment_user)
experiment_user.set_enrollment(self.experiment, TEST_ALTERNATIVE)
self.assertEqual(self.experiment.goal_count(TEST_ALTERNATIVE, TEST_GOAL), 0)
experiment_user.record_goal(TEST_GOAL)
self.assertEqual(self.experiment.goal_count(TEST_ALTERNATIVE, TEST_GOAL), 1)
def test_can_record_goal_multiple_times(self):
experiment_user = create_user(self.request)
self.confirm_human(experiment_user)
experiment_user.set_enrollment(self.experiment, TEST_ALTERNATIVE)
experiment_user.record_goal(TEST_GOAL)
experiment_user.record_goal(TEST_GOAL)
experiment_user.record_goal(TEST_GOAL)
self.assertEqual(self.experiment.goal_count(TEST_ALTERNATIVE, TEST_GOAL), 1)
def test_counts_increment_immediately_once_confirmed_human(self):
experiment_user = create_user(self.request)
self.confirm_human(experiment_user)
experiment_user.set_enrollment(self.experiment, TEST_ALTERNATIVE)
self.assertEqual(self.participants(TEST_ALTERNATIVE), 1, "Did not count participant after confirm human")
def test_cannot_save_empty_attributes(self):
owner = User.objects.first()
# version=17: large number to avoid conflicts with global setup
experiment = Experiment(nes_id=1,
title='',
description='',
owner=owner,
version=17,
slug='',
sent_date=datetime.utcnow())
# TODO: slug='' does not raises ValidationError
with self.assertRaises(ValidationError):
experiment.save()
experiment.full_clean()
class BotTestCase(TestCase):
def setUp(self):
self.experiment = Experiment(name='backgroundcolor', state=ENABLED_STATE)
self.experiment.save()
self.request = request_factory.get('/', HTTP_USER_AGENT='GoogleBot/2.1')
def test_user_does_not_enroll(self):
experiment_user = create_user(self.request)
experiment_user.set_enrollment(self.experiment, TEST_ALTERNATIVE)
self.assertEqual(self.experiment.participant_count(TEST_ALTERNATIVE), 0, "Bot counted towards results")
def test_bot_in_control_group(self):
experiment_user = create_user(self.request)
experiment_user.set_enrollment(self.experiment, TEST_ALTERNATIVE)
self.assertEqual(experiment_user.get_enrollment(self.experiment), CONTROL_GROUP, "Bot alternative is not control")
self.assertEqual(experiment_user.is_enrolled(self.experiment.name, TEST_ALTERNATIVE, self.request), False, "Bot in test alternative")
self.assertEqual(experiment_user.is_enrolled(self.experiment.name, CONTROL_GROUP, self.request), True, "Bot not in control group")
def addexperiment(request, checksum):
dir_path = 'media/datasets/' + checksum + "/"
dump_file = dir_path + "dataset.csv"
meta_file = dir_path + "metadata.json"
if request.method == "POST":
auth = request.META['HTTP_AUTHORIZATION'].split()
username, password = base64.b64decode(
auth[1]).decode("utf-8").split(':')
user = authenticate(username=username, password=password)
if user is None or user.is_active == False:
return HttpResponse('AUTHENTICATION_FAILED')
if Experiment.objects.filter(checksum=checksum).exists():
return HttpResponse('DATASET_ALREADY_IN_DB')
with open(meta_file, "r") as f:
experiment_meta = json.load(f)
sensors_abbrs = list(pd.read_csv(dump_file, nrows=1).columns)
sensors_abbrs.remove('time')
sensors = {
sensor['abbreviation']: sensor['id']
for sensor in list(
Sensor.objects.filter(abbreviation__in=sensors_abbrs).values(
'abbreviation', 'id'))
}
with open(dump_file, "r") as f:
num_lines = sum(1 for line in f) - 1
experiment_start_unix = int(
time.mktime(
time.strptime(experiment_meta['start_time'],
'%Y-%m-%d %H:%M:%S'))) * (10**6)
rock = Rock.objects.get(id=experiment_meta['rock_id'])
experiment = Experiment(start_time=experiment_meta['start_time'],
description=experiment_meta['description'],
rock_id=rock,
checksum=checksum,
nr_data_points=num_lines * len(sensors_abbrs))
experiment.sensors = [id for id in sensors.values()]
experiment.save()
add_experiment_to_db.delay(checksum)
return HttpResponse('EXPERIMENT_BEING_ADDED_TO_THE_DB')
def handle(self, *args, **options):
try:
user_data = {}
if hasattr(self.stdin, 'isatty') and not self.stdin.isatty():
raise NotRunningInTTYException("Not running in a TTY")
for field_name in Experiment.REQUIRED_FIELDS:
field = Experiment._meta.get_field(field_name)
user_data[field_name] = None
while user_data[field_name] is None:
message = field_name.upper() + ': '
user_data[field_name] = self.get_input_data(field, message)
except KeyboardInterrupt:
self.stderr.write("\nOperation cancelled.")
sys.exit(1)
except NotRunningInTTYException:
self.stdout.write(
"Experiment creation skipped due to not running in a TTY. "
"You can run `manage.py create_exp` in your project "
"to create one manually.")
app_name = user_data['name']
if app_name not in settings.INSTALLED_APPS:
try:
src = 'exp0'
dest = app_name
shutil.copytree(src, dest)
except Exception as e:
self.stderr.write("Directory not copied. Error: %s" % str(e))
try:
template_dir = app_name + '/templates/' + app_name
os.makedirs(template_dir)
with open("vlab/settings.py", "a") as myFile:
myFile.write("INSTALLED_APPS += ('%s',)\n" % app_name)
myFile.close()
experiment = Experiment(**user_data)
experiment.save()
except Exception as e:
self.stderr.write("Directory not copied. Error 1: %s" % str(e))
shutil.rmtree(app_name)
def get_experiment(name, description, category, publishable, user, project, publication_list):
#check if experiment exist, if so return, if not create new
try:
year_later = datetime.now() + timedelta(days=(1*365))
experiment = Experiment(name=name, description=description, category=category,
publishable=publishable, date_created=datetime.now(), date_embargo=year_later)
experiment.validate_unique()
experiment.save()
experiment.projects.add(project)
for publication in publication_list:
experiment.publications.add(publication)
experiment_contributor = ExperimentContributor.experiment_contributor_objects.create_experiment_as_superuser(experiment, user)
experiment_contributor.validate_unique()
experiment_contributor.save()
except ValidationError as e:
experiment = Experiment.objects.get(name=name)
return experiment
class BotTests(object):
def setUp(self):
self.experiment = Experiment(name='backgroundcolor', state=ENABLED_STATE)
self.experiment.save()
self.experiment_counter = ExperimentCounter()
def test_user_does_not_enroll(self):
self.experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
self.assertEqual(self.experiment_counter.participant_count(self.experiment, TEST_ALTERNATIVE), 0, "Bot counted towards results")
def test_user_does_not_fire_goals(self):
self.experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
self.experiment_user.goal(TEST_GOAL)
self.assertEqual(self.experiment_counter.participant_count(self.experiment, TEST_ALTERNATIVE), 0, "Bot counted towards results")
def test_bot_in_control_group(self):
self.experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
self.assertEqual(self.experiment_user.get_alternative(EXPERIMENT_NAME), 'control', "Bot enrolled in a group")
self.assertEqual(self.experiment_user.is_enrolled(self.experiment.name, TEST_ALTERNATIVE), False, "Bot in test alternative")
self.assertEqual(self.experiment_user.is_enrolled(self.experiment.name, CONTROL_GROUP), True, "Bot not in control group")
def tearDown(self):
self.experiment_counter.delete(self.experiment)
class BotTests(object):
def setUp(self):
self.experiment = Experiment(name='backgroundcolor', state=ENABLED_STATE)
self.experiment.save()
self.experiment_counter = ExperimentCounter()
def test_user_does_not_enroll(self):
self.experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
self.assertEqual(self.experiment_counter.participant_count(self.experiment, TEST_ALTERNATIVE), 0, "Bot counted towards results")
def test_user_does_not_fire_goals(self):
self.experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
self.experiment_user.goal(TEST_GOAL)
self.assertEqual(self.experiment_counter.participant_count(self.experiment, TEST_ALTERNATIVE), 0, "Bot counted towards results")
def test_bot_in_control_group(self):
self.experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
self.assertEqual(self.experiment_user.get_alternative(EXPERIMENT_NAME), 'control', "Bot enrolled in a group")
self.assertEqual(self.experiment_user.is_enrolled(self.experiment.name, TEST_ALTERNATIVE), False, "Bot in test alternative")
self.assertEqual(self.experiment_user.is_enrolled(self.experiment.name, CONTROL_GROUP), True, "Bot not in control group")
def tearDown(self):
self.experiment_counter.delete(self.experiment)
class WebUserTests(object):
def setUp(self):
self.experiment = Experiment(name=EXPERIMENT_NAME, state=ENABLED_STATE)
self.experiment.save()
self.request = request_factory.get('/')
self.request.session = DatabaseSession()
self.experiment_counter = ExperimentCounter()
def tearDown(self):
self.experiment_counter.delete(self.experiment)
def test_enrollment_initially_control(self):
experiment_user = participant(self.request)
self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME), 'control', "Default Enrollment wasn't control")
def test_user_enrolls(self):
experiment_user = participant(self.request)
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME), TEST_ALTERNATIVE, "Wrong Alternative Set")
def test_record_goal_increments_counts(self):
experiment_user = participant(self.request)
experiment_user.confirm_human()
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
self.assertEqual(self.experiment_counter.goal_count(self.experiment, TEST_ALTERNATIVE, TEST_GOAL), 0)
experiment_user.goal(TEST_GOAL)
self.assertEqual(self.experiment_counter.goal_count(self.experiment, TEST_ALTERNATIVE, TEST_GOAL), 1, "Did not increment Goal count")
def test_can_record_goal_multiple_times(self):
experiment_user = participant(self.request)
experiment_user.confirm_human()
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
experiment_user.goal(TEST_GOAL)
experiment_user.goal(TEST_GOAL)
experiment_user.goal(TEST_GOAL)
self.assertEqual(self.experiment_counter.goal_count(self.experiment, TEST_ALTERNATIVE, TEST_GOAL), 1, "Did not increment goal count correctly")
self.assertEqual(self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, TEST_GOAL), {3: 1}, "Incorrect goal count distribution")
def test_counts_increment_immediately_once_confirmed_human(self):
experiment_user = participant(self.request)
experiment_user.confirm_human()
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
self.assertEqual(self.experiment_counter.participant_count(self.experiment, TEST_ALTERNATIVE), 1, "Did not count participant after confirm human")
def test_visit_increases_goal(self):
thetime = timezone.now()
with patch('experiments.utils.now', return_value=thetime):
experiment_user = participant(self.request)
experiment_user.confirm_human()
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
experiment_user.visit()
self.assertEqual(self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, VISIT_NOT_PRESENT_COUNT_GOAL), {1: 1}, "Not Present Visit was not correctly counted")
self.assertEqual(self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, VISIT_PRESENT_COUNT_GOAL), {}, "Present Visit was not correctly counted")
with patch('experiments.utils.now', return_value=thetime + timedelta(hours=7)):
experiment_user.visit()
self.assertEqual(self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, VISIT_NOT_PRESENT_COUNT_GOAL), {2: 1}, "No Present Visit was not correctly counted")
self.assertEqual(self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, VISIT_PRESENT_COUNT_GOAL), {1: 1}, "Present Visit was not correctly counted")
def test_visit_twice_increases_once(self):
experiment_user = participant(self.request)
experiment_user.confirm_human()
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
experiment_user.visit()
experiment_user.visit()
self.assertEqual(self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, VISIT_NOT_PRESENT_COUNT_GOAL), {1: 1}, "Visit was not correctly counted")
self.assertEqual(self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, VISIT_PRESENT_COUNT_GOAL), {}, "Present Visit was not correctly counted")
def test_user_force_enrolls(self):
experiment_user = participant(self.request)
experiment_user.enroll(EXPERIMENT_NAME, ['control', 'alternative1', 'alternative2'], force_alternative='alternative2')
self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME), 'alternative2')
def test_user_does_not_force_enroll_to_new_alternative(self):
alternatives = ['control', 'alternative1', 'alternative2']
experiment_user = participant(self.request)
experiment_user.enroll(EXPERIMENT_NAME, alternatives)
alternative = experiment_user.get_alternative(EXPERIMENT_NAME)
self.assertIsNotNone(alternative)
other_alternative = random.choice(list(set(alternatives) - set(alternative)))
experiment_user.enroll(EXPERIMENT_NAME, alternatives, force_alternative=other_alternative)
self.assertEqual(alternative, experiment_user.get_alternative(EXPERIMENT_NAME))
def test_second_force_enroll_does_not_change_alternative(self):
alternatives = ['control', 'alternative1', 'alternative2']
experiment_user = participant(self.request)
experiment_user.enroll(EXPERIMENT_NAME, alternatives, force_alternative='alternative1')
alternative = experiment_user.get_alternative(EXPERIMENT_NAME)
self.assertIsNotNone(alternative)
other_alternative = random.choice(list(set(alternatives) - set(alternative)))
experiment_user.enroll(EXPERIMENT_NAME, alternatives, force_alternative=other_alternative)
self.assertEqual(alternative, experiment_user.get_alternative(EXPERIMENT_NAME))
def scheduleHeuristicSignatureRecording(self, job):
# Retrieve the heuristic version
try:
heuristic_version = HeuristicVersion.objects.get(id=job.command.parameters[0])
except:
alert = Alert()
alert.message = 'Unknown heuristic version ID: %d' % job.command.parameters[0]
job.outStream.write("ERROR - %s\n" % alert.message)
job.markAsFailed(alert=alert)
return
job.markAsRunning()
# If the heuristic already has a signature, delete it
try:
signature = heuristic_version.signature[0]
except:
signature = None
if signature is not None:
if signature.experiment.isRunning() or signature.experiment.isScheduled:
self.channel.sendMessage(Message('CANCEL_EXPERIMENT', [signature.experiment.id]))
job.markAsDone()
return
signature.experiment.configuration.delete()
signature.delete()
# Retrieve the recording configuration
try:
configuration_template = Configuration.objects.get(experiment_type=Configuration.SIGNATURE,
name__startswith='template/')
except:
alert = Alert()
alert.message = 'No signature recording configuration found'
job.outStream.write("ERROR - %s\n" % alert.message)
job.markAsFailed(alert=alert)
return
# Create the configuration
configuration = Configuration()
configuration.name = configuration_template.name.replace('template/', '%s/' % heuristic_version.fullname())
configuration.heuristics = configuration_template.heuristics
configuration.experiment_type = configuration_template.experiment_type
configuration.task = configuration_template.task
configuration.save()
# Add the heuristic version
configuration.heuristics_set.add(heuristic_version)
# Save the configuration
configuration.save()
# Add the settings
for setting in configuration_template.settings.all():
configuration.addSetting(setting.name, setting.value)
# Create the experiment
experiment = Experiment()
experiment.name = configuration.name
experiment.configuration = configuration
experiment.save()
# Create the signature
signature = HeuristicSignature()
signature.heuristic_version = heuristic_version
signature.experiment = experiment
signature.save()
self.channel.sendMessage(Message('RUN_EXPERIMENT', [experiment.id]))
job.markAsDone()
def scheduleHeuristicEvaluation(self, job):
# Retrieve the heuristic version
try:
heuristic_version = HeuristicVersion.objects.get(id=job.command.parameters[0])
except:
alert = Alert()
alert.message = 'Unknown heuristic version ID: %d' % job.command.parameters[0]
job.outStream.write("ERROR - %s\n" % alert.message)
job.markAsFailed(alert=alert)
return
job.markAsRunning()
# If the heuristic is already evaluated, remove the results from the database
if heuristic_version.evaluated:
configurations = map(lambda x: x.experiment.configuration, heuristic_version.evaluation_results.all())
heuristic_version.evaluation_results.all().delete()
heuristic_version.evaluated = False
heuristic_version.rank = None
heuristic_version.save()
for configuration in configurations:
configuration.delete()
# If the heuristic is currently evaluated, remove the existing results from
# the database and cancel the running experiments
elif heuristic_version.evaluation_results.count() > 0:
for evaluation_results in heuristic_version.evaluation_results.filter(experiment__status=Experiment.STATUS_RUNNING):
self.channel.sendMessage(Message('CANCEL_EXPERIMENT', [evaluation_results.experiment.id]))
configurations = map(lambda x: x.experiment.configuration, heuristic_version.evaluation_results.exclude(experiment__status=Experiment.STATUS_RUNNING))
heuristic_version.evaluation_results.exclude(experiment__status=Experiment.STATUS_RUNNING).delete()
for configuration in configurations:
configuration.delete()
job.markAsDone()
return
# Retrieve all the evaluation configurations
evaluation_configurations = Configuration.objects.filter(experiment_type=Configuration.EVALUATION,
name__startswith='template/')
if evaluation_configurations.count() == 0:
alert = Alert()
alert.message = 'No evaluation configuration found'
job.outStream.write("ERROR - %s\n" % alert.message)
job.markAsFailed(alert=alert)
return
# Schedule the evaluation experiments
for evaluation_configuration in evaluation_configurations:
# Create the configuration
configuration = Configuration()
configuration.name = evaluation_configuration.name.replace('template/', '%s/' % heuristic_version.fullname())
configuration.heuristics = evaluation_configuration.heuristics
configuration.experiment_type = evaluation_configuration.experiment_type
configuration.task = evaluation_configuration.task
configuration.save()
# Add the list of heuristic versions
for version in evaluation_configuration.heuristics_set.all():
configuration.heuristics_set.add(version)
if heuristic_version not in evaluation_configuration.heuristics_set.all():
configuration.heuristics_set.add(heuristic_version)
else:
configuration.addSetting('PREDICTOR_SETUP/ADDITIONAL_HEURISTICS', heuristic_version.absolutename())
# Add the list of instruments
for instrument in evaluation_configuration.instruments_set.all():
configuration.instruments_set.add(instrument)
# Save the configuration
configuration.save()
# Add the settings
for setting in evaluation_configuration.settings.all():
configuration.addSetting(setting.name, setting.value)
# Create the experiment
experiment = Experiment()
experiment.name = configuration.name
experiment.configuration = configuration
experiment.save()
# Create the evaluation results entry
evaluation_results = HeuristicEvaluationResults()
evaluation_results.heuristic_version = heuristic_version
evaluation_results.evaluation_config = evaluation_configuration
evaluation_results.experiment = experiment
evaluation_results.save()
self.channel.sendMessage(Message('RUN_EXPERIMENT', [experiment.id]))
job.markAsDone()
def schedulePublicExperiments(self, job):
job.markAsRunning()
# Retrieve the configuration templates
for template in Configuration.objects.filter(experiment_type=Configuration.PUBLIC, name__startswith='template/'):
# Retrieve the list of heuristic versions already used in a public experiment
# based on this template
already_used_heuristic_versions = []
previous = None
try:
previous = Configuration.objects.filter(experiment_type=Configuration.PUBLIC,
name__startswith=template.name.replace('template/', '')).order_by('-experiment__creation_date')[0]
if previous.experiment.status == Experiment.STATUS_RUNNING:
continue
already_used_heuristic_versions = list(previous.heuristics_set.all())
except:
pass
# Ensure that some new heuristics exist since the last public experiment
# based on this template
current_heuristic_versions = map(lambda x: x.latest_public_version,
list(Heuristic.objects.filter(latest_public_version__isnull=False, simple=False)))
new_heuristic_versions = filter(lambda x: x not in already_used_heuristic_versions, current_heuristic_versions)
if len(new_heuristic_versions) == 0:
continue
# Cancel the previous experiment if needed
if (previous is not None) and (previous.experiment.status == Experiment.STATUS_SCHEDULED):
self.channel.sendMessage(Message('CANCEL_EXPERIMENT', [previous.experiment.id]))
continue
# Create the configuration
configuration = Configuration()
configuration.name = template.name.replace('template/', '') + '/' + datetime.now().strftime('%Y-%m-%d %H:%M:%S')
configuration.heuristics = template.heuristics
configuration.experiment_type = template.experiment_type
configuration.task = template.task
configuration.save()
# Add the list of heuristic versions
for heuristic_version in current_heuristic_versions:
configuration.heuristics_set.add(heuristic_version)
# Add the list of instruments
for instrument in template.instruments_set.all():
configuration.instruments_set.add(instrument)
# Save the configuration
configuration.save()
# Add the settings
for setting in template.settings.all():
configuration.addSetting(setting.name, setting.value)
# Create the experiment
experiment = Experiment()
experiment.name = configuration.name
experiment.configuration = configuration
experiment.save()
self.channel.sendMessage(Message('RUN_EXPERIMENT', [experiment.id]))
job.markAsDone()
class WebUserTests:
def setUp(self):
self.experiment = Experiment(name=EXPERIMENT_NAME, state=ENABLED_STATE)
self.experiment.save()
self.request = request_factory.get('/')
self.request.session = DatabaseSession()
def tearDown(self):
self.experiment.delete()
def confirm_human(self, experiment_user):
pass
def participants(self, alternative):
return self.experiment.participant_count(alternative)
def enrollment_initially_none(self, ):
experiment_user = participant(self.request)
self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME),
None)
def test_user_enrolls(self):
experiment_user = participant(self.request)
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME),
TEST_ALTERNATIVE)
def test_record_goal_increments_counts(self):
experiment_user = participant(self.request)
self.confirm_human(experiment_user)
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
self.assertEqual(
self.experiment.goal_count(TEST_ALTERNATIVE, TEST_GOAL), 0)
experiment_user.goal(TEST_GOAL)
self.assertEqual(
self.experiment.goal_count(TEST_ALTERNATIVE, TEST_GOAL), 1)
def test_can_record_goal_multiple_times(self):
experiment_user = participant(self.request)
self.confirm_human(experiment_user)
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
experiment_user.goal(TEST_GOAL)
experiment_user.goal(TEST_GOAL)
experiment_user.goal(TEST_GOAL)
self.assertEqual(
self.experiment.goal_count(TEST_ALTERNATIVE, TEST_GOAL), 1)
def test_counts_increment_immediately_once_confirmed_human(self):
experiment_user = participant(self.request)
self.confirm_human(experiment_user)
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
self.assertEqual(self.participants(TEST_ALTERNATIVE), 1,
"Did not count participant after confirm human")
def test_visit_increases_goal(self):
experiment_user = participant(self.request)
self.confirm_human(experiment_user)
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
experiment_user.visit()
self.assertEqual(
self.experiment.goal_distribution(TEST_ALTERNATIVE,
VISIT_COUNT_GOAL), {1: 1})
def test_visit_twice_increases_once(self):
experiment_user = participant(self.request)
self.confirm_human(experiment_user)
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
experiment_user.visit()
experiment_user.visit()
self.assertEqual(
self.experiment.goal_distribution(TEST_ALTERNATIVE,
VISIT_COUNT_GOAL), {1: 1})
class WebUserTests(object):
def setUp(self):
self.experiment = Experiment(name=EXPERIMENT_NAME, state=ENABLED_STATE)
self.experiment.save()
self.request = request_factory.get('/')
self.request.session = DatabaseSession()
self.experiment_counter = ExperimentCounter()
def tearDown(self):
self.experiment_counter.delete(self.experiment)
def test_enrollment_initially_control(self):
experiment_user = participant(self.request)
self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME),
'control', "Default Enrollment wasn't control")
def test_user_enrolls(self):
experiment_user = participant(self.request)
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME),
TEST_ALTERNATIVE, "Wrong Alternative Set")
def test_record_goal_increments_counts(self):
experiment_user = participant(self.request)
experiment_user.confirm_human()
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
self.assertEqual(
self.experiment_counter.goal_count(self.experiment,
TEST_ALTERNATIVE, TEST_GOAL), 0)
experiment_user.goal(TEST_GOAL)
self.assertEqual(
self.experiment_counter.goal_count(self.experiment,
TEST_ALTERNATIVE, TEST_GOAL), 1,
"Did not increment Goal count")
def test_can_record_goal_multiple_times(self):
experiment_user = participant(self.request)
experiment_user.confirm_human()
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
experiment_user.goal(TEST_GOAL)
experiment_user.goal(TEST_GOAL)
experiment_user.goal(TEST_GOAL)
self.assertEqual(
self.experiment_counter.goal_count(self.experiment,
TEST_ALTERNATIVE, TEST_GOAL), 1,
"Did not increment goal count correctly")
self.assertEqual(
self.experiment_counter.goal_distribution(self.experiment,
TEST_ALTERNATIVE,
TEST_GOAL), {3: 1},
"Incorrect goal count distribution")
def test_counts_increment_immediately_once_confirmed_human(self):
experiment_user = participant(self.request)
experiment_user.confirm_human()
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
self.assertEqual(
self.experiment_counter.participant_count(self.experiment,
TEST_ALTERNATIVE), 1,
"Did not count participant after confirm human")
def test_visit_increases_goal(self):
thetime = timezone.now()
with mock.patch('experiments.utils.now', return_value=thetime):
experiment_user = participant(self.request)
experiment_user.confirm_human()
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
experiment_user.visit()
self.assertEqual(
self.experiment_counter.goal_distribution(
self.experiment, TEST_ALTERNATIVE,
VISIT_NOT_PRESENT_COUNT_GOAL), {1: 1},
"Not Present Visit was not correctly counted")
self.assertEqual(
self.experiment_counter.goal_distribution(
self.experiment, TEST_ALTERNATIVE,
VISIT_PRESENT_COUNT_GOAL), {},
"Present Visit was not correctly counted")
with mock.patch('experiments.utils.now',
return_value=thetime + timedelta(hours=7)):
experiment_user.visit()
self.assertEqual(
self.experiment_counter.goal_distribution(
self.experiment, TEST_ALTERNATIVE,
VISIT_NOT_PRESENT_COUNT_GOAL), {2: 1},
"No Present Visit was not correctly counted")
self.assertEqual(
self.experiment_counter.goal_distribution(
self.experiment, TEST_ALTERNATIVE,
VISIT_PRESENT_COUNT_GOAL), {1: 1},
"Present Visit was not correctly counted")
def test_visit_twice_increases_once(self):
experiment_user = participant(self.request)
experiment_user.confirm_human()
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
experiment_user.visit()
experiment_user.visit()
self.assertEqual(
self.experiment_counter.goal_distribution(
self.experiment, TEST_ALTERNATIVE,
VISIT_NOT_PRESENT_COUNT_GOAL), {1: 1},
"Visit was not correctly counted")
self.assertEqual(
self.experiment_counter.goal_distribution(
self.experiment, TEST_ALTERNATIVE, VISIT_PRESENT_COUNT_GOAL),
{}, "Present Visit was not correctly counted")
def test_user_force_enrolls(self):
experiment_user = participant(self.request)
experiment_user.enroll(EXPERIMENT_NAME,
['control', 'alternative1', 'alternative2'],
force_alternative='alternative2')
self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME),
'alternative2')
def test_user_does_not_force_enroll_to_new_alternative(self):
alternatives = ['control', 'alternative1', 'alternative2']
experiment_user = participant(self.request)
experiment_user.enroll(EXPERIMENT_NAME, alternatives)
alternative = experiment_user.get_alternative(EXPERIMENT_NAME)
self.assertIsNotNone(alternative)
other_alternative = random.choice(
list(set(alternatives) - set(alternative)))
experiment_user.enroll(EXPERIMENT_NAME,
alternatives,
force_alternative=other_alternative)
self.assertEqual(alternative,
experiment_user.get_alternative(EXPERIMENT_NAME))
def test_second_force_enroll_does_not_change_alternative(self):
alternatives = ['control', 'alternative1', 'alternative2']
experiment_user = participant(self.request)
experiment_user.enroll(EXPERIMENT_NAME,
alternatives,
force_alternative='alternative1')
alternative = experiment_user.get_alternative(EXPERIMENT_NAME)
self.assertIsNotNone(alternative)
other_alternative = random.choice(
list(set(alternatives) - set(alternative)))
experiment_user.enroll(EXPERIMENT_NAME,
alternatives,
force_alternative=other_alternative)
self.assertEqual(alternative,
experiment_user.get_alternative(EXPERIMENT_NAME))
class WebUserTests(object):
def setUp(self):
self.experiment = Experiment(name=EXPERIMENT_NAME, state=ENABLED_STATE)
self.experiment.save()
self.request = request_factory.get("/")
self.request.session = DatabaseSession()
self.experiment_counter = ExperimentCounter()
def tearDown(self):
self.experiment_counter.delete(self.experiment)
def confirm_human(self, experiment_user):
pass
def participants(self, alternative):
return self.experiment_counter.participant_count(self.experiment, alternative)
def enrollment_initially_none(self,):
experiment_user = participant(self.request)
self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME), None)
def test_user_enrolls(self):
experiment_user = participant(self.request)
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME), TEST_ALTERNATIVE)
def test_record_goal_increments_counts(self):
experiment_user = participant(self.request)
self.confirm_human(experiment_user)
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
self.assertEqual(self.experiment_counter.goal_count(self.experiment, TEST_ALTERNATIVE, TEST_GOAL), 0)
experiment_user.goal(TEST_GOAL)
self.assertEqual(self.experiment_counter.goal_count(self.experiment, TEST_ALTERNATIVE, TEST_GOAL), 1)
def test_can_record_goal_multiple_times(self):
experiment_user = participant(self.request)
self.confirm_human(experiment_user)
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
experiment_user.goal(TEST_GOAL)
experiment_user.goal(TEST_GOAL)
experiment_user.goal(TEST_GOAL)
self.assertEqual(self.experiment_counter.goal_count(self.experiment, TEST_ALTERNATIVE, TEST_GOAL), 1)
def test_counts_increment_immediately_once_confirmed_human(self):
experiment_user = participant(self.request)
self.confirm_human(experiment_user)
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
self.assertEqual(self.participants(TEST_ALTERNATIVE), 1, "Did not count participant after confirm human")
def test_visit_increases_goal(self):
experiment_user = participant(self.request)
self.confirm_human(experiment_user)
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
experiment_user.visit()
self.assertEqual(
self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, VISIT_COUNT_GOAL), {1: 1}
)
def test_visit_twice_increases_once(self):
experiment_user = participant(self.request)
self.confirm_human(experiment_user)
experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE)
experiment_user.visit()
experiment_user.visit()
self.assertEqual(
self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, VISIT_COUNT_GOAL), {1: 1}
)
