def stop_experiment(project, experiment_id):
"""
Get an experiment.
:param project: The project
:param experiment_id: The id of the experiment
:return: (start, end_date)
"""
service = _get_service(project)
#get current state, change name to current name appended with end date
current = get_experiment(project, experiment_id)
if current['status'] != 'ENDED':
#get start time from name
start = datetime.strptime(current['name'], 'IGA - %Y-%m-%d %H:%M:%S')
end_str = datetime.now(project_setting(
project, 'time_zone')).strftime('%Y-%m-%d %H:%M:%S')
# remove timezone and nanoseconds as we also did in the start
end = datetime.strptime(end_str, '%Y-%m-%d %H:%M:%S')
current['name'] = current['name'] + " - " + end_str
current['status'] = 'ENDED'
s = service.management().experiments().update(
accountId=project_setting(project, 'ga_account'),
webPropertyId=project_setting(project, 'ga_property'),
profileId=project_setting(project, 'ga_profile'),
experimentId=experiment_id,
body=current).execute()
else:
dates = current['name'].split(" - ")
start = datetime.strptime(dates[1], '%Y-%m-%d %H:%M:%S')
end = datetime.strptime(dates[2], '%Y-%m-%d %H:%M:%S')
vars = [c['name'] for c in current['variations']]
return vars, start, end
def __init__(self, project):
self.project = project
self.db = MySQLdb.connect(
host=project_setting(self.project, 'db_host'),
user=project_setting(self.project, 'db_user'),
passwd=project_setting(self.project, 'db_password'),
db=project_setting(self.project, 'db_db')
)
def list_experiments(project):
"""
List the experiments.
:param project: Project to list the experiments for.
:return: A list consisting of all the experiments.
"""
service = _get_service(project)
s = service.management().experiments().list(
accountId=project_setting(project, 'ga_account'),
webPropertyId=project_setting(project, 'ga_property'),
profileId=project_setting(project, 'ga_profile')).execute()
return s.get('items')
def _make_initial_population(self):
pop_size = project_setting(self.project, 'population_size')
start_code = project_setting(self.project, 'start_code')
vars = search_space(self.project)
# start code is always in population
pop = [start_code]
# rest are randomly generated samples
for i in xrange(0, pop_size):
ind = []
for v in vars:
ind.append(str(np.random.choice(vars[v], 1)[0]))
pop.append("-".join(ind))
self._new_population(pop)
def experiment(project):
f = get_fitness(project)
e = f.get_current_experiment()
prefix = project_setting(project, 'prefix')
body = 'function getQueryVariable(variable){var query = window.location.search.substring(1);var vars = query.split("&");for (var i=0;i<vars.length;i++) {var pair = vars[i].split("=");if(pair[0] == variable){return pair[1];}}return(false);}' \
'var code = getQueryVariable(\'iga-code\');console.log(code);' \
'if (code === false) {'
if e is None:
body += 'console.log("[iga] Currently no active experiments");'
else:
vars = json.dumps(e['variations']).replace('"', '\\"')
body += 'document.write("<sc"+"ript src=\'http" + (document.location.protocol == \'https:\' ? \'s://ssl\' : \'://www\') + ".google-analytics.com/cx/api.js?experiment=' + \
e['experiment_id'] + '\'><\\/script>");' \
'document.write("<script>var chosenVariation = cxApi.chooseVariation(); var variations =' + vars + '; var code = variations[chosenVariation]; ' \
'document.write(\\"<sc\\"+\\"ript src=\'' + prefix + 'code-\\"+code+\\".js\'></scri\\"+\\"pt>\\");</scri"+"pt>");'
body += '} else {' \
'document.write("<sc"+"ript src=\'' + prefix + 'code-"+code+".js\'></scri"+"pt>");}'
return Response(body,
status=200,
mimetype="application/javascript",
headers={
'Access-Control-Allow-Origin': '*',
'Cache-Control': 'max-age=3600'
})
def get_experiment(project, experiment_id):
"""
Get an experiment.
:param project: The project
:param experiment_id: The id of the experiment
:return: Experiment object
"""
service = _get_service(project)
s = service.management().experiments().get(
accountId=project_setting(project, 'ga_account'),
webPropertyId=project_setting(project, 'ga_property'),
profileId=project_setting(project, 'ga_profile'),
experimentId=experiment_id).execute()
return s
def remove_experiment(project, experiment_id):
"""
Remove an experiment.
:param project: The project
:param experiment_id: The id of the experiment
:return: Resource object
"""
service = _get_service(project)
s = service.management().experiments().delete(
accountId=project_setting(project, 'ga_account'),
webPropertyId=project_setting(project, 'ga_property'),
profileId=project_setting(project, 'ga_profile'),
experimentId=experiment_id).execute()
return s
def run(project):
sa = project_setting(project, 'search_algorithm')
if sa == 'IGA':
iga = get_IGA(project)
iga.run()
elif sa == 'BF':
bf = get_bf(project)
bf.run()
else:
raise Exception("No valid search algorithm found %s" % sa)
def start_experiments(project, variations):
"""
Start experiments.
:param project: The project for which the experiments should start.
:param variations: The variations of the experiments (list of variation names).
:return: Experiment Id.
"""
service = _get_service(project)
start_code = project_setting(project, 'start_code')
base_url = project_setting(project, 'example_url')
body = {
'name':
'IGA - ' + datetime.now(project_setting(
project, 'time_zone')).strftime('%Y-%m-%d %H:%M:%S'),
'variations': [{
'name':
v,
'url': (base_url if start_code == v else '?iga-code=' + v)
} for v in variations],
'servingFramework':
'API',
'objectiveMetric':
'ga:pageviews',
'equalWeighting':
True,
'status':
'RUNNING'
}
return service.management().experiments().insert(
accountId=project_setting(project, 'ga_account'),
webPropertyId=project_setting(project, 'ga_property'),
profileId=project_setting(project, 'ga_profile'),
body=body).execute()['id']
def _new_experiment(self, variations=[]):
# everything is cached, we don't want to send something to analytics now..
if variations is []:
pass
else:
# default variation is always present and the first
default_population = project_setting(self.project, 'start_code')
if default_population in variations:
variations.remove(default_population)
variations.insert(0, default_population)
# start experiment in GA
ex_id = start_experiments(self.project, variations)
self._save_current_experiment({"experiment_id":ex_id, "variations":variations})
def get_fitness(self, population=[]):
"""
Call this function when you want to retrieve the results of a population
:param population:
:return:
"""
if False:
raise Exception("Fatal get_fitness called while population not valid")
else:
default_population = project_setting(self.project, 'start_code')
hits = self.db.try_to_find_fitness_for(population)
# Calculate part of population that is not yet cached and should therefore come from ga
variations = [p for p in population if p not in hits.keys()]
# when we need data from google analytics
if len(variations) > 0:
log = {}
ex = self.get_current_experiment()
# retrieve fitness we need from analytics
score = get_experiment_score(self.project, ex['experiment_id'])
experiment_vars = ex['variations']
if score.shape[1] < 1:
self.db.delay_exp(ex['id'], 3600)
else:
base = 0.0
for i in range(score.shape[1]):
if experiment_vars[i] == default_population:
base = score[1,i]
for i in range(score.shape[1]):
v = experiment_vars[i]
if v in variations:
variations.remove(v)
hits[v] = self._calc_fitness(score[1,i], base)
log[v] = (hits[v], score[1,i], int(score[0,i]))
# log all results
self.save_ga(ex['id'], log, base)
# make sure to return in correct order
return [hits[p] for p in population]
else:
return [hits[p] for p in population]
return False
def _save_current_experiment(self, ex):
pop = ex['variations']
time_per = project_setting(self.project, 'evaluation_time_per_individual')
id = self.db.save_ga_experiment(ex['experiment_id'],86300)
self.db.save_ga_population(id, pop)
def _s3_connect(project):
return boto3.resource('s3').Bucket(project_setting(project, 's3_bucket'))
def _list_s3(project):
return boto3.client('s3').list_objects(
Bucket=project_setting(project, 's3_bucket'))
def _get_experiment_data(project,
experiment_id,
metrics='ga:pageviews, ga:exitRate',
start_date='30daysAgo',
end_date='today'):
"""
Get data for an experiment.
:param project: The project to get the data for.
:param experiment_id: The ID of the experiment.
:param metrics: The metrics which should be retrieved.
:param start_date: The start date for which to include data for.
:param end_date: The end date for which to include data for.
:return:
"""
ids = 'ga:%s' % project_setting(project, 'ga_profile')
filters = 'ga:experimentId==%s' % experiment_id
service = _get_service(project)
s = service.data().ga().get(ids=ids,
start_date=start_date,
end_date=end_date,
metrics=metrics,
dimensions='ga:experimentVariant',
filters=filters).execute()
# Modify the rows such that it is possible to do computations on the numbers
data = {}
column_names = ['index'
] + [metric.strip() for metric in metrics.split(',')]
for column in column_names:
data[column] = []
if 'rows' in s:
for row in s['rows']:
for index in range(len(row)):
value = row[index]
value = int(value) if index == 0 else float(value)
column = column_names[index]
data[column].append(value)
s['data'] = data
# Ugly: bounceRate is percentage (not ratio), so convert it to ratios
if 'ga:bounceRate' in s['data']:
s['data']['ga:bounceRate'] = [
item / 100. for item in s['data']['ga:bounceRate']
]
# Make sessions ints
if 'ga:sessions' in s['data']:
s['data']['ga:sessions'] = [
int(item) for item in s['data']['ga:sessions']
]
# Ugly: exitRate is percentage (not ratio), so convert it to ratios
if 'ga:exitRate' in s['data']:
s['data']['ga:exitRate'] = [
item / 100. for item in s['data']['ga:exitRate']
]
# Make sessions ints
if 'ga:pageviews' in s['data']:
s['data']['ga:pageviews'] = [
int(item) for item in s['data']['ga:pageviews']
]
# stop & log
vars, start, stop = stop_experiment(project, experiment_id)
totals = s['totalsForAllResults']
#_log_experiment_results(project, experiment_id, start.isoformat(), stop.isoformat(), totals['ga:pageviews'], totals['ga:exitRate'], vars, s['data'])
return s