def experiment_dashboard(exp_uid, app_id, exp_key):
"""
Endpoint that renders the experiment dashboard.
Inputs: ::\n
(string) exp_uid, exp_uid for a current experiment.
"""
# Not a particularly good way to do this.
alg_label_list = rm.get_algs_for_exp_uid(exp_uid)
exp_start_data = rm.get_app_exp_uid_start_date(exp_uid) + ' UTC'
participant_uids = rm.get_participant_uids(exp_uid)
num_participants = len(participant_uids)
last_datetime = None
num_queries = 0
for participant_uid in participant_uids:
queries = rm.get_participant_data(participant_uid, exp_uid)
num_queries += len(queries)
this_datetime = utils.str2datetime(
queries[-1]['timestamp_query_generated'])
if last_datetime == None or this_datetime > last_datetime:
last_datetime = this_datetime
last_activity = utils.datetime2str(last_datetime) + ' UTC'
# Migrate this code to use keychain
docs, didSucceed, message = db.getDocsByPattern('next_frontend_base',
'keys', {
'object_id': exp_uid,
'type': 'perm'
})
perm_key = docs[0]['_id']
alg_list = [{
'alg_label': alg['alg_label'],
'alg_label_clean': '_'.join(alg['alg_label'].split())
} for alg in alg_label_list]
if (constants.NEXT_BACKEND_GLOBAL_HOST
and constants.NEXT_BACKEND_GLOBAL_PORT):
host_url = 'http://{}:{}'.format(constants.NEXT_BACKEND_GLOBAL_HOST,
constants.NEXT_BACKEND_GLOBAL_PORT)
else:
host_url = ''
print 'host_url', host_url
print constants.NEXT_BACKEND_GLOBAL_HOST, constants.NEXT_BACKEND_GLOBAL_PORT
env = Environment(loader=ChoiceLoader([
PackageLoader('next.apps.{}'.format(app_id), 'dashboard'),
PackageLoader('next.dashboard', 'templates')
]))
template = env.get_template('{}.html'.format(app_id))
return template.render(app_id=app_id,
exp_uid=exp_uid,
alg_list=alg_list,
host_url=host_url,
perm_key=perm_key,
url_for=url_for,
exp_start_data=exp_start_data,
num_participants=num_participants,
num_queries=num_queries,
last_activity=last_activity)
def refresh_domain_hashes(self):
# see what workers are out there
worker_pings = None
while worker_pings == None:
try:
# one could also use app.control.inspect().active_queues()
worker_pings = app.control.inspect().ping()
except:
worker_pings = None
worker_names = worker_pings.keys()
domain_names_with_dups = [item.split('@')[1] for item in worker_names]
domain_names = list(set(domain_names_with_dups)) # remove duplicates!
timestamp = utils.datetime2str(utils.datetimeNow())
print "[ %s ] domains with active workers = %s" % (timestamp,
str(domain_names))
replicated_domains_hash = []
for domain in domain_names:
for i in range(self.num_replicas):
replicated_domains_hash.append(
(domain, self.hash(domain + '_replica_' + str(i)), i))
replicated_domains_hash = sorted(replicated_domains_hash,
key=lambda x: x[1])
self.r.set('replicated_domains_hash',
json.dumps(replicated_domains_hash))
def initExp(self, exp_uid, args_json):
try:
self.helper.ensure_indices(self.app_id,self.butler.db, self.butler.ell)
args_dict = self.helper.convert_json(args_json)
args_dict = verifier.verify(args_dict, self.reference_dict['initExp']['args'])
args_dict['exp_uid'] = exp_uid # to get doc from db
args_dict['start_date'] = utils.datetime2str(utils.datetimeNow())
self.butler.admin.set(uid=exp_uid,value={'exp_uid': exp_uid, 'app_id':self.app_id, 'start_date':str(utils.datetimeNow())})
utils.debug_print("ASD "+str(args_dict))
args_dict['args'] = self.init_app(exp_uid, args_dict['args']['alg_list'], args_dict['args'])
args_dict['git_hash'] = git_hash
self.butler.experiment.set(value=args_dict)
return '{}', True, ''
except Exception, error:
exc_type, exc_value, exc_traceback = sys.exc_info()
full_error = str(traceback.format_exc())+'\n'+str(error)
utils.debug_print("initExp Exception: " + full_error, color='red')
log_entry = { 'exp_uid':exp_uid,'task':'initExp','error':full_error,'timestamp':utils.datetimeNow(),'args_json':args_json }
self.butler.ell.log( self.app_id+':APP-EXCEPTION', log_entry )
traceback.print_tb(exc_traceback)
return '{}', False, str(error)
def initExp(self, exp_uid, args_json):
try:
self.helper.ensure_indices(self.app_id,self.butler.db, self.butler.ell)
args_dict = self.helper.convert_json(args_json)
args_dict = verifier.verify(args_dict, self.reference_dict['initExp']['args'])
args_dict['exp_uid'] = exp_uid # to get doc from db
args_dict['start_date'] = utils.datetime2str(utils.datetimeNow())
self.butler.admin.set(uid=exp_uid,value={'exp_uid': exp_uid, 'app_id':self.app_id, 'start_date':str(utils.datetimeNow())})
self.butler.experiment.set(value={'exp_uid': exp_uid})
args_dict['args'] = self.init_app(exp_uid, args_dict['args']['alg_list'], args_dict['args'])
args_dict['git_hash'] = git_hash
self.butler.experiment.set_many(key_value_dict=args_dict)
return '{}', True, ''
except Exception, error:
exc_type, exc_value, exc_traceback = sys.exc_info()
full_error = str(traceback.format_exc())+'\n'+str(error)
utils.debug_print("initExp Exception: " + full_error, color='red')
log_entry = { 'exp_uid':exp_uid,'task':'initExp','error':full_error,'timestamp':utils.datetimeNow(),'args_json':args_json }
self.butler.ell.log( self.app_id+':APP-EXCEPTION', log_entry )
traceback.print_tb(exc_traceback)
return '{}', False, str(error)
def refresh_domain_hashes(self):
# see what workers are out there
worker_pings = None
while worker_pings==None:
try:
# one could also use app.control.inspect().active_queues()
worker_pings = app.control.inspect().ping()
except:
worker_pings = None
worker_names = worker_pings.keys()
domain_names_with_dups = [ item.split('@')[1] for item in worker_names]
domain_names = list(set(domain_names_with_dups)) # remove duplicates!
timestamp = utils.datetime2str(utils.datetimeNow())
print "[ %s ] domains with active workers = %s" % (timestamp,str(domain_names))
replicated_domains_hash = []
for domain in domain_names:
for i in range(self.num_replicas):
replicated_domains_hash.append((domain, self.hash(domain+'_replica_'+str(i)), i))
replicated_domains_hash = sorted( replicated_domains_hash, key = lambda x: x[1])
self.r.set('replicated_domains_hash',json.dumps(replicated_domains_hash))
def initExp(self,exp_uid,args_json,db,ell):
"""
initialize the project and necessary experiments
Expected input (in json structure with string keys):
(int) n: number of arms
(int) k: number of objects to display
(float) failure_probability : confidence
[optional] (list of dicts) alg_list : with fields (Defaults given by Info.get_app_default_alg_list)
(string) alg_id : valid alg_id for this app_id
(string) alg_label : unique identifier for algorithm (e.g. may have experiment with repeated alg_id's, but alg_labels must be unqiue, will also be used for plot legends
[optional] (string) test_alg_label : must be one of the alg_label's in alg_list (Default is self)
[optional] (dict) algorithm_management_settings : dictionary with fields (string) 'mode' and (dict) 'params'. mode in {'pure_exploration','explore_exploit','fixed_proportions'}. Default is 'fixed_proportions' and allocates uniform probability to each algorithm. If mode=fixed_proportions then params is a dictionary that contains the field 'proportions' which is a list of dictionaries with fields 'alg_label' and 'proportion' for all algorithms in alg_list. All proportions must be positive and sum to 1 over all algs in alg_list
[optional] (string) participant_to_algorithm_management : in {'one_to_one','one_to_many'}. Default is 'one_to_many'.
[optional] (string) instructions
[optional] (string) debrief
Expected output:
if error:
return (JSON) '{}', (bool) False, (str) error_str
else:
return (JSON) '{}', (bool) True,''
Usage:
initExp_response_json,didSucceed,message = app.initExp(exp_uid,initExp_args_json)
Example input:
initExp_args_json = {"participant_to_algorithm_management": "one_to_many", "alg_list": [{"alg_label": "BR_LilUCB", "alg_id": "BR_LilUCB", "params": {}}], "algorithm_management_settings": {"params": {"proportions": [{"alg_label": "BR_LilUCB", "proportion": 1.0}]}, "mode": "fixed_proportions"}, "failure_probability": 0.01, "n": 10}
Example output:
initExp_response_json = {}
"""
try:
app_id = self.app_id
# remove any reminants of an experiment if it exists
didSucceed,message = db.delete_docs_with_filter('experiments_admin',{'exp_uid':exp_uid})
didSucceed,message = db.delete_docs_with_filter(app_id+':experiments',{'exp_uid':exp_uid})
didSucceed,message = db.delete_docs_with_filter(app_id+':queries',{'exp_uid':exp_uid})
didSucceed,message = db.delete_docs_with_filter(app_id+':participants',{'exp_uid':exp_uid})
didSucceed,message = db.delete_docs_with_filter(app_id+':algorithms',{'exp_uid':exp_uid})
didSucceed,message = ell.delete_logs_with_filter(app_id+':APP-CALL',{'exp_uid':exp_uid})
didSucceed,message = ell.delete_logs_with_filter(app_id+':APP-RESPONSE',{'exp_uid':exp_uid})
didSucceed,message = ell.delete_logs_with_filter(app_id+':APP-EXCEPTION',{'exp_uid':exp_uid})
didSucceed,message = ell.delete_logs_with_filter(app_id+':ALG-DURATION',{'exp_uid':exp_uid})
didSucceed,message = ell.delete_logs_with_filter(app_id+':ALG-EVALUATION',{'exp_uid':exp_uid})
# add indexes (only adds them if they do not already exist)
didSucceed,message = db.ensure_index('experiments_admin',{'exp_uid':1})
didSucceed,message = db.ensure_index(app_id+':experiments',{'exp_uid':1})
didSucceed,message = db.ensure_index(app_id+':queries',{'query_uid':1})
didSucceed,message = db.ensure_index(app_id+':queries',{'exp_uid':1})
didSucceed,message = db.ensure_index(app_id+':queries',{'alg_uid':1})
didSucceed,message = db.ensure_index(app_id+':queries',{'participant_uid':1})
didSucceed,message = db.ensure_index(app_id+':participants',{'participant_uid':1})
didSucceed,message = db.ensure_index(app_id+':participants',{'exp_uid':1})
didSucceed,message = db.ensure_index(app_id+':algorithms',{'alg_uid':1})
didSucceed,message = db.ensure_index(app_id+':algorithms',{'exp_uid':1})
didSucceed,message = ell.ensure_index(app_id+':APP-CALL',{'exp_uid':1})
didSucceed,message = ell.ensure_index(app_id+':APP-CALL',{'timestamp':1})
didSucceed,message = ell.ensure_index(app_id+':APP-CALL',{'exp_uid':1,'timestamp':1})
didSucceed,message = ell.ensure_index(app_id+':APP-CALL',{'exp_uid':1,'task':1})
didSucceed,message = ell.ensure_index(app_id+':APP-RESPONSE',{'exp_uid':1})
didSucceed,message = ell.ensure_index(app_id+':APP-RESPONSE',{'timestamp':1})
didSucceed,message = ell.ensure_index(app_id+':APP-RESPONSE',{'exp_uid':1,'timestamp':1})
didSucceed,message = ell.ensure_index(app_id+':APP-RESPONSE',{'exp_uid':1,'task':1})
didSucceed,message = ell.ensure_index(app_id+':APP-EXCEPTION',{'exp_uid':1})
didSucceed,message = ell.ensure_index(app_id+':APP-EXCEPTION',{'timestamp':1})
didSucceed,message = ell.ensure_index(app_id+':APP-EXCEPTION',{'exp_uid':1,'timestamp':1})
didSucceed,message = ell.ensure_index(app_id+':APP-EXCEPTION',{'exp_uid':1,'task':1})
didSucceed,message = ell.ensure_index(app_id+':ALG-DURATION',{'exp_uid':1})
didSucceed,message = ell.ensure_index(app_id+':ALG-DURATION',{'alg_uid':1})
didSucceed,message = ell.ensure_index(app_id+':ALG-DURATION',{'timestamp':1})
didSucceed,message = ell.ensure_index(app_id+':ALG-DURATION',{'exp_uid':1,'timestamp':1})
didSucceed,message = ell.ensure_index(app_id+':ALG-DURATION',{'alg_uid':1,'task':1})
didSucceed,message = ell.ensure_index(app_id+':ALG-EVALUATION',{'exp_uid':1})
didSucceed,message = ell.ensure_index(app_id+':ALG-EVALUATION',{'alg_uid':1})
didSucceed,message = ell.ensure_index(app_id+':ALG-EVALUATION',{'timestamp':1})
didSucceed,message = ell.ensure_index(app_id+':ALG-EVALUATION',{'exp_uid':1,'timestamp':1})
db.set('experiments_admin',exp_uid,'exp_uid',exp_uid)
db.set('experiments_admin',exp_uid,'app_id',app_id)
db.set('experiments_admin',exp_uid,'start_date',utils.datetime2str(utils.datetimeNow()))
log_entry = { 'exp_uid':exp_uid,'task':'initExp','json':args_json,'timestamp':utils.datetimeNow() }
ell.log( app_id+':APP-CALL', log_entry )
# convert args_json to args_dict
try:
args_dict = json.loads(args_json)
except:
error = "%s.initExp input args_json is in improper format" % self.app_id
return '{}',False,error
# check for the fields that must be contained in args or error occurs
necessary_fields = ['n','k','failure_probability']
for field in necessary_fields:
try:
args_dict[field]
except KeyError:
error = "%s.initExp input arguments missing field: %s" % (self.app_id,str(field))
return '{}',False,error
n = args_dict['n']
k = args_dict['k']
delta = args_dict['failure_probability']
if 'alg_list' in args_dict:
alg_list = args_dict['alg_list']
supportedAlgs = utils.get_app_supported_algs(self.app_id)
for algorithm in alg_list:
if algorithm['alg_id'] not in supportedAlgs:
error = "%s.initExp unsupported algorithm '%s' in alg_list" % (self.app_id,alg_id)
return '{}',False,error
else:
alg_list = utils.get_app_default_alg_list(self.app_id)
if 'instructions' not in args_dict:
instructions = utils.get_app_default_instructions(app_id)
else:
instructions = args_dict['instructions']
if 'debrief' not in args_dict:
debrief = utils.get_app_default_instructions(app_id)
else:
debrief = args_dict['debrief']
if 'num_tries' not in args_dict:
num_tries = utils.get_app_default_num_tries(app_id)
else:
num_tries = args_dict['num_tries']
if 'context_type' not in args_dict:
context_type = 'none'
else:
context_type = args_dict['context_type']
if 'context' not in args_dict:
context = ''
else:
context = args_dict['context']
# ALGORITHM_MANAGEMENT_MODE FORMATTING CHECK
if 'algorithm_management_settings' not in args_dict:
params = {}
params['proportions'] = []
for algorithm in alg_list:
params['proportions'].append( { 'alg_label': algorithm['alg_label'] , 'proportion':1./len(alg_list)} )
algorithm_management_settings = {}
algorithm_management_settings['mode'] = 'fixed_proportions'
algorithm_management_settings['params'] = params
else:
algorithm_management_settings = args_dict['algorithm_management_settings']
try:
mode = algorithm_management_settings['mode']
params = algorithm_management_settings['params']
except:
error = "%s.initExp algorithm_management_settings must be a dictionary with fields 'mode' and 'params'" % (self.app_id)
return '{}',False,error
if mode == 'fixed_proportions':
try:
algorithm_proportions_list = params['proportions']
except:
error = "%s.initExp algorithm_management_settings['params'] must be a dictionary with field 'proportions'" % (self.app_id)
return '{}',False,error
# check if alg_labels are properly labeled
for proportion_item in algorithm_proportions_list:
proportion = proportion_item['proportion']
target_alg_label = proportion_item['alg_label']
target_alg_label_in_alg_list = False
for algorithm in alg_list:
if algorithm['alg_label']==target_alg_label:
target_alg_label_in_alg_list = True
if not target_alg_label_in_alg_list:
error = "%s.initExp algorithm_management_settings['params']['proportions'] must be a list of dictionaries, each dictionary containing the fields 'alg_label' and 'proportion'. The 'alg_label' value must be one of the alg_labels in a provided alg_list and 'proportion' must be nonnegative and sum to 1 : '%s' not in provided alg_list" % (self.app_id,target_alg_label)
return '{}',False,error
elif mode == 'pure_exploration':
error = "%s.initExp Sorry, '%s' is not yet supported." % (self.app_id,mode)
return '{}',False,error
elif mode == 'explore_exploit':
error = "%s.initExp Sorry, '%s' is not yet supported." % (self.app_id,mode)
return '{}',False,error
else:
error = "%s.initExp unsupported algorithm_management_mode: '%s'. Must be in {'pure_exploration','explore_exploit','fixed_proportions'}" % (self.app_id,algorithm_management_mode)
return '{}',False,error
# ALGORITHM_MANAGEMENT_MODE FORMATTING CHECK
if 'participant_to_algorithm_management' not in args_dict:
participant_to_algorithm_management = 'one_to_many'
else:
participant_to_algorithm_management = args_dict['participant_to_algorithm_management']
if participant_to_algorithm_management not in ['one_to_many','one_to_one']:
error = "%s.initExp unsupported participant_to_algorithm_management: '%s'. Must be in {'one_to_many','one_to_one'}" % (self.app_id,participant_to_algorithm_management)
return '{}',False,error
# assign uid to each algorithm and save it
for algorithm in alg_list:
alg_uid = utils.getNewUID()
algorithm['alg_uid'] = alg_uid
db.set(app_id+':algorithms',alg_uid,'alg_uid',alg_uid)
db.set(app_id+':algorithms',alg_uid,'exp_uid',exp_uid)
db.set(app_id+':experiments',exp_uid,'exp_uid',exp_uid)
db.set(app_id+':experiments',exp_uid,'app_id',app_id)
db.set(app_id+':experiments',exp_uid,'n',n)
db.set(app_id+':experiments',exp_uid,'k',k)
db.set(app_id+':experiments',exp_uid,'failure_probability',delta)
db.set(app_id+':experiments',exp_uid,'alg_list',alg_list)
db.set(app_id+':experiments',exp_uid,'algorithm_management_settings',algorithm_management_settings)
db.set(app_id+':experiments',exp_uid,'participant_to_algorithm_management',participant_to_algorithm_management)
db.set(app_id+':experiments',exp_uid,'instructions',instructions)
db.set(app_id+':experiments',exp_uid,'debrief',debrief)
db.set(app_id+':experiments',exp_uid,'context_type',context_type)
db.set(app_id+':experiments',exp_uid,'context',context)
db.set(app_id+':experiments',exp_uid,'num_tries',num_tries)
# now create intitialize each algorithm
for algorithm in alg_list:
alg_id = algorithm['alg_id']
alg_uid = algorithm['alg_uid']
db.set(app_id+':algorithms',alg_uid,'alg_id',alg_id)
db.set(app_id+':algorithms',alg_uid,'alg_uid',alg_uid)
db.set(app_id+':algorithms',alg_uid,'exp_uid',exp_uid)
# get sandboxed database for the specific app_id,alg_id,exp_uid - closing off the rest of the database to the algorithm
rc = ResourceClient(app_id,exp_uid,alg_uid,db)
# get specific algorithm to make calls to
alg = utils.get_app_alg(self.app_id,alg_id)
# call initExp
didSucceed,dt = utils.timeit(alg.initExp)(resource=rc,n=n,k=k,failure_probability=delta)
log_entry = { 'exp_uid':exp_uid,'alg_uid':alg_uid,'task':'initExp','duration':dt,'timestamp':utils.datetimeNow() }
ell.log( app_id+':ALG-DURATION', log_entry )
response_json = '{}'
log_entry = { 'exp_uid':exp_uid,'task':'initExp','json':response_json,'timestamp':utils.datetimeNow() }
ell.log( app_id+':APP-RESPONSE', log_entry )
return response_json,True,''
except Exception, err:
error = traceback.format_exc()
log_entry = { 'exp_uid':exp_uid,'task':'initExp','error':error,'timestamp':utils.datetimeNow(),'args_json':args_json }
ell.log( app_id+':APP-EXCEPTION', log_entry )
return '{}',False,error
def initExp(self, exp_uid, args_json, db, ell):
"""
initialize the project and necessary experiments
Expected input (in json structure with string keys):
(int) n: number of arms
(float) failure_probability : confidence
[optional] (list of dicts) alg_list : with fields (Defaults given by Info.get_app_default_alg_list)
(string) alg_id : valid alg_id for this app_id
(string) alg_label : unique identifier for algorithm (e.g. may have experiment with repeated alg_id's, but alg_labels must be unqiue, will also be used for plot legends
[optional] (string) test_alg_label : must be one of the alg_label's in alg_list (Default is self)
[optional] (dict) algorithm_management_settings : dictionary with fields (string) 'mode' and (dict) 'params'. mode in {'pure_exploration','explore_exploit','fixed_proportions'}. Default is 'fixed_proportions' and allocates uniform probability to each algorithm. If mode=fixed_proportions then params is a dictionary that contains the field 'proportions' which is a list of dictionaries with fields 'alg_label' and 'proportion' for all algorithms in alg_list. All proportions must be positive and sum to 1 over all algs in alg_list
[optional] (string) participant_to_algorithm_management : in {'one_to_one','one_to_many'}. Default is 'one_to_many'.
[optional] (string) instructions
[optional] (string) debrief
[optional] (int) num_tries
Expected output:
if error:
return (JSON) '{}', (bool) False, (str) error_str
else:
return (JSON) '{}', (bool) True,''
Usage:
initExp_response_json,didSucceed,message = app.initExp(db_API,exp_uid,initExp_args_json)
Example input:
initExp_args_json = {"participant_to_algorithm_management": "one_to_many", "alg_list": [{"alg_label": "BR_LilUCB", "alg_id": "BR_LilUCB", "params": {}}], "algorithm_management_settings": {"params": {"proportions": [{"alg_label": "BR_LilUCB", "proportion": 1.0}]}, "mode": "fixed_proportions"}, "failure_probability": 0.01, "n": 10}
Example output:
initExp_response_json = {}
"""
try:
app_id = self.app_id
# remove any reminants of an experiment if it exists
didSucceed, message = db.delete_docs_with_filter(
'experiments_admin', {'exp_uid': exp_uid})
didSucceed, message = db.delete_docs_with_filter(
app_id + ':experiments', {'exp_uid': exp_uid})
didSucceed, message = db.delete_docs_with_filter(
app_id + ':queries', {'exp_uid': exp_uid})
didSucceed, message = db.delete_docs_with_filter(
app_id + ':participants', {'exp_uid': exp_uid})
didSucceed, message = db.delete_docs_with_filter(
app_id + ':algorithms', {'exp_uid': exp_uid})
didSucceed, message = ell.delete_logs_with_filter(
app_id + ':APP-CALL', {'exp_uid': exp_uid})
didSucceed, message = ell.delete_logs_with_filter(
app_id + ':APP-RESPONSE', {'exp_uid': exp_uid})
didSucceed, message = ell.delete_logs_with_filter(
app_id + ':APP-EXCEPTION', {'exp_uid': exp_uid})
didSucceed, message = ell.delete_logs_with_filter(
app_id + ':ALG-DURATION', {'exp_uid': exp_uid})
didSucceed, message = ell.delete_logs_with_filter(
app_id + ':ALG-EVALUATION', {'exp_uid': exp_uid})
# add indexes (only adds them if they do not already exist)
didSucceed, message = db.ensure_index('experiments_admin',
{'exp_uid': 1})
didSucceed, message = db.ensure_index(app_id + ':experiments',
{'exp_uid': 1})
didSucceed, message = db.ensure_index(app_id + ':queries',
{'query_uid': 1})
didSucceed, message = db.ensure_index(app_id + ':queries',
{'exp_uid': 1})
didSucceed, message = db.ensure_index(app_id + ':queries',
{'alg_uid': 1})
didSucceed, message = db.ensure_index(app_id + ':queries',
{'participant_uid': 1})
didSucceed, message = db.ensure_index(app_id + ':participants',
{'participant_uid': 1})
didSucceed, message = db.ensure_index(app_id + ':participants',
{'exp_uid': 1})
didSucceed, message = db.ensure_index(app_id + ':algorithms',
{'alg_uid': 1})
didSucceed, message = db.ensure_index(app_id + ':algorithms',
{'exp_uid': 1})
didSucceed, message = ell.ensure_index(app_id + ':APP-CALL',
{'exp_uid': 1})
didSucceed, message = ell.ensure_index(app_id + ':APP-CALL',
{'timestamp': 1})
didSucceed, message = ell.ensure_index(app_id + ':APP-CALL', {
'exp_uid': 1,
'timestamp': 1
})
didSucceed, message = ell.ensure_index(app_id + ':APP-CALL', {
'exp_uid': 1,
'task': 1
})
didSucceed, message = ell.ensure_index(app_id + ':APP-RESPONSE',
{'exp_uid': 1})
didSucceed, message = ell.ensure_index(app_id + ':APP-RESPONSE',
{'timestamp': 1})
didSucceed, message = ell.ensure_index(app_id + ':APP-RESPONSE', {
'exp_uid': 1,
'timestamp': 1
})
didSucceed, message = ell.ensure_index(app_id + ':APP-RESPONSE', {
'exp_uid': 1,
'task': 1
})
didSucceed, message = ell.ensure_index(app_id + ':APP-EXCEPTION',
{'exp_uid': 1})
didSucceed, message = ell.ensure_index(app_id + ':APP-EXCEPTION',
{'timestamp': 1})
didSucceed, message = ell.ensure_index(app_id + ':APP-EXCEPTION', {
'exp_uid': 1,
'timestamp': 1
})
didSucceed, message = ell.ensure_index(app_id + ':APP-EXCEPTION', {
'exp_uid': 1,
'task': 1
})
didSucceed, message = ell.ensure_index(app_id + ':ALG-DURATION',
{'exp_uid': 1})
didSucceed, message = ell.ensure_index(app_id + ':ALG-DURATION',
{'alg_uid': 1})
didSucceed, message = ell.ensure_index(app_id + ':ALG-DURATION',
{'timestamp': 1})
didSucceed, message = ell.ensure_index(app_id + ':ALG-DURATION', {
'exp_uid': 1,
'timestamp': 1
})
didSucceed, message = ell.ensure_index(app_id + ':ALG-DURATION', {
'alg_uid': 1,
'task': 1
})
didSucceed, message = ell.ensure_index(app_id + ':ALG-EVALUATION',
{'exp_uid': 1})
didSucceed, message = ell.ensure_index(app_id + ':ALG-EVALUATION',
{'alg_uid': 1})
didSucceed, message = ell.ensure_index(app_id + ':ALG-EVALUATION',
{'timestamp': 1})
didSucceed, message = ell.ensure_index(app_id + ':ALG-EVALUATION',
{
'exp_uid': 1,
'timestamp': 1
})
db.set('experiments_admin', exp_uid, 'exp_uid', exp_uid)
db.set('experiments_admin', exp_uid, 'app_id', app_id)
db.set('experiments_admin', exp_uid, 'start_date',
utils.datetime2str(utils.datetimeNow()))
log_entry = {
'exp_uid': exp_uid,
'task': 'initExp',
'json': args_json,
'timestamp': utils.datetimeNow()
}
ell.log(app_id + ':APP-CALL', log_entry)
# convert args_json to args_dict
try:
args_dict = json.loads(args_json)
except:
error = "%s.initExp input args_json is in improper format" % self.app_id
return '{}', False, error
# check for the fields that must be contained in args or error occurs
necessary_fields = ['n', 'failure_probability']
for field in necessary_fields:
try:
args_dict[field]
except KeyError:
error = "%s.initExp input arguments missing field: %s" % (
self.app_id, str(field))
return '{}', False, error
n = args_dict['n']
delta = args_dict['failure_probability']
if 'alg_list' in args_dict:
alg_list = args_dict['alg_list']
supportedAlgs = utils.get_app_supported_algs(self.app_id)
for algorithm in alg_list:
if algorithm['alg_id'] not in supportedAlgs:
error = "%s.initExp unsupported algorithm '%s' in alg_list" % (
self.app_id, alg_id)
return '{}', False, error
else:
alg_list = utils.get_app_default_alg_list(self.app_id)
if 'instructions' not in args_dict:
instructions = utils.get_app_default_instructions(app_id)
else:
instructions = args_dict['instructions']
if 'debrief' not in args_dict:
debrief = utils.get_app_default_instructions(app_id)
else:
debrief = args_dict['debrief']
if 'num_tries' not in args_dict:
num_tries = utils.get_app_default_num_tries(app_id)
else:
num_tries = args_dict['num_tries']
if 'context_type' not in args_dict:
context_type = 'none'
else:
context_type = args_dict['context_type']
if 'context' not in args_dict:
context = ''
else:
context = args_dict['context']
# ALGORITHM_MANAGEMENT_MODE FORMATTING CHECK
if 'algorithm_management_settings' not in args_dict:
params = {}
params['proportions'] = []
for algorithm in alg_list:
params['proportions'].append({
'alg_label':
algorithm['alg_label'],
'proportion':
1. / len(alg_list)
})
algorithm_management_settings = {}
algorithm_management_settings['mode'] = 'fixed_proportions'
algorithm_management_settings['params'] = params
else:
algorithm_management_settings = args_dict[
'algorithm_management_settings']
try:
mode = algorithm_management_settings['mode']
params = algorithm_management_settings['params']
except:
error = "%s.initExp algorithm_management_settings must be a dictionary with fields 'mode' and 'params'" % (
self.app_id)
return '{}', False, error
if mode == 'fixed_proportions':
try:
algorithm_proportions_list = params['proportions']
except:
error = "%s.initExp algorithm_management_settings['params'] must be a dictionary with field 'proportions'" % (
self.app_id)
return '{}', False, error
# check if alg_labels are properly labeled
for proportion_item in algorithm_proportions_list:
proportion = proportion_item['proportion']
target_alg_label = proportion_item['alg_label']
target_alg_label_in_alg_list = False
for algorithm in alg_list:
if algorithm['alg_label'] == target_alg_label:
target_alg_label_in_alg_list = True
if not target_alg_label_in_alg_list:
error = "%s.initExp algorithm_management_settings['params']['proportions'] must be a list of dictionaries, each dictionary containing the fields 'alg_label' and 'proportion'. The 'alg_label' value must be one of the alg_labels in a provided alg_list and 'proportion' must be nonnegative and sum to 1 : '%s' not in provided alg_list" % (
self.app_id, target_alg_label)
return '{}', False, error
elif mode == 'pure_exploration':
error = "%s.initExp Sorry, '%s' is not yet supported." % (
self.app_id, mode)
return '{}', False, error
elif mode == 'explore_exploit':
error = "%s.initExp Sorry, '%s' is not yet supported." % (
self.app_id, mode)
return '{}', False, error
else:
error = "%s.initExp unsupported algorithm_management_mode: '%s'. Must be in {'pure_exploration','explore_exploit','fixed_proportions'}" % (
self.app_id, algorithm_management_mode)
return '{}', False, error
# ALGORITHM_MANAGEMENT_MODE FORMATTING CHECK
if 'participant_to_algorithm_management' not in args_dict:
participant_to_algorithm_management = 'one_to_many'
else:
participant_to_algorithm_management = args_dict[
'participant_to_algorithm_management']
if participant_to_algorithm_management not in [
'one_to_many', 'one_to_one'
]:
error = "%s.initExp unsupported participant_to_algorithm_management: '%s'. Must be in {'one_to_many','one_to_one'}" % (
self.app_id, participant_to_algorithm_management)
return '{}', False, error
# assign uid to each algorithm and save it
for algorithm in alg_list:
alg_uid = utils.getNewUID()
algorithm['alg_uid'] = alg_uid
db.set(app_id + ':algorithms', alg_uid, 'alg_uid', alg_uid)
db.set(app_id + ':algorithms', alg_uid, 'exp_uid', exp_uid)
db.set(app_id + ':experiments', exp_uid, 'exp_uid', exp_uid)
db.set(app_id + ':experiments', exp_uid, 'app_id', app_id)
db.set(app_id + ':experiments', exp_uid, 'n', n)
db.set(app_id + ':experiments', exp_uid, 'failure_probability',
delta)
db.set(app_id + ':experiments', exp_uid, 'alg_list', alg_list)
db.set(app_id + ':experiments', exp_uid,
'algorithm_management_settings',
algorithm_management_settings)
db.set(app_id + ':experiments', exp_uid,
'participant_to_algorithm_management',
participant_to_algorithm_management)
db.set(app_id + ':experiments', exp_uid, 'instructions',
instructions)
db.set(app_id + ':experiments', exp_uid, 'debrief', debrief)
db.set(app_id + ':experiments', exp_uid, 'context_type',
context_type)
db.set(app_id + ':experiments', exp_uid, 'context', context)
db.set(app_id + ':experiments', exp_uid, 'num_tries', num_tries)
# now create intitialize each algorithm
for algorithm in alg_list:
alg_id = algorithm['alg_id']
alg_uid = algorithm['alg_uid']
db.set(app_id + ':algorithms', alg_uid, 'alg_id', alg_id)
db.set(app_id + ':algorithms', alg_uid, 'alg_uid', alg_uid)
db.set(app_id + ':algorithms', alg_uid, 'exp_uid', exp_uid)
# get sandboxed database for the specific app_id,alg_id,exp_uid - closing off the rest of the database to the algorithm
rc = ResourceClient(app_id, exp_uid, alg_uid, db)
# get specific algorithm to make calls to
alg = utils.get_app_alg(self.app_id, alg_id)
# call initExp
didSucceed, dt = utils.timeit(alg.initExp)(
resource=rc, n=n, failure_probability=delta)
log_entry = {
'exp_uid': exp_uid,
'alg_uid': alg_uid,
'task': 'initExp',
'duration': dt,
'timestamp': utils.datetimeNow()
}
ell.log(app_id + ':ALG-DURATION', log_entry)
response_json = '{}'
log_entry = {
'exp_uid': exp_uid,
'task': 'initExp',
'json': response_json,
'timestamp': utils.datetimeNow()
}
ell.log(app_id + ':APP-RESPONSE', log_entry)
return response_json, True, ''
except Exception, err:
error = traceback.format_exc()
log_entry = {
'exp_uid': exp_uid,
'task': 'initExp',
'error': error,
'timestamp': utils.datetimeNow(),
'args_json': args_json
}
ell.log(app_id + ':APP-EXCEPTION', log_entry)
return '{}', False, error
#!/usr/bin/python
import time
import traceback
import next.utils as utils
import next.broker.broker
broker = next.broker.broker.JobBroker()
while (1):
timestamp = utils.datetime2str(utils.datetimeNow())
try:
broker.refresh_domain_hashes()
except Exception, err:
error = traceback.format_exc()
print error
# wait
time.sleep(2)