def getModel(self, exp_uid, args_json):
try:
args_dict = self.helper.convert_json(args_json)
args_dict = verifier.verify(args_dict, self.reference_dict['getModel']['args'])
alg_label = args_dict['args']['alg_label']
args = self.butler.experiment.get(key='args')
for algorithm in args['alg_list']:
if alg_label == algorithm['alg_label']:
alg_id = algorithm['alg_id']
myapp_response = self.call_app_fn(alg_label, alg_id, 'getModel', args_dict)
myapp_response['exp_uid'] = exp_uid
myapp_response['alg_label'] = alg_label
# Log the response of the getModel in ALG-EVALUATION
if args_dict['args']['logging']:
alg_log_entry = {'exp_uid': exp_uid, 'alg_label':alg_label, 'task': 'getModel', 'timestamp': str(utils.datetimeNow())}
alg_log_entry.update(myapp_response)
self.butler.log('ALG-EVALUATION', alg_log_entry)
return json.dumps({'args': myapp_response,
'meta': {'log_entry_durations':self.log_entry_durations,
'timestamp': str(utils.datetimeNow())}}), True, ''
except Exception, error:
exc_type, exc_value, exc_traceback = sys.exc_info()
full_error = str(traceback.format_exc())+'\n'+str(error)
utils.debug_print("getModel Exception: " + full_error, color='red')
log_entry = { 'exp_uid':exp_uid,'task':'getModel','error':full_error,'timestamp':utils.datetimeNow(),'args_json':args_json }
self.butler.ell.log( self.app_id+':APP-EXCEPTION', log_entry )
traceback.print_tb(exc_traceback)
return Exception(error)
def getModel(self, exp_uid, args_json):
try:
args_dict = self.helper.convert_json(args_json)
args_dict = verifier.verify(args_dict, self.reference_dict['getModel']['args'])
alg_label = args_dict['args']['alg_label']
args = self.butler.experiment.get(key='args')
for algorithm in args['alg_list']:
if alg_label == algorithm['alg_label']:
alg_id = algorithm['alg_id']
myapp_response = self.call_app_fn(alg_label, alg_id, 'getModel', args_dict)
myapp_response['exp_uid'] = exp_uid
myapp_response['alg_label'] = alg_label
# Log the response of the getModel in ALG-EVALUATION
if args_dict['args']['logging']:
alg_log_entry = {'exp_uid': exp_uid, 'alg_label':alg_label, 'task': 'getModel', 'timestamp': str(utils.datetimeNow())}
alg_log_entry.update(myapp_response)
self.butler.log('ALG-EVALUATION', alg_log_entry)
return json.dumps({'args': myapp_response,
'meta': {'log_entry_durations':self.log_entry_durations,
'timestamp': str(utils.datetimeNow())}}), True, ''
except Exception, error:
exc_type, exc_value, exc_traceback = sys.exc_info()
full_error = str(traceback.format_exc())+'\n'+str(error)
utils.debug_print("getModel Exception: " + full_error, color='red')
log_entry = { 'exp_uid':exp_uid,'task':'getModel','error':full_error,'timestamp':utils.datetimeNow(),'args_json':args_json }
self.butler.ell.log( self.app_id+':APP-EXCEPTION', log_entry )
traceback.print_tb(exc_traceback)
return Exception(error)
def dashboardAsync(self, app_id, exp_uid, args, ignore_result=False):
"""
Run a task (task_name) on a set of args with a given app_id, and exp_uid.
Waits for computation to finish and returns the answer unless ignore_result=True in which case its a non-blocking call.
No guarantees about order of execution.
Inputs: ::\n
(string) app_id, (string) exp_id, (string) task_name, (json) args
Outputs: ::\n
task_name(app_id, exp_id, args)
"""
submit_timestamp = utils.datetimeNow('string')
domain = self.__get_domain_for_job(app_id + "_" + exp_uid)
if next.constants.CELERY_ON:
result = tasks.apply_dashboard.apply_async(
args=[app_id, exp_uid, args, submit_timestamp],
exchange='dashboard@' + domain,
routing_key='dashboard@' + domain)
if ignore_result:
return True
else:
return result.get(interval=0.001)
else:
result = tasks.apply_dashboard(app_id, exp_uid, args,
submit_timestamp)
if ignore_result:
return True
else:
return result
def refresh_domain_hashes(self):
# see what workers are out there
worker_pings = None
while worker_pings == None:
try:
# one could also use app.control.inspect().active_queues()
worker_pings = app.control.inspect().ping()
except:
worker_pings = None
worker_names = worker_pings.keys()
domain_names_with_dups = [item.split('@')[1] for item in worker_names]
domain_names = list(set(domain_names_with_dups)) # remove duplicates!
timestamp = utils.datetime2str(utils.datetimeNow())
print "[ %s ] domains with active workers = %s" % (timestamp,
str(domain_names))
replicated_domains_hash = []
for domain in domain_names:
for i in range(self.num_replicas):
replicated_domains_hash.append(
(domain, self.hash(domain + '_replica_' + str(i)), i))
replicated_domains_hash = sorted(replicated_domains_hash,
key=lambda x: x[1])
self.r.set('replicated_domains_hash',
json.dumps(replicated_domains_hash))
def init_alg(self, exp_uid, algorithm, alg_args):
butler = Butler(self.app_id, exp_uid, self.myApp.TargetManager,
self.butler.db, self.butler.ell,
algorithm['alg_label'], algorithm['alg_id'])
alg = utils.get_app_alg(self.app_id, algorithm['alg_id'])
if 'args' in self.algs_reference_dict['initExp']:
alg_args = verifier.verify(
alg_args, self.algs_reference_dict['initExp']['args'])
# I got rid of a timeit function here; it wasn't handling the
# argument unpacking correctly? --Scott, 2016-3-7
# TODO: put dt back in and change log_entry to relfect that
alg_response = alg.initExp(butler, **alg_args)
alg_response = verifier.verify(
{'rets': alg_response},
{'rets': self.algs_reference_dict['initExp']['rets']})
log_entry = {
'exp_uid': exp_uid,
'alg_label': algorithm['alg_label'],
'task': 'initExp',
'duration': -1,
'timestamp': utils.datetimeNow()
}
self.butler.log('ALG-DURATION', log_entry)
def applySyncByNamespace(self, app_id, exp_uid, task_name, args, namespace=None, ignore_result=False,time_limit=0):
"""
Run a task (task_name) on a set of args with a given app_id, and exp_uid asynchronously.
Waits for computation to finish and returns the answer unless ignore_result=True in which case its a non-blocking call.
If this method is called a sequence of times with the same namespace (defaults to exp_uid if not set) it is guaranteed that they will execute in order, each job finishing before the next begins
Inputs: ::\n
(string) app_id, (string) exp_id, (string) task_name, (json) args
"""
if namespace==None:
namespace=exp_uid
domain = self.__get_domain_for_job(app_id+"_"+exp_uid)
job_uid = utils.getNewUID()
submit_timestamp = utils.datetimeNow('string')
if time_limit == 0:
soft_time_limit = None
hard_time_limit = None
else:
soft_time_limit = time_limit
hard_time_limit = time_limit + .01
result = tasks.apply_sync_by_namespace.apply_async(args=[app_id,exp_uid,task_name, args, namespace, job_uid, submit_timestamp, time_limit], exchange='sync@'+domain, routing_key='sync@'+domain,soft_time_limit=soft_time_limit,time_limit=hard_time_limit)
if ignore_result:
return True
else:
return result.get(interval=.001)
def dashboardAsync(self, app_id, exp_uid, args, ignore_result=False):
"""
Run a task (task_name) on a set of args with a given app_id, and exp_uid.
Waits for computation to finish and returns the answer unless ignore_result=True in which case its a non-blocking call.
No guarantees about order of execution.
Inputs: ::\n
(string) app_id, (string) exp_id, (string) task_name, (json) args
Outputs: ::\n
task_name(app_id, exp_id, args)
"""
submit_timestamp = utils.datetimeNow('string')
domain = self.__get_domain_for_job(app_id+"_"+exp_uid)
if next.constants.CELERY_ON:
result = tasks.apply_dashboard.apply_async(args=[app_id,
exp_uid,
args,
submit_timestamp],
exchange='dashboard@'+domain,
routing_key='dashboard@'+domain)
if ignore_result:
return True
else:
return result.get(interval=0.001)
else:
result = tasks.apply_dashboard(app_id,exp_uid, args, submit_timestamp)
if ignore_result:
return True
else:
return result
def test_log(lapi):
B = 'test_log'
now = utils.datetimeNow()
log_entry = {'a': 2, 'timestamp': now}
lapi.log(B, log_entry)
assert lapi._bucket(B).find_one({'timestamp': now}).get('a') == 2
def test_log(lapi):
B = 'test_log'
now = utils.datetimeNow()
log_entry = {'a': 2, 'timestamp': now}
lapi.log(B, log_entry)
assert lapi._bucket(B).find_one({'timestamp': now}).get('a') == 2
def applySyncByNamespace(self, app_id, exp_uid, task_name, args, namespace=None, ignore_result=False,time_limit=0):
"""
Run a task (task_name) on a set of args with a given app_id, and exp_uid asynchronously.
Waits for computation to finish and returns the answer unless ignore_result=True in which case its a non-blocking call.
If this method is called a sequence of times with the same namespace (defaults to exp_uid if not set) it is guaranteed that they will execute in order, each job finishing before the next begins
Inputs: ::\n
(string) app_id, (string) exp_id, (string) task_name, (json) args
"""
submit_timestamp = utils.datetimeNow('string')
if namespace==None:
namespace=exp_uid
domain = self.__get_domain_for_job(app_id+"_"+exp_uid)
num_queues = next.constants.CELERY_SYNC_WORKER_COUNT
# assign namespaces to queues (with worker of concurrency 1) in round-robbin
try:
namespace_cnt = int(self.r.get(namespace+"_cnt"))
except:
pipe = self.r.pipeline(True)
while 1:
try:
pipe.watch(namespace+"_cnt","namespace_counter")
if not pipe.exists(namespace+"_cnt"):
if not pipe.exists('namespace_counter'):
namespace_counter = 0
else:
namespace_counter = pipe.get('namespace_counter')
pipe.multi()
pipe.set(namespace+"_cnt",int(namespace_counter)+1)
pipe.set('namespace_counter',int(namespace_counter)+1)
pipe.execute()
else:
pipe.unwatch()
break
except redis.exceptions.WatchError:
continue
finally:
pipe.reset()
namespace_cnt = int(self.r.get(namespace+"_cnt"))
queue_number = (namespace_cnt % num_queues) + 1
queue_name = 'sync_queue_'+str(queue_number)+'@'+domain
job_uid = utils.getNewUID()
if time_limit == 0:
soft_time_limit = None
hard_time_limit = None
else:
soft_time_limit = time_limit
hard_time_limit = time_limit + .01
result = tasks.apply_sync_by_namespace.apply_async(args=[app_id,exp_uid,task_name, args, namespace, job_uid, submit_timestamp, time_limit], queue=queue_name,soft_time_limit=soft_time_limit,time_limit=hard_time_limit)
if ignore_result:
return True
else:
return result.get(interval=.001)
def test_get_logs_with_filter(lapi):
B = 'test_get_logs_with_filter'
now = utils.datetimeNow()
log_entry = {'a': 2, 'timestamp': now}
lapi.log(B, log_entry)
retrieved_entry = lapi.get_logs_with_filter(B, {'timestamp': now})
assert len(retrieved_entry) == 1
assert retrieved_entry[0].get('a') == 2
def test_get_logs_with_filter(lapi):
B = 'test_get_logs_with_filter'
now = utils.datetimeNow()
log_entry = {'a': 2, 'timestamp': now}
lapi.log(B, log_entry)
retrieved_entry = lapi.get_logs_with_filter(B, {'timestamp': now})
assert len(retrieved_entry) == 1
assert retrieved_entry[0].get('a') == 2
def getQuery(self, exp_uid, args_json):
try:
args_dict = self.helper.convert_json(args_json)
args_dict = verifier.verify(args_dict, self.reference_dict['getQuery']['args'])
experiment_dict = self.butler.experiment.get()
alg_list = experiment_dict['args']['alg_list']
participant_to_algorithm_management = experiment_dict['args']['participant_to_algorithm_management']
algorithm_management_settings = experiment_dict['args']['algorithm_management_settings']
# Create the participant dictionary in participants bucket if needed. Also pull out label and id for this algorithm
participant_uid = args_dict['args'].get('participant_uid', args_dict['exp_uid'])
# Check to see if the first participant has come by and if not, save to db
participant_doc = self.butler.participants.get(uid=participant_uid)
first_participant_query = participant_doc==None
if first_participant_query:
participant_doc = {}
self.butler.participants.set(uid=participant_uid, value={'exp_uid':exp_uid, 'participant_uid':participant_uid})
if (participant_uid == exp_uid) or (participant_to_algorithm_management == 'one_to_many') or (first_participant_query):
if algorithm_management_settings['mode'] == 'fixed_proportions':
prop = [prop_item['proportion'] for prop_item in algorithm_management_settings['params']]
chosen_alg = numpy.random.choice(alg_list, p=prop)
elif algorithm_management_settings['mode'] == 'custom' :
chosen_alg = self.myApp.chooseAlg(self.butler, alg_list, args_dict['args'])
else:
chosen_alg = numpy.random.choice(alg_list)
alg_id = chosen_alg['alg_id']
alg_label = chosen_alg['alg_label']
if (first_participant_query) and (participant_to_algorithm_management=='one_to_one'):
self.butler.participants.set(uid=participant_uid, key='alg_id',value=alg_id)
self.butler.participants.set(uid=participant_uid, key='alg_label',value=alg_label)
elif (participant_to_algorithm_management=='one_to_one'):
alg_id = participant_doc['alg_id']
alg_label = participant_doc['alg_label']
query_uid = utils.getNewUID()
args_dict['args'].update(query_uid=query_uid)
query_doc = self.call_app_fn(alg_label, alg_id, 'getQuery', args_dict)
query_doc.update({'participant_uid':participant_uid,
'alg_id':alg_id,
'exp_uid':exp_uid,
'alg_label':alg_label,
'timestamp_query_generated':str(utils.datetimeNow()),
'query_uid':query_uid})
self.butler.queries.set(uid=query_uid, value=query_doc)
return json.dumps({'args':query_doc,'meta':{'log_entry_durations':self.log_entry_durations}}), True,''
except Exception, error:
exc_type, exc_value, exc_traceback = sys.exc_info()
full_error = str(traceback.format_exc())+'\n'+str(error)
utils.debug_print("getQuery Exception: " + full_error, color='red')
log_entry = { 'exp_uid':exp_uid,'task':'getQuery','error':full_error,'timestamp':utils.datetimeNow(),'args_json':args_json }
self.butler.ell.log( self.app_id+':APP-EXCEPTION', log_entry )
traceback.print_tb(exc_traceback)
return '{}', False, str(error)
def initExp(self, exp_uid, args_json):
try:
self.helper.ensure_indices(self.app_id, self.butler.db,
self.butler.ell)
args_dict = self.helper.convert_json(args_json)
args_dict = verifier.verify(args_dict,
self.reference_dict['initExp']['args'])
args_dict['exp_uid'] = exp_uid # to get doc from db
args_dict['start_date'] = utils.datetime2str(utils.datetimeNow())
self.butler.admin.set(uid=exp_uid,
value={
'exp_uid': exp_uid,
'app_id': self.app_id,
'start_date': str(utils.datetimeNow())
})
utils.debug_print(
"Launching app with arguments: {}".format(args_dict))
self.butler.experiment.set(value={'exp_uid': exp_uid})
args_dict['args'] = self.init_app(exp_uid,
args_dict['args']['alg_list'],
args_dict['args'])
args_dict['git_hash'] = git_hash
self.butler.experiment.set_many(key_value_dict=args_dict)
return '{}', True, ''
except Exception, error:
exc_type, exc_value, exc_traceback = sys.exc_info()
full_error = str(traceback.format_exc()) + '\n' + str(error)
utils.debug_print("initExp Exception: " + full_error, color='red')
log_entry = {
'exp_uid': exp_uid,
'task': 'initExp',
'error': full_error,
'timestamp': utils.datetimeNow(),
'args_json': args_json
}
self.butler.ell.log(self.app_id + ':APP-EXCEPTION', log_entry)
traceback.print_tb(exc_traceback)
return '{}', False, str(error)
def processAnswer(self, exp_uid, args_json):
try:
args_dict = self.helper.convert_json(args_json)
args_dict = verifier.verify(
args_dict, self.reference_dict['processAnswer']['args'])
# Update timing info in query
query = self.butler.queries.get(uid=args_dict['args']['query_uid'])
timestamp_answer_received = args_dict['args'].get(
'timestamp_answer_received', None)
delta_datetime = utils.str2datetime(timestamp_answer_received) - \
utils.str2datetime(query['timestamp_query_generated'])
round_trip_time = delta_datetime.total_seconds()
response_time = float(args_dict['args'].get('response_time', 0.))
query_update = self.call_app_fn(query['alg_label'],
query['alg_id'], 'processAnswer',
args_dict)
query_update.update({
'response_time':
response_time,
'network_delay':
round_trip_time - response_time,
'timestamp_answer_received':
timestamp_answer_received
})
self.butler.queries.set_many(uid=args_dict['args']['query_uid'],
key_value_dict=query_update)
return json.dumps({
'args': {},
'meta': {
'log_entry_durations': self.log_entry_durations
}
}), True, ''
except Exception, error:
exc_type, exc_value, exc_traceback = sys.exc_info()
full_error = str(traceback.format_exc()) + '\n' + str(error)
utils.debug_print("processAnswer Exception: " + full_error,
color='red')
log_entry = {
'exp_uid': exp_uid,
'task': 'processAnswer',
'error': full_error,
'timestamp': utils.datetimeNow(),
'args_json': args_json
}
self.butler.ell.log(self.app_id + ':APP-EXCEPTION', log_entry)
traceback.print_tb(exc_traceback)
raise Exception(error)
def most_current_embedding(self,app,butler,alg_label):
"""
Description: Returns embedding in the form of a list of dictionaries, which is conveneint for downstream applications
Expected input:
(string) alg_label : must be a valid alg_label contained in alg_list list of dicts
Expected output (in dict):
plot_type : 'scatter2d_noaxis'
(float) x_min : minimum x-value to display in viewing box
(float) x_max : maximum x-value to display in viewing box
(float) y_min : minimum y-value to display in viewing box
(float) y_max : maximum y-value to display in viewing box
(list of dicts with fields) data :
(int) index : index of target
(float) x : x-value of target
(float) y : y-value of target
"""
TargetManager = butler.targets
item = app.getModel(json.dumps({'exp_uid':app.exp_uid, 'args':{'alg_label':alg_label}}))
embedding = item['X']
data = []
x_min = numpy.float('inf')
x_max = -numpy.float('inf')
y_min = numpy.float('inf')
y_max = -numpy.float('inf')
for idx,target in enumerate(embedding):
target_dict = {}
target_dict['target'] = TargetManager.get_target_item(app.exp_uid, idx)
target_dict['x'] = target[0] # this is what will actually be plotted,
try:
target_dict['y'] = target[1] # takes first two components, (could be replaced by PCA)
except:
target_dict['y'] = 0.
target_dict['darray'] = target
x_min = min(x_min,target[0])
x_max = max(x_max,target[0])
y_min = min(y_min,target[1])
y_max = max(y_max,target[1])
data.append(target_dict)
return_dict = {'timestamp':str(utils.datetimeNow()),
'x_min':x_min, 'x_max':x_max, 'y_min':y_min, 'y_max':y_max, 'data':data,
'plot_type':'scatter2d_noaxis'}
return return_dict
def log(self,bucket_id,log_dict):
"""
Saves log_dict to PermStore as an individual document for later recall.
Inputs:
(string) bucket_id, (dict with string values) log_dict
Outputs:
(bool) didSucceed, (string) message
Usage: ::\n
didSucceed,message = db.log(bucket_id,doc_uid)
"""
timestamp = utils.datetimeNow()
log_dict.update({ 'timestamp':timestamp })
return self.permStore.setDoc(constants.logs_database_id,bucket_id,None,log_dict)
def log(self,bucket_id,log_dict):
"""
Saves log_dict to PermStore as an individual document for later recall.
Inputs:
(string) bucket_id, (dict with string values) log_dict
Outputs:
(bool) didSucceed, (string) message
Usage: ::\n
didSucceed,message = db.log(bucket_id,doc_uid)
"""
timestamp = utils.datetimeNow()
log_dict.update({ 'timestamp':timestamp })
return self.permStore.setDoc(constants.logs_database_id,bucket_id,None,log_dict)
def applySyncByNamespace(self,
app_id,
exp_uid,
task_name,
args,
namespace=None,
ignore_result=False,
time_limit=0):
"""
Run a task (task_name) on a set of args with a given app_id, and exp_uid asynchronously.
Waits for computation to finish and returns the answer unless ignore_result=True in which case its a non-blocking call.
If this method is called a sequence of times with the same namespace (defaults to exp_uid if not set) it is guaranteed that they will execute in order, each job finishing before the next begins
Inputs: ::\n
(string) app_id, (string) exp_id, (string) task_name, (json) args
"""
if namespace == None:
namespace = exp_uid
domain = self.__get_domain_for_job(app_id + "_" + exp_uid)
job_uid = utils.getNewUID()
submit_timestamp = utils.datetimeNow('string')
if time_limit == 0:
soft_time_limit = None
hard_time_limit = None
else:
soft_time_limit = time_limit
hard_time_limit = time_limit + .01
result = tasks.apply_sync_by_namespace.apply_async(
args=[
app_id, exp_uid, task_name, args, namespace, job_uid,
submit_timestamp, time_limit
],
exchange='sync@' + domain,
routing_key='sync@' + domain,
soft_time_limit=soft_time_limit,
time_limit=hard_time_limit)
if ignore_result:
return True
else:
return result.get(interval=.001)
def initExp(self, exp_uid, args_json):
try:
self.helper.ensure_indices(self.app_id,self.butler.db, self.butler.ell)
args_dict = self.helper.convert_json(args_json)
args_dict = verifier.verify(args_dict, self.reference_dict['initExp']['args'])
args_dict['exp_uid'] = exp_uid # to get doc from db
args_dict['start_date'] = utils.datetime2str(utils.datetimeNow())
self.butler.admin.set(uid=exp_uid,value={'exp_uid': exp_uid, 'app_id':self.app_id, 'start_date':str(utils.datetimeNow())})
self.butler.experiment.set(value={'exp_uid': exp_uid})
args_dict['args'] = self.init_app(exp_uid, args_dict['args']['alg_list'], args_dict['args'])
args_dict['git_hash'] = git_hash
self.butler.experiment.set_many(key_value_dict=args_dict)
return '{}', True, ''
except Exception, error:
exc_type, exc_value, exc_traceback = sys.exc_info()
full_error = str(traceback.format_exc())+'\n'+str(error)
utils.debug_print("initExp Exception: " + full_error, color='red')
log_entry = { 'exp_uid':exp_uid,'task':'initExp','error':full_error,'timestamp':utils.datetimeNow(),'args_json':args_json }
self.butler.ell.log( self.app_id+':APP-EXCEPTION', log_entry )
traceback.print_tb(exc_traceback)
return '{}', False, str(error)
def refresh_domain_hashes(self):
# see what workers are out there
worker_pings = None
while worker_pings==None:
try:
# one could also use app.control.inspect().active_queues()
worker_pings = app.control.inspect().ping()
except:
worker_pings = None
worker_names = worker_pings.keys()
domain_names_with_dups = [ item.split('@')[1] for item in worker_names]
domain_names = list(set(domain_names_with_dups)) # remove duplicates!
timestamp = utils.datetime2str(utils.datetimeNow())
print "[ %s ] domains with active workers = %s" % (timestamp,str(domain_names))
replicated_domains_hash = []
for domain in domain_names:
for i in range(self.num_replicas):
replicated_domains_hash.append((domain, self.hash(domain+'_replica_'+str(i)), i))
replicated_domains_hash = sorted( replicated_domains_hash, key = lambda x: x[1])
self.r.set('replicated_domains_hash',json.dumps(replicated_domains_hash))
def predict(self, exp_uid, args_json, db, ell):
"""
Description: uses current model empirical estimates to forecast the ranking of the arms in order of likelhood of being the best from most to least likely
Expected input:
(string) predict_id : 'predict_labels'
(dict) params : dictionary with fields
(string) alg_label : describes target algorithm to use
Expected output (in json structure):
(list of dicts with fields):
(int) index : index of target
(int) rank : rank that the algorithm assigns to index (rank 0 is most likely the best arm)
"""
try:
app_id = self.app_id
log_entry = {
'exp_uid': exp_uid,
'task': 'predict',
'json': args_json,
'timestamp': utils.datetimeNow()
}
ell.log(app_id + ':APP-CALL', log_entry)
# convert args_json to args_dict
try:
args_dict = json.loads(args_json)
except:
error = "%s.predict failed to convert input args_json due to improper format" % (
self.app_id)
return '{}', False, error
# check for the fields that must be contained in args or error occurs
necessary_fields = ['predict_id', 'params']
for field in necessary_fields:
try:
args_dict[field]
except KeyError:
error = "%s.predict input arguments missing field: %s" % (
self.app_id, str(field))
return '{}', False, error
predict_id = args_dict['predict_id']
params = args_dict['params']
if predict_id == "predict_labels":
alg_label = params['alg_label']
# get list of algorithms associated with project
alg_list, didSucceed, message = db.get(app_id + ':experiments',
exp_uid, 'alg_list')
# get alg_id
for algorithm in alg_list:
if alg_label == algorithm['alg_label']:
alg_id = algorithm['alg_id']
alg_uid = algorithm['alg_uid']
num_reported_answers, didSucceed, message = db.get(
app_id + ':experiments', exp_uid,
'num_reported_answers_for_' + alg_uid)
if type(num_reported_answers) != int:
num_reported_answers = 0
# get sandboxed database for the specific app_id,alg_id,exp_uid - closing off the rest of the database to the algorithm
rc = ResourceClient(app_id, exp_uid, alg_uid, db)
# get specific algorithm to make calls to
alg = utils.get_app_alg(self.app_id, alg_id)
# call getQuery
predicted_labels, dt = utils.timeit(alg.predict)(resource=rc)
log_entry_durations = {
'exp_uid': exp_uid,
'alg_uid': alg_uid,
'task': 'predict',
'duration': dt
}
log_entry_durations.update(rc.getDurations())
meta = {'log_entry_durations': log_entry_durations}
test_labels, didSucceed, message = db.get(
app_id + ':experiments', exp_uid, 'test_labels')
num_errors = 0
for answer in test_labels:
target_index, target_label = answer
if target_label * predicted_labels[target_index] < 0:
num_errors += 1.
error = num_errors / float(max(1., len(test_labels)))
log_entry = {
'exp_uid': exp_uid,
'alg_uid': alg_uid,
'timestamp': utils.datetimeNow()
}
log_entry.update({
'error': error,
'num_reported_answers': num_reported_answers
})
ell.log(app_id + ':ALG-EVALUATION', log_entry)
response_args_dict = {
'exp_uid': exp_uid,
'alg_uid': alg_uid,
'error': error,
'num_reported_answers': num_reported_answers
}
args_out = {'args': response_args_dict, 'meta': meta}
predict_json = json.dumps(args_out)
log_entry = {
'exp_uid': exp_uid,
'task': 'predict',
'json': predict_json,
'timestamp': utils.datetimeNow()
}
ell.log(app_id + ':APP-RESPONSE', log_entry)
return predict_json, True, ''
except Exception, err:
error = traceback.format_exc()
log_entry = {
'exp_uid': exp_uid,
'task': 'predict',
'error': str(error),
'timestamp': utils.datetimeNow(),
'args_json': args_json
}
didSucceed, message = ell.log(app_id + ':APP-EXCEPTION', log_entry)
return '{}', False, error
def applySyncByNamespace(self,
app_id,
exp_uid,
alg_id,
alg_label,
task_name,
args,
namespace=None,
ignore_result=False,
time_limit=0):
"""
Run a task (task_name) on a set of args with a given app_id, and exp_uid asynchronously.
Waits for computation to finish and returns the answer unless ignore_result=True in which case its a non-blocking call.
If this method is called a sequence of times with the same namespace (defaults to exp_uid if not set) it is guaranteed that they will execute in order, each job finishing before the next begins
Inputs: ::\n
(string) app_id, (string) exp_id, (string) task_name, (json) args
"""
submit_timestamp = utils.datetimeNow('string')
if namespace == None:
namespace = exp_uid
domain = self.__get_domain_for_job(app_id + "_" + exp_uid)
num_queues = next.constants.CELERY_SYNC_WORKER_COUNT
# assign namespaces to queues (with worker of concurrency 1) in round-robbin
try:
namespace_cnt = int(self.r.get(namespace + "_cnt"))
except:
pipe = self.r.pipeline(True)
while 1:
try:
pipe.watch(namespace + "_cnt", "namespace_counter")
if not pipe.exists(namespace + "_cnt"):
if not pipe.exists('namespace_counter'):
namespace_counter = 0
else:
namespace_counter = pipe.get('namespace_counter')
pipe.multi()
pipe.set(namespace + "_cnt",
int(namespace_counter) + 1)
pipe.set('namespace_counter',
int(namespace_counter) + 1)
pipe.execute()
else:
pipe.unwatch()
break
except redis.exceptions.WatchError:
continue
finally:
pipe.reset()
namespace_cnt = int(self.r.get(namespace + "_cnt"))
queue_number = (namespace_cnt % num_queues) + 1
queue_name = 'sync_queue_' + str(queue_number) + '@' + domain
job_uid = utils.getNewUID()
if time_limit == 0:
soft_time_limit = None
hard_time_limit = None
else:
soft_time_limit = time_limit
hard_time_limit = time_limit + .01
if next.constants.CELERY_ON:
result = tasks.apply_sync_by_namespace.apply_async(
args=[
app_id, exp_uid, alg_id, alg_label, task_name, args,
namespace, job_uid, submit_timestamp, time_limit
],
queue=queue_name,
soft_time_limit=soft_time_limit,
time_limit=hard_time_limit)
if ignore_result:
return True
else:
return result.get(interval=.001)
else:
result = tasks.apply_sync_by_namespace(app_id, exp_uid, alg_id,
alg_label, task_name, args,
namespace, job_uid,
submit_timestamp,
time_limit)
if ignore_result:
return True
else:
return result
def getQuery(self, exp_uid, args_json, db, ell):
"""
A request to ask which k arms to duel next
Expected input (in jsonstructure with string keys):
(int) k : number of objects to display
[optional] (string) participant_uid : unique identifier of session for a participant answering questions (that is, an email address is not good enough as the participant could participate in multiple exp_uids so it would not be unique against all experiments), if key non-existant particpant_uid is assigned as exp_uid.
Expected output (in json structure with string keys):
(list) target_indices : list of k target indexes e.g. [ (int) target_index_1, ... , (int) target_index_k ]
(str) query_uid : unique identifier of query (used to look up for processAnswer)
"""
try:
app_id = self.app_id
log_entry = {
'exp_uid': exp_uid,
'task': 'getQuery',
'json': args_json,
'timestamp': utils.datetimeNow()
}
ell.log(app_id + ':APP-CALL', log_entry)
# convert args_json to args_dict
try:
args_dict = json.loads(args_json)
except:
error = "%s.initExp input args_json is in improper format" % self.app_id
return '{}', False, error
# get list of algorithms associated with project
alg_list, didSucceed, message = db.get(app_id + ':experiments',
exp_uid, 'alg_list')
alg_label_to_alg_id = {}
alg_label_to_alg_uid = {}
for algorithm in alg_list:
alg_label_to_alg_id[
algorithm['alg_label']] = algorithm['alg_id']
alg_label_to_alg_uid[
algorithm['alg_label']] = algorithm['alg_uid']
algorithm_management_settings, didSucceed, message = db.get(
app_id + ':experiments', exp_uid,
'algorithm_management_settings')
# ASSIGN ALGORITHM TO PARTICIPANT
if 'participant_uid' in args_dict:
participant_uid = args_dict['participant_uid']
else:
participant_uid = exp_uid
participant_doc_exists, didSucceed, message = db.exists(
app_id + ':participants', participant_uid, 'participant_uid')
first_participant_query = not participant_doc_exists
if first_participant_query:
db.set(app_id + ':participants', participant_uid,
'participant_uid', participant_uid)
db.set(app_id + ':participants', participant_uid, 'exp_uid',
exp_uid)
participant_to_algorithm_management, didSucceed, message = db.get(
app_id + ':experiments', exp_uid,
'participant_to_algorithm_management')
if (participant_uid == exp_uid) or (
participant_to_algorithm_management
== 'one_to_many') or (first_participant_query):
if algorithm_management_settings[
'mode'] == 'fixed_proportions':
proportions_list = algorithm_management_settings['params'][
'proportions']
prop = [
prop_item['proportion']
for prop_item in proportions_list
]
prop_item = numpy.random.choice(alg_list, p=prop)
else:
raise Exception('algorithm_management_mode : ' +
algorithm_management_settings['mode'] +
' not implemented')
alg_id = alg_label_to_alg_id[prop_item['alg_label']]
alg_uid = alg_label_to_alg_uid[prop_item['alg_label']]
alg_label = prop_item['alg_label']
if (first_participant_query) and (
participant_to_algorithm_management == 'one_to_one'):
db.set(app_id + ':participants', participant_uid, 'alg_id',
alg_id)
db.set(app_id + ':participants', participant_uid,
'alg_uid', alg_uid)
elif (participant_to_algorithm_management == 'one_to_one'):
# If here, then alg_uid should already be assigned in participant doc
alg_id, didSucceed, message = db.get(app_id + ':participants',
participant_uid, 'alg_id')
alg_uid, didSucceed, message = db.get(app_id + ':participants',
participant_uid,
'alg_uid')
else:
raise Exception('participant_to_algorithm_management : ' +
participant_to_algorithm_management +
' not implemented')
# get sandboxed database for the specific app_id,alg_id,exp_uid - closing off the rest of the database to the algorithm
rc = ResourceClient(app_id, exp_uid, alg_uid, db)
# get specific algorithm to make calls to
alg = utils.get_app_alg(self.app_id, alg_id)
# call getQuery
targets, dt = utils.timeit(alg.getQuery)(resource=rc)
# update targets
target_list = []
for target in targets:
target_list.append({'index': target})
# check for context
context_type, didSucceed, message = db.get(app_id + ':experiments',
exp_uid, 'context_type')
context, didSucceed, message = db.get(app_id + ':experiments',
exp_uid, 'context')
# log
log_entry_durations = {
'exp_uid': exp_uid,
'alg_uid': alg_uid,
'task': 'getQuery',
'duration': dt
}
log_entry_durations.update(rc.getDurations())
meta = {'log_entry_durations': log_entry_durations}
# create JSON query payload
timestamp = str(utils.datetimeNow())
query_uid = utils.getNewUID()
query = {}
query['query_uid'] = query_uid
query['target_indices'] = target_list
# save query data to database
query_doc = {}
query_doc.update(query)
query_doc['participant_uid'] = participant_uid
query_doc['alg_uid'] = alg_uid
query_doc['exp_uid'] = exp_uid
query_doc['alg_label'] = alg_label
query_doc['timestamp_query_generated'] = timestamp
for field in query_doc:
db.set(app_id + ':queries', query_uid, field, query_doc[field])
# add context after updating query doc to avoid redundant information
query['context_type'] = context_type
query['context'] = context
args_out = {'args': query, 'meta': meta}
response_json = json.dumps(args_out)
log_entry = {
'exp_uid': exp_uid,
'task': 'getQuery',
'json': response_json,
'timestamp': utils.datetimeNow()
}
ell.log(app_id + ':APP-RESPONSE', log_entry)
return response_json, True, ''
except Exception, err:
error = traceback.format_exc()
log_entry = {
'exp_uid': exp_uid,
'task': 'getQuery',
'error': error,
'timestamp': utils.datetimeNow(),
'args_json': args_json
}
ell.log(app_id + ':APP-EXCEPTION', log_entry)
return '{}', False, error
def getQuery(self,exp_uid,args_json,db,ell):
"""
A request to ask which two arms to duel next
Expected input (in jsonstructure with string keys):
[optional] (string) participant_uid : unique identifier of session for a participant answering questions (that is, an email address is not good enough as the participant could participate in multiple exp_uids so it would not be unique against all experiments), if key non-existant particpant_uid is assigned as exp_uid.
Expected output (in json structure with string keys):
(list) target_indices : list that stores dictionary of targets with fields:
{
(int) index : the index of the target of relevance
(str) label : in {'left','right'} for display
(int) flag : integer for algorithm's use
}
(str) query_uid : unique identifier of query (used to look up for processAnswer)
"""
try:
app_id = self.app_id
log_entry = { 'exp_uid':exp_uid,'task':'getQuery','json':args_json,'timestamp':utils.datetimeNow() }
ell.log( app_id+':APP-CALL', log_entry )
# convert args_json to args_dict
try:
args_dict = json.loads(args_json)
except:
error = "%s.initExp input args_json is in improper format" % self.app_id
return '{}',False,error
# get list of algorithms associated with project
alg_list,didSucceed,message = db.get(app_id+':experiments',exp_uid,'alg_list')
alg_label_to_alg_id = {}
alg_label_to_alg_uid = {}
for algorithm in alg_list:
alg_label_to_alg_id[ algorithm['alg_label'] ] = algorithm['alg_id']
alg_label_to_alg_uid[ algorithm['alg_label'] ] = algorithm['alg_uid']
algorithm_management_settings,didSucceed,message = db.get(app_id+':experiments',exp_uid,'algorithm_management_settings')
# ASSIGN ALGORITHM TO PARTICIPANT
if 'participant_uid' in args_dict:
participant_uid = args_dict['participant_uid']
else:
participant_uid = exp_uid
participant_doc_exists,didSucceed,message = db.exists(app_id+':participants',participant_uid,'participant_uid')
first_participant_query = not participant_doc_exists
if first_participant_query:
db.set(app_id+':participants',participant_uid,'participant_uid',participant_uid)
db.set(app_id+':participants',participant_uid,'exp_uid',exp_uid)
participant_to_algorithm_management,didSucceed,message = db.get(app_id+':experiments',exp_uid,'participant_to_algorithm_management')
if (participant_uid==exp_uid) or (participant_to_algorithm_management=='one_to_many') or (first_participant_query):
if algorithm_management_settings['mode']=='fixed_proportions':
proportions_list = algorithm_management_settings['params']['proportions']
prop = [ prop_item['proportion'] for prop_item in proportions_list ]
prop_item = numpy.random.choice(alg_list,p=prop)
else:
raise Exception('algorithm_management_mode : '+algorithm_management_settings['mode']+' not implemented')
alg_id = alg_label_to_alg_id[ prop_item['alg_label'] ]
alg_uid = alg_label_to_alg_uid[ prop_item['alg_label'] ]
alg_label = prop_item['alg_label']
if (first_participant_query) and (participant_to_algorithm_management=='one_to_one'):
db.set(app_id+':participants',participant_uid,'alg_id',alg_id)
db.set(app_id+':participants',participant_uid,'alg_uid',alg_uid)
elif (participant_to_algorithm_management=='one_to_one'):
# If here, then alg_uid should already be assigned in participant doc
alg_id,didSucceed,message = db.get(app_id+':participants',participant_uid,'alg_id')
alg_uid,didSucceed,message = db.get(app_id+':participants',participant_uid,'alg_uid')
else:
raise Exception('participant_to_algorithm_management : '+participant_to_algorithm_management+' not implemented')
# get sandboxed database for the specific app_id,alg_id,exp_uid - closing off the rest of the database to the algorithm
rc = ResourceClient(app_id,exp_uid,alg_uid,db)
# get specific algorithm to make calls to
alg = utils.get_app_alg(self.app_id,alg_id)
# call getQuery
index_left,index_right,index_painted,dt = utils.timeit(alg.getQuery)(resource=rc)
# check for context
context_type,didSucceed,message = db.get(app_id+':experiments',exp_uid,'context_type')
context,didSucceed,message = db.get(app_id+':experiments',exp_uid,'context')
# log
log_entry_durations = { 'exp_uid':exp_uid,'alg_uid':alg_uid,'task':'getQuery','duration':dt }
log_entry_durations.update( rc.getDurations() )
meta = {'log_entry_durations':log_entry_durations}
# create JSON query payload
if index_left==index_painted:
targets = [ {'index':index_left,'label':'left','flag':1}, {'index':index_right,'label':'right','flag':0} ]
else:
targets = [ {'index':index_left,'label':'left','flag':0}, {'index':index_right,'label':'right','flag':1} ]
timestamp = str(utils.datetimeNow())
query_uid = utils.getNewUID()
query = {}
query['query_uid'] = query_uid
query['target_indices'] = targets
# save query data to database
query_doc = {}
query_doc.update(query)
query_doc['participant_uid'] = participant_uid
query_doc['alg_uid'] = alg_uid
query_doc['exp_uid'] = exp_uid
query_doc['alg_label'] = alg_label
query_doc['timestamp_query_generated'] = timestamp
db.set_doc(app_id+':queries',query_uid,query_doc)
# add context after updating query doc to avoid redundant information
query['context_type'] = context_type
query['context'] = context
args_out = {'args':query,'meta':meta}
response_json = json.dumps(args_out)
log_entry = { 'exp_uid':exp_uid,'task':'getQuery','json':response_json,'timestamp':utils.datetimeNow() }
ell.log( app_id+':APP-RESPONSE', log_entry )
return response_json,True,''
except Exception, err:
error = traceback.format_exc()
log_entry = { 'exp_uid':exp_uid,'task':'getQuery','error':error,'timestamp':utils.datetimeNow(),'args_json':args_json }
ell.log( app_id+':APP-EXCEPTION', log_entry )
return '{}',False,error
def predict(self, exp_uid, args_json, db, ell):
"""
uses current model empirical estimates to forecast which index has the highest mean
Expected input (in json structure with string keys):
(string) predict_id : identifier for the desired prediction
(list) params : dictionary of stat_id specific fields.
##########
Description: uses current model empirical estimates to forecast the ranking of the arms in order of likelhood of being the best from most to least likely
Expected input:
(string) predict_id : 'arm_ranking'
(dict) params : dictionary with fields
(string) alg_label : describes target algorithm to use
Expected output (in json structure):
(list of dicts with fields):
(int) index : index of target
(int) rank : rank that the algorithm assigns to index (rank 0 is most likely the best arm)
"""
try:
app_id = self.app_id
log_entry = {
'exp_uid': exp_uid,
'task': 'predict',
'json': args_json,
'timestamp': utils.datetimeNow()
}
ell.log(app_id + ':APP-CALL', log_entry)
# convert args_json to args_dict
try:
args_dict = json.loads(args_json)
except:
error = "%s.predict failed to convert input args_json due to improper format" % (
self.app_id)
return '{}', False, error
# check for the fields that must be contained in args or error occurs
necessary_fields = ['predict_id', 'params']
for field in necessary_fields:
try:
args_dict[field]
except KeyError:
error = "%s.predict input arguments missing field: %s" % (
self.app_id, str(field))
return '{}', False, error
predict_id = args_dict['predict_id']
params = args_dict['params']
if predict_id == "arm_ranking":
alg_label = params['alg_label']
# get list of algorithms associated with project
alg_list, didSucceed, message = db.get(app_id + ':experiments',
exp_uid, 'alg_list')
# get alg_id
for algorithm in alg_list:
if alg_label == algorithm['alg_label']:
alg_id = algorithm['alg_id']
alg_uid = algorithm['alg_uid']
num_reported_answers, didSucceed, message = db.get(
app_id + ':experiments', exp_uid,
'num_reported_answers_for_' + alg_uid)
if type(num_reported_answers) != int:
num_reported_answers = 0
# get sandboxed database for the specific app_id,alg_id,exp_uid - closing off the rest of the database to the algorithm
rc = ResourceClient(app_id, exp_uid, alg_uid, db)
# get specific algorithm to make calls to
alg = utils.get_app_alg(self.app_id, alg_id)
# call getQuery
scores, precisions, dt = utils.timeit(alg.predict)(resource=rc)
log_entry_durations = {
'exp_uid': exp_uid,
'alg_uid': alg_uid,
'task': 'predict',
'duration': dt
}
log_entry_durations.update(rc.getDurations())
meta = {'log_entry_durations': log_entry_durations}
import numpy
ranks = (-numpy.array(scores)).argsort().tolist()
n = len(scores)
indexes = numpy.array(range(n))[ranks]
scores = numpy.array(scores)[ranks]
precisions = numpy.array(precisions)[ranks]
ranks = range(n)
targets = []
for index in range(n):
targets.append({
'index': indexes[index],
'rank': ranks[index],
'score': scores[index],
'precision': precisions[index]
})
log_entry = {
'exp_uid': exp_uid,
'alg_uid': alg_uid,
'timestamp': utils.datetimeNow()
}
log_entry.update({
'targets': targets,
'num_reported_answers': num_reported_answers
})
ell.log(app_id + ':ALG-EVALUATION', log_entry)
response_args_dict = {}
args_out = {'args': response_args_dict, 'meta': meta}
predict_json = json.dumps(args_out)
log_entry = {
'exp_uid': exp_uid,
'task': 'predict',
'json': predict_json,
'timestamp': utils.datetimeNow()
}
ell.log(app_id + ':APP-RESPONSE', log_entry)
return predict_json, True, ''
except Exception, err:
error = traceback.format_exc()
log_entry = {
'exp_uid': exp_uid,
'task': 'predict',
'error': str(error),
'timestamp': utils.datetimeNow(),
'args_json': args_json
}
didSucceed, message = ell.log(app_id + ':APP-EXCEPTION', log_entry)
return '{}', False, error
def processAnswer(self,exp_uid,args_json,db,ell):
"""
reporting back the reward of pulling the arm suggested by getQuery
Expected input (in json structure with string keys):
(index) index_winner : index of the winner in (must be index of left or right target in target_indices)
(str) query_uid : unique identifier of query
Expected output (comma separated):
if error:
return (JSON) '{}', (bool) False, (str) error
else:
return (JSON) '{}', (bool) True,''
"""
try:
app_id = self.app_id
log_entry = { 'exp_uid':exp_uid,'task':'processAnswer','json':args_json,'timestamp':utils.datetimeNow() }
ell.log( app_id+':APP-CALL', log_entry )
# convert args_json to args_dict
try:
args_dict = json.loads(args_json)
except:
error = "%s.processAnswer input args_json is in improper format" % self.app_id
return '{}',False,error
# check for the fields that must be contained in args or error occurs
necessary_fields = ['index_winner','query_uid']
for field in necessary_fields:
try:
args_dict[field]
except KeyError:
error = "%s.processAnswer input arguments missing field: %s" % (self.app_id,str(field))
return '{}',False,error
# get list of algorithms associated with project
alg_list,didSucceed,message = db.get(app_id+':experiments',exp_uid,'alg_list')
# get alg_id
query_uid = args_dict['query_uid']
alg_uid,didSucceed,message = db.get(app_id+':queries',query_uid,'alg_uid')
if not didSucceed:
raise Exception("Failed to retrieve query with query_uid="+query_uid)
for algorithm in alg_list:
if alg_uid == algorithm['alg_uid']:
alg_id = algorithm['alg_id']
alg_label = algorithm['alg_label']
test_alg_label = algorithm['test_alg_label']
num_reported_answers,didSucceed,message = db.increment(app_id+':experiments',exp_uid,'num_reported_answers_for_'+alg_uid)
# get sandboxed database for the specific app_id,alg_id,exp_uid - closing off the rest of the database to the algorithm
rc = ResourceClient(app_id,exp_uid,alg_uid,db)
# get specific algorithm to make calls to
alg = utils.get_app_alg(self.app_id,alg_id)
targets,didSucceed,message = db.get(app_id+':queries',query_uid,'target_indices')
for target in targets:
if target['label'] == 'center':
index_center = target['index']
elif target['label'] == 'left':
index_left = target['index']
elif target['label'] == 'right':
index_right = target['index']
index_winner = args_dict['index_winner']
# update query doc
timestamp_query_generated,didSucceed,message = db.get(app_id+':queries',query_uid,'timestamp_query_generated')
datetime_query_generated = utils.str2datetime(timestamp_query_generated)
timestamp_answer_received = args_dict.get('meta',{}).get('timestamp_answer_received',None)
if timestamp_answer_received == None:
datetime_answer_received = datetime_query_generated
else:
datetime_answer_received = utils.str2datetime(timestamp_answer_received)
delta_datetime = datetime_answer_received - datetime_query_generated
round_trip_time = delta_datetime.seconds + delta_datetime.microseconds/1000000.
response_time = float(args_dict.get('response_time',0.))
db.set(app_id+':queries',query_uid,'response_time',response_time)
db.set(app_id+':queries',query_uid,'network_delay',round_trip_time-response_time)
db.set(app_id+':queries',query_uid,'index_winner',index_winner)
q = [index_left,index_right,index_center]
if index_winner==index_right:
q = [index_right,index_left,index_center]
db.set(app_id+':queries',query_uid,'q',q)
# call processAnswer
didSucceed,dt = utils.timeit(alg.processAnswer)(resource=rc,index_center=index_center,index_left=index_left,index_right=index_right,index_winner=index_winner)
log_entry_durations = { 'exp_uid':exp_uid,'alg_uid':alg_uid,'task':'processAnswer','duration':dt }
log_entry_durations.update( rc.getDurations() )
meta = {'log_entry_durations':log_entry_durations}
# check if we're going to evaluate this loss
n,didSucceed,message = db.get(app_id+':experiments',exp_uid,'n')
if num_reported_answers % ((n+4)/4) == 0:
predict_id = 'get_embedding'
params = {'alg_label':alg_label}
predict_args_dict = {'predict_id':predict_id,'params':params}
predict_args_json = json.dumps(predict_args_dict)
db.submit_job(app_id,exp_uid,'predict',predict_args_json,ignore_result=True)
###############
response_args_dict = {}
args_out = {'args':response_args_dict,'meta':meta}
response_json = json.dumps(args_out)
log_entry = { 'exp_uid':exp_uid,'task':'processAnswer','json':response_json,'timestamp':utils.datetimeNow() }
ell.log( app_id+':APP-RESPONSE', log_entry )
return response_json,True,""
except Exception, err:
error = traceback.format_exc()
log_entry = { 'exp_uid':exp_uid,'task':'processAnswer','error':error,'timestamp':utils.datetimeNow() }
ell.log( app_id+':APP-EXCEPTION', log_entry )
return '{}',False,error
def predict(self,exp_uid,args_json,db,ell):
"""
Have the model learned by some particular algorithm predict a variety of stuff
Expected input (in json structure with string keys):
(string) predict_id : identifier for the desired prediction
(list) params : dictionary of stat_id specific fields.
##########
Description: Each algorithm (with an associated alg_label) has a test_alg_label associated with it.
Expected input:
(string) predict_id : 'evaluate_on_test'
(dict) params : dictionary with fields
(string) alg_label : describes target algorithm to test
(string) test_alg_label : describes the algorithm that whos triplets are evaluated on the target algorithm
Expected output (in json structure):
(float) num_reported_answers : number of reported answers after which the calculation was made
(float) error : 0/1 loss on test set
"""
try:
app_id = self.app_id
log_entry = { 'exp_uid':exp_uid,'task':'predict','json':args_json,'timestamp':utils.datetimeNow() }
ell.log( app_id+':APP-CALL', log_entry )
# convert args_json to args_dict
try:
args_dict = json.loads(args_json)
except:
error = "%s.predict input args_json is in improper format" % self.app_id
return '{}',False,error
# check for the fields that must be contained in args or error occurs
necessary_fields = ['predict_id','params']
for field in necessary_fields:
try:
args_dict[field]
except KeyError:
error = "%s.predict input arguments missing field: %s" % (self.app_id,str(field))
return '{}',False,error
predict_id = args_dict['predict_id']
params = args_dict['params']
alg_label = params['alg_label']
# get list of algorithms associated with project
alg_list,didSucceed,message = db.get(app_id+':experiments',exp_uid,'alg_list')
# get alg_id
for algorithm in alg_list:
if alg_label == algorithm['alg_label']:
alg_id = algorithm['alg_id']
alg_uid = algorithm['alg_uid']
meta = {}
if predict_id=='get_embedding':
# get sandboxed database for the specific app_id,alg_id,exp_uid - closing off the rest of the database to the algorithm
rc = ResourceClient(app_id,exp_uid,alg_uid,db)
# get specific algorithm to make calls to
alg = utils.get_app_alg(self.app_id,alg_id)
##### Get Embedding #####
Xd,num_reported_answers,dt = utils.timeit(alg.predict)(rc)
log_entry_durations = { 'exp_uid':exp_uid,'alg_uid':alg_uid,'task':'predict','duration':dt }
log_entry_durations.update( rc.getDurations() )
meta = {'log_entry_durations':log_entry_durations}
params['Xd'] = Xd
params['num_reported_answers'] = num_reported_answers
log_entry = { 'exp_uid':exp_uid,'alg_uid':alg_uid,'timestamp':utils.datetimeNow() }
log_entry.update( params )
ell.log( app_id+':ALG-EVALUATION', log_entry )
params['timestamp'] = str(log_entry['timestamp'])
response_args_dict = params
elif predict_id=='get_queries':
# get list of triplets from test
queries,didSucceed,message = db.get_docs_with_filter(app_id+':queries',{'alg_uid':alg_uid})
S = []
for query in queries:
if 'q' in query.keys():
q = query['q']
S.append(q)
params['queries'] = S
params['num_reported_answers'] = len(S)
response_args_dict = params
args_out = {'args':response_args_dict,'meta':meta}
predict_json = json.dumps(args_out)
log_entry = { 'exp_uid':exp_uid,'task':'predict','json':predict_json,'timestamp':utils.datetimeNow() }
ell.log( app_id+':APP-RESPONSE', log_entry )
return predict_json,True,''
except Exception, err:
error = traceback.format_exc()
log_entry = { 'exp_uid':exp_uid,'task':'predict','error':str(error),'timestamp':utils.datetimeNow() }
didSucceed,message = ell.log( app_id+':APP-EXCEPTION', log_entry )
return '{}',False,error
def processAnswer(self, exp_uid, args_json, db, ell):
"""
reporting back the reward of pulling the arm suggested by getQuery
Expected input (in json structure with string keys):
(str) query_uid : unique identifier of query
(int) index_winner : index of arm must be {index_left,index_right}
Expected output (comma separated):
if error:
return (JSON) '{}', (bool) False, (str) error
else:
return (JSON) '{}', (bool) True,''
Usage:
processAnswer_args_json,didSucceed,message = app.processAnswer(db_API,exp_uid,processAnswer_args_json)
Example input:
processAnswer_args_json = {"query_uid": "4d02a9924f92138287edd17ca5feb6e1", "index_winner": 8}
Example output:
processAnswer_response_json = {}
"""
try:
app_id = self.app_id
log_entry = {
'exp_uid': exp_uid,
'task': 'processAnswer',
'json': args_json,
'timestamp': utils.datetimeNow()
}
ell.log(app_id + ':APP-CALL', log_entry)
# convert args_json to args_dict
try:
args_dict = json.loads(args_json)
except:
error = "%s.processAnswer input args_json is in improper format" % self.app_id
return '{}', False, error
# check for the fields that must be contained in args or error occurs
necessary_fields = ['index_winner', 'query_uid']
for field in necessary_fields:
try:
args_dict[field]
except KeyError:
error = "%s.processAnswer input arguments missing field: %s" % (
self.app_id, str(field))
return '{}', False, error
# get list of algorithms associated with project
alg_list, didSucceed, message = db.get(app_id + ':experiments',
exp_uid, 'alg_list')
# get alg_id
query_uid = args_dict['query_uid']
alg_uid, didSucceed, message = db.get(app_id + ':queries',
query_uid, 'alg_uid')
for algorithm in alg_list:
if alg_uid == algorithm['alg_uid']:
alg_id = algorithm['alg_id']
alg_label = algorithm['alg_label']
num_reported_answers, didSucceed, message = db.increment(
app_id + ':experiments', exp_uid,
'num_reported_answers_for_' + alg_uid)
# get sandboxed database for the specific app_id,alg_id,exp_uid - closing off the rest of the database to the algorithm
rc = ResourceClient(app_id, exp_uid, alg_uid, db)
# get specific algorithm to make calls to
alg = utils.get_app_alg(self.app_id, alg_id)
targets, didSucceed, message = db.get(app_id + ':queries',
query_uid, 'target_indices')
for target in targets:
if target['label'] == 'left':
index_left = target['index']
if target['label'] == 'right':
index_right = target['index']
if target['flag'] == 1:
index_painted = target['index']
index_winner = args_dict['index_winner']
# update query doc
timestamp_query_generated, didSucceed, message = db.get(
app_id + ':queries', query_uid, 'timestamp_query_generated')
datetime_query_generated = utils.str2datetime(
timestamp_query_generated)
timestamp_answer_received = args_dict.get('meta', {}).get(
'timestamp_answer_received', None)
if timestamp_answer_received == None:
datetime_answer_received = datetime_query_generated
else:
datetime_answer_received = utils.str2datetime(
timestamp_answer_received)
delta_datetime = datetime_answer_received - datetime_query_generated
round_trip_time = delta_datetime.seconds + delta_datetime.microseconds / 1000000.
response_time = float(args_dict.get('response_time', 0.))
db.set(app_id + ':queries', query_uid, 'response_time',
response_time)
db.set(app_id + ':queries', query_uid, 'network_delay',
round_trip_time - response_time)
db.set(app_id + ':queries', query_uid, 'index_winner',
index_winner)
# call processAnswer
didSucceed, dt = utils.timeit(alg.processAnswer)(
resource=rc,
index_left=index_left,
index_right=index_right,
index_painted=index_painted,
index_winner=index_winner)
log_entry_durations = {
'exp_uid': exp_uid,
'alg_uid': alg_uid,
'task': 'processAnswer',
'duration': dt
}
log_entry_durations.update(rc.getDurations())
meta = {'log_entry_durations': log_entry_durations}
###############
predict_id = 'arm_ranking'
params = {'alg_label': alg_label}
predict_args_dict = {'predict_id': predict_id, 'params': params}
predict_args_json = json.dumps(predict_args_dict)
db.submit_job(app_id,
exp_uid,
'predict',
predict_args_json,
ignore_result=True)
###############
response_args_dict = {}
args_out = {'args': response_args_dict, 'meta': meta}
response_json = json.dumps(args_out)
log_entry = {
'exp_uid': exp_uid,
'task': 'processAnswer',
'json': response_json,
'timestamp': utils.datetimeNow()
}
ell.log(app_id + ':APP-RESPONSE', log_entry)
return response_json, True, ""
except Exception, err:
error = traceback.format_exc()
log_entry = {
'exp_uid': exp_uid,
'task': 'processAnswer',
'error': error,
'timestamp': utils.datetimeNow(),
'args_json': args_json
}
ell.log(app_id + ':APP-EXCEPTION', log_entry)
return '{}', False, error
def getStats(self, exp_uid, args_json, db, ell):
"""
Get statistics for the experiment and its algorithms
Expected input (in json structure with string keys):
(string) stat_id : identifier for the desired statistic
(dict) params : dictionary of stat_id specific fields.
See Next.utils.get_app_supported_stats(app_id) ResourceManager.get_app_supported_stats(app_id)
for a description of each of the available stats and their inputs and outputs
"""
try:
app_id = self.app_id
log_entry = {
'exp_uid': exp_uid,
'task': 'getStats',
'json': args_json,
'timestamp': utils.datetimeNow()
}
ell.log(app_id + ':APP-CALL', log_entry)
# convert args_json to args_dict
try:
args_dict = json.loads(args_json)
except:
error = "%s.getStats input args_json is in improper format" % self.app_id
return '{}', False, error
# check for the fields that must be contained in args or error occurs
necessary_fields = ['stat_id', 'params']
for field in necessary_fields:
try:
args_dict[field]
except KeyError:
error = "%s.getStats input arguments missing field: %s" % (
self.app_id, str(field))
return '{}', False, error
stat_id = args_dict['stat_id']
params = args_dict['params']
dashboard = DuelingBanditsPureExplorationDashboard(db, ell)
# input task
if stat_id == "api_activity_histogram":
task = params['task']
activity_stats = dashboard.api_activity_histogram(
self.app_id, exp_uid, task)
stats = activity_stats
# input Noneokay
elif stat_id == "api_processAnswer_activity_stacked_histogram":
activity_stats = dashboard.api_processAnswer_activity_stacked_histogram(
self.app_id, exp_uid)
stats = activity_stats
# input task
elif stat_id == "compute_duration_multiline_plot":
task = params['task']
compute_stats = dashboard.compute_duration_multiline_plot(
self.app_id, exp_uid, task)
stats = compute_stats
# input task, alg_label
elif stat_id == "compute_duration_detailed_stacked_area_plot":
task = params['task']
alg_label = params['alg_label']
compute_detailed_stats = dashboard.compute_duration_detailed_stacked_area_plot(
self.app_id, exp_uid, task, alg_label)
stats = compute_detailed_stats
# input alg_label
elif stat_id == "response_time_histogram":
alg_label = params['alg_label']
response_time_stats = dashboard.response_time_histogram(
self.app_id, exp_uid, alg_label)
stats = response_time_stats
# input alg_label
elif stat_id == "network_delay_histogram":
alg_label = params['alg_label']
network_delay_stats = dashboard.network_delay_histogram(
self.app_id, exp_uid, alg_label)
stats = network_delay_stats
elif stat_id == "most_current_ranking":
alg_label = params['alg_label']
stats = dashboard.most_current_ranking(self.app_id, exp_uid,
alg_label)
response_json = json.dumps(stats)
log_entry = {
'exp_uid': exp_uid,
'task': 'getStats',
'json': response_json,
'timestamp': utils.datetimeNow()
}
ell.log(app_id + ':APP-RESPONSE', log_entry)
return response_json, True, ''
except Exception, err:
error = traceback.format_exc()
log_entry = {
'exp_uid': exp_uid,
'task': 'getStats',
'error': error,
'timestamp': utils.datetimeNow(),
'args_json': args_json
}
ell.log(app_id + ':APP-EXCEPTION', log_entry)
return '{}', False, error
def reportAnswer(self,exp_uid,args_json,db,ell):
"""
reporting back the reward of pulling the arm suggested by getQuery
Expected input:
(str) query_uid : unique identifier of query
(int) index_winner : index of arm must be one of the indices given by getQuery
Expected output (comma separated):
if error:
return (JSON) '{}', (bool) False, (str) error
else:
return (JSON) '{}', (bool) True,''
Usage:
reportAnswer_args_json,didSucceed,message = app.reportAnswer(exp_uid,reportAnswer_args_json)
Example input:
reportAnswer_args_json = {"query_uid": "4d02a9924f92138287edd17ca5feb6e1", "index_winner": 8}
Example output:
reportAnswer_response_json = {}
"""
try:
app_id = self.app_id
log_entry = { 'exp_uid':exp_uid,'task':'reportAnswer','json':args_json,'timestamp':utils.datetimeNow() }
ell.log( app_id+':APP-CALL', log_entry )
# convert args_json to args_dict
try:
args_dict = json.loads(args_json)
except:
error = "%s.reportAnswer input args_json is in improper format" % self.app_id
return '{}',False,error
# check for the fields that must be contained in args or error occurs
necessary_fields = ['index_winner','query_uid']
for field in necessary_fields:
try:
args_dict[field]
except KeyError:
error = "%s.reportAnswer input arguments missing field: %s" % (self.app_id,str(field))
return '{}',False,error
# get list of algorithms associated with project
alg_list,didSucceed,message = db.get(app_id+':experiments',exp_uid,'alg_list')
# get alg_id
query_uid = args_dict['query_uid']
alg_uid,didSucceed,message = db.get(app_id+':queries',query_uid,'alg_uid')
for algorithm in alg_list:
if alg_uid == algorithm['alg_uid']:
alg_id = algorithm['alg_id']
alg_label = algorithm['alg_label']
num_reported_answers,didSucceed,message = db.increment(app_id+':experiments',exp_uid,'num_reported_answers_for_'+alg_uid)
# get sandboxed database for the specific app_id,alg_id,exp_uid - closing off the rest of the database to the algorithm
rc = ResourceClient(app_id,exp_uid,alg_uid,db)
# get specific algorithm to make calls to
alg = utils.get_app_alg(self.app_id,alg_id)
# get targets associated with the specific query
targets,didSucceed,message = db.get(app_id+':queries',query_uid,'target_indices')
target_indices = []
for target in targets:
target_indices.append(target['index'])
# get the index winner
index_winner = args_dict['index_winner']
# update query doc
timestamp_query_generated,didSucceed,message = db.get(app_id+':queries',query_uid,'timestamp_query_generated')
datetime_query_generated = utils.str2datetime(timestamp_query_generated)
timestamp_answer_received = args_dict.get('meta',{}).get('timestamp_answer_received',None)
if timestamp_answer_received == None:
datetime_answer_received = datetime_query_generated
else:
datetime_answer_received = utils.str2datetime(timestamp_answer_received)
delta_datetime = datetime_answer_received - datetime_query_generated
round_trip_time = delta_datetime.seconds + delta_datetime.microseconds/1000000.
response_time = float(args_dict.get('response_time',0.))
db.set(app_id+':queries',query_uid,'response_time',response_time)
db.set(app_id+':queries',query_uid,'network_delay',round_trip_time-response_time)
db.set(app_id+':queries',query_uid,'index_winner',index_winner)
# call reportAnswer
didSucceed,dt = utils.timeit(alg.reportAnswer)(resource=rc,targets=target_indices,index_winner=index_winner)
log_entry_durations = { 'exp_uid':exp_uid,'alg_uid':alg_uid,'task':'reportAnswer','duration':dt }
log_entry_durations.update( rc.getDurations() )
meta = {'log_entry_durations':log_entry_durations}
# calling predict
###############
predict_id = 'arm_ranking'
params = {'alg_label':alg_label}
predict_args_dict = {'predict_id':predict_id,'params':params}
predict_args_json = json.dumps(predict_args_dict)
db.submit_job(app_id,exp_uid,'predict',predict_args_json,ignore_result=True)
###############
response_args_dict = {}
args_out = {'args':response_args_dict,'meta':meta}
response_json = json.dumps(args_out)
log_entry = { 'exp_uid':exp_uid,'task':'reportAnswer','json':response_json,'timestamp':utils.datetimeNow() }
ell.log( app_id+':APP-RESPONSE', log_entry )
return response_json,True,""
except Exception, err:
error = traceback.format_exc()
log_entry = { 'exp_uid':exp_uid,'task':'reportAnswer','error':error,'timestamp':utils.datetimeNow(),'args_json':args_json }
ell.log( app_id+':APP-EXCEPTION', log_entry )
return '{}',False,error
def daemonProcess(self,exp_uid,args_json,db,ell):
try:
app_id = self.app_id
log_entry = { 'exp_uid':exp_uid,'task':'daemonProcess','json':args_json,'timestamp':utils.datetimeNow() }
ell.log( app_id+':APP-CALL', log_entry )
# convert args_json to args_dict
try:
args_dict = json.loads(args_json)
except:
error = "%s.daemonProcess input args_json is in improper format" % self.app_id
return '{}',False,error
# check for the fields that must be contained in args or error occurs
necessary_fields = ['alg_uid','daemon_args']
for field in necessary_fields:
try:
args_dict[field]
except KeyError:
error = "%s.daemonProcess input arguments missing field: %s" % (self.app_id,str(field))
return '{}',False,error
alg_daemon_args = args_dict['daemon_args']
alg_uid = args_dict['alg_uid']
alg_id,didSucceed,message = db.get(app_id+':algorithms',alg_uid,'alg_id')
# get sandboxed database for the specific app_id,alg_id,exp_uid - closing off the rest of the database to the algorithm
rc = ResourceClient(app_id,exp_uid,alg_uid,db)
# get specific algorithm to make calls to
alg = utils.get_app_alg(self.app_id,alg_id)
didSucceed,dt = utils.timeit(alg.daemonProcess)(resource=rc,daemon_args_dict=alg_daemon_args)
log_entry_durations = { 'exp_uid':exp_uid,'alg_uid':alg_uid,'task':'daemonProcess','duration':dt }
log_entry_durations.update( rc.getDurations() )
meta = {'log_entry_durations':log_entry_durations}
daemon_message = {}
args_out = {'args':daemon_message,'meta':meta}
response_json = json.dumps(args_out)
log_entry = { 'exp_uid':exp_uid,'task':'daemonProcess','json':response_json,'timestamp':utils.datetimeNow() }
ell.log( app_id+':APP-RESPONSE', log_entry )
return response_json,True,''
except Exception, err:
error = traceback.format_exc()
log_entry = { 'exp_uid':exp_uid,'task':'daemonProcess','error':error,'timestamp':utils.datetimeNow(),'args_json':args_json }
ell.log( app_id+':APP-EXCEPTION', log_entry )
return '{}',False,error
def daemonProcess(self, exp_uid, args_json, db, ell):
try:
app_id = self.app_id
log_entry = {
'exp_uid': exp_uid,
'task': 'daemonProcess',
'json': args_json,
'timestamp': utils.datetimeNow()
}
ell.log(app_id + ':APP-CALL', log_entry)
# convert args_json to args_dict
try:
args_dict = json.loads(args_json)
except:
error = "%s.daemonProcess input args_json is in improper format" % self.app_id
return '{}', False, error
# check for the fields that must be contained in args or error occurs
necessary_fields = ['alg_uid', 'daemon_args']
for field in necessary_fields:
try:
args_dict[field]
except KeyError:
error = "%s.daemonProcess input arguments missing field: %s" % (
self.app_id, str(field))
return '{}', False, error
alg_daemon_args = args_dict['daemon_args']
alg_uid = args_dict['alg_uid']
alg_id, didSucceed, message = db.get(app_id + ':algorithms',
alg_uid, 'alg_id')
# get sandboxed database for the specific app_id,alg_id,exp_uid - closing off the rest of the database to the algorithm
# rc = ResourceClient(app_id,exp_uid,alg_uid,db)
rc = ResourceClient(app_id, exp_uid, alg_uid, db)
# get specific algorithm to make calls to
alg = utils.get_app_alg(self.app_id, alg_id)
didSucceed, dt = utils.timeit(alg.daemonProcess)(
resource=rc, daemon_args_dict=alg_daemon_args)
log_entry_durations = {
'exp_uid': exp_uid,
'alg_uid': alg_uid,
'task': 'daemonProcess',
'duration': dt
}
log_entry_durations.update(rc.getDurations())
meta = {'log_entry_durations': log_entry_durations}
daemon_message = {}
args_out = {'args': daemon_message, 'meta': meta}
response_json = json.dumps(args_out)
log_entry = {
'exp_uid': exp_uid,
'task': 'daemonProcess',
'json': response_json,
'timestamp': utils.datetimeNow()
}
ell.log(app_id + ':APP-RESPONSE', log_entry)
return response_json, True, ''
except Exception, err:
error = traceback.format_exc()
log_entry = {
'exp_uid': exp_uid,
'task': 'daemonProcess',
'error': error,
'timestamp': utils.datetimeNow(),
'args_json': args_json
}
ell.log(app_id + ':APP-EXCEPTION', log_entry)
return '{}', False, error
def getStats(self,exp_uid,args_json,db,ell):
"""
Get statistics for the experiment and its algorithms
Expected input (in json structure with string keys):
(string) stat_id : identifier for the desired statistic
(dict) params : dictionary of stat_id specific fields.
See Next.utils.get_app_supported_stats(app_id) ResourceManager.get_app_supported_stats(app_id)
for a description of each of the available stats and their inputs and outputs
"""
try:
app_id = self.app_id
log_entry = { 'exp_uid':exp_uid,'task':'getStats','json':args_json,'timestamp':utils.datetimeNow() }
ell.log( app_id+':APP-CALL', log_entry )
# convert args_json to args_dict
try:
args_dict = json.loads(args_json)
except:
error = "%s.getStats input args_json is in improper format" % self.app_id
return '{}',False,error
# check for the fields that must be contained in args or error occurs
necessary_fields = ['stat_id','params']
for field in necessary_fields:
try:
args_dict[field]
except KeyError:
error = "%s.getStats input arguments missing field: %s" % (self.app_id,str(field))
return '{}',False,error
stat_id = args_dict['stat_id']
params = args_dict['params']
dashboard = TupleBanditsPureExplorationDashboard(db,ell)
# input task
if stat_id == "api_activity_histogram":
task = params['task']
activity_stats = dashboard.api_activity_histogram(self.app_id,exp_uid,task)
stats = activity_stats
# input Noneokay
elif stat_id == "api_reportAnswer_activity_stacked_histogram":
activity_stats = dashboard.api_reportAnswer_activity_stacked_histogram(self.app_id,exp_uid)
stats = activity_stats
# input task
elif stat_id == "compute_duration_multiline_plot":
task = params['task']
compute_stats = dashboard.compute_duration_multiline_plot(self.app_id,exp_uid,task)
stats = compute_stats
# input task, alg_label
elif stat_id == "compute_duration_detailed_stacked_area_plot":
task = params['task']
alg_label = params['alg_label']
compute_detailed_stats = dashboard.compute_duration_detailed_stacked_area_plot(self.app_id,exp_uid,task,alg_label)
stats = compute_detailed_stats
# input alg_label
elif stat_id == "response_time_histogram":
alg_label = params['alg_label']
response_time_stats = dashboard.response_time_histogram(self.app_id,exp_uid,alg_label)
stats = response_time_stats
# input alg_label
elif stat_id == "network_delay_histogram":
alg_label = params['alg_label']
network_delay_stats = dashboard.network_delay_histogram(self.app_id,exp_uid,alg_label)
stats = network_delay_stats
elif stat_id == "most_current_ranking":
alg_label = params['alg_label']
stats = dashboard.most_current_ranking(self.app_id,exp_uid,alg_label)
response_json = json.dumps(stats)
log_entry = { 'exp_uid':exp_uid,'task':'getStats','json':response_json,'timestamp':utils.datetimeNow() }
ell.log( app_id+':APP-RESPONSE', log_entry )
return response_json,True,''
except Exception, err:
error = traceback.format_exc()
log_entry = { 'exp_uid':exp_uid,'task':'getStats','error':error,'timestamp':utils.datetimeNow(),'args_json':args_json }
ell.log( app_id+':APP-EXCEPTION', log_entry )
return '{}',False,error
def initExp(self, exp_uid, args_json, db, ell):
"""
initialize the project and necessary experiments
Expected input (in json structure with string keys):
(int) n: number of arms
(float) failure_probability : confidence
[optional] (list of dicts) alg_list : with fields (Defaults given by Info.get_app_default_alg_list)
(string) alg_id : valid alg_id for this app_id
(string) alg_label : unique identifier for algorithm (e.g. may have experiment with repeated alg_id's, but alg_labels must be unqiue, will also be used for plot legends
[optional] (string) test_alg_label : must be one of the alg_label's in alg_list (Default is self)
[optional] (dict) algorithm_management_settings : dictionary with fields (string) 'mode' and (dict) 'params'. mode in {'pure_exploration','explore_exploit','fixed_proportions'}. Default is 'fixed_proportions' and allocates uniform probability to each algorithm. If mode=fixed_proportions then params is a dictionary that contains the field 'proportions' which is a list of dictionaries with fields 'alg_label' and 'proportion' for all algorithms in alg_list. All proportions must be positive and sum to 1 over all algs in alg_list
[optional] (string) participant_to_algorithm_management : in {'one_to_one','one_to_many'}. Default is 'one_to_many'.
[optional] (string) instructions
[optional] (string) debrief
[optional] (int) num_tries
Expected output:
if error:
return (JSON) '{}', (bool) False, (str) error_str
else:
return (JSON) '{}', (bool) True,''
Usage:
initExp_response_json,didSucceed,message = app.initExp(db_API,exp_uid,initExp_args_json)
Example input:
initExp_args_json = {"participant_to_algorithm_management": "one_to_many", "alg_list": [{"alg_label": "BR_LilUCB", "alg_id": "BR_LilUCB", "params": {}}], "algorithm_management_settings": {"params": {"proportions": [{"alg_label": "BR_LilUCB", "proportion": 1.0}]}, "mode": "fixed_proportions"}, "failure_probability": 0.01, "n": 10}
Example output:
initExp_response_json = {}
"""
try:
app_id = self.app_id
# remove any reminants of an experiment if it exists
didSucceed, message = db.delete_docs_with_filter(
'experiments_admin', {'exp_uid': exp_uid})
didSucceed, message = db.delete_docs_with_filter(
app_id + ':experiments', {'exp_uid': exp_uid})
didSucceed, message = db.delete_docs_with_filter(
app_id + ':queries', {'exp_uid': exp_uid})
didSucceed, message = db.delete_docs_with_filter(
app_id + ':participants', {'exp_uid': exp_uid})
didSucceed, message = db.delete_docs_with_filter(
app_id + ':algorithms', {'exp_uid': exp_uid})
didSucceed, message = ell.delete_logs_with_filter(
app_id + ':APP-CALL', {'exp_uid': exp_uid})
didSucceed, message = ell.delete_logs_with_filter(
app_id + ':APP-RESPONSE', {'exp_uid': exp_uid})
didSucceed, message = ell.delete_logs_with_filter(
app_id + ':APP-EXCEPTION', {'exp_uid': exp_uid})
didSucceed, message = ell.delete_logs_with_filter(
app_id + ':ALG-DURATION', {'exp_uid': exp_uid})
didSucceed, message = ell.delete_logs_with_filter(
app_id + ':ALG-EVALUATION', {'exp_uid': exp_uid})
# add indexes (only adds them if they do not already exist)
didSucceed, message = db.ensure_index('experiments_admin',
{'exp_uid': 1})
didSucceed, message = db.ensure_index(app_id + ':experiments',
{'exp_uid': 1})
didSucceed, message = db.ensure_index(app_id + ':queries',
{'query_uid': 1})
didSucceed, message = db.ensure_index(app_id + ':queries',
{'exp_uid': 1})
didSucceed, message = db.ensure_index(app_id + ':queries',
{'alg_uid': 1})
didSucceed, message = db.ensure_index(app_id + ':queries',
{'participant_uid': 1})
didSucceed, message = db.ensure_index(app_id + ':participants',
{'participant_uid': 1})
didSucceed, message = db.ensure_index(app_id + ':participants',
{'exp_uid': 1})
didSucceed, message = db.ensure_index(app_id + ':algorithms',
{'alg_uid': 1})
didSucceed, message = db.ensure_index(app_id + ':algorithms',
{'exp_uid': 1})
didSucceed, message = ell.ensure_index(app_id + ':APP-CALL',
{'exp_uid': 1})
didSucceed, message = ell.ensure_index(app_id + ':APP-CALL',
{'timestamp': 1})
didSucceed, message = ell.ensure_index(app_id + ':APP-CALL', {
'exp_uid': 1,
'timestamp': 1
})
didSucceed, message = ell.ensure_index(app_id + ':APP-CALL', {
'exp_uid': 1,
'task': 1
})
didSucceed, message = ell.ensure_index(app_id + ':APP-RESPONSE',
{'exp_uid': 1})
didSucceed, message = ell.ensure_index(app_id + ':APP-RESPONSE',
{'timestamp': 1})
didSucceed, message = ell.ensure_index(app_id + ':APP-RESPONSE', {
'exp_uid': 1,
'timestamp': 1
})
didSucceed, message = ell.ensure_index(app_id + ':APP-RESPONSE', {
'exp_uid': 1,
'task': 1
})
didSucceed, message = ell.ensure_index(app_id + ':APP-EXCEPTION',
{'exp_uid': 1})
didSucceed, message = ell.ensure_index(app_id + ':APP-EXCEPTION',
{'timestamp': 1})
didSucceed, message = ell.ensure_index(app_id + ':APP-EXCEPTION', {
'exp_uid': 1,
'timestamp': 1
})
didSucceed, message = ell.ensure_index(app_id + ':APP-EXCEPTION', {
'exp_uid': 1,
'task': 1
})
didSucceed, message = ell.ensure_index(app_id + ':ALG-DURATION',
{'exp_uid': 1})
didSucceed, message = ell.ensure_index(app_id + ':ALG-DURATION',
{'alg_uid': 1})
didSucceed, message = ell.ensure_index(app_id + ':ALG-DURATION',
{'timestamp': 1})
didSucceed, message = ell.ensure_index(app_id + ':ALG-DURATION', {
'exp_uid': 1,
'timestamp': 1
})
didSucceed, message = ell.ensure_index(app_id + ':ALG-DURATION', {
'alg_uid': 1,
'task': 1
})
didSucceed, message = ell.ensure_index(app_id + ':ALG-EVALUATION',
{'exp_uid': 1})
didSucceed, message = ell.ensure_index(app_id + ':ALG-EVALUATION',
{'alg_uid': 1})
didSucceed, message = ell.ensure_index(app_id + ':ALG-EVALUATION',
{'timestamp': 1})
didSucceed, message = ell.ensure_index(app_id + ':ALG-EVALUATION',
{
'exp_uid': 1,
'timestamp': 1
})
db.set('experiments_admin', exp_uid, 'exp_uid', exp_uid)
db.set('experiments_admin', exp_uid, 'app_id', app_id)
db.set('experiments_admin', exp_uid, 'start_date',
utils.datetime2str(utils.datetimeNow()))
log_entry = {
'exp_uid': exp_uid,
'task': 'initExp',
'json': args_json,
'timestamp': utils.datetimeNow()
}
ell.log(app_id + ':APP-CALL', log_entry)
# convert args_json to args_dict
try:
args_dict = json.loads(args_json)
except:
error = "%s.initExp input args_json is in improper format" % self.app_id
return '{}', False, error
# check for the fields that must be contained in args or error occurs
necessary_fields = ['n', 'failure_probability']
for field in necessary_fields:
try:
args_dict[field]
except KeyError:
error = "%s.initExp input arguments missing field: %s" % (
self.app_id, str(field))
return '{}', False, error
n = args_dict['n']
delta = args_dict['failure_probability']
if 'alg_list' in args_dict:
alg_list = args_dict['alg_list']
supportedAlgs = utils.get_app_supported_algs(self.app_id)
for algorithm in alg_list:
if algorithm['alg_id'] not in supportedAlgs:
error = "%s.initExp unsupported algorithm '%s' in alg_list" % (
self.app_id, alg_id)
return '{}', False, error
else:
alg_list = utils.get_app_default_alg_list(self.app_id)
if 'instructions' not in args_dict:
instructions = utils.get_app_default_instructions(app_id)
else:
instructions = args_dict['instructions']
if 'debrief' not in args_dict:
debrief = utils.get_app_default_instructions(app_id)
else:
debrief = args_dict['debrief']
if 'num_tries' not in args_dict:
num_tries = utils.get_app_default_num_tries(app_id)
else:
num_tries = args_dict['num_tries']
if 'context_type' not in args_dict:
context_type = 'none'
else:
context_type = args_dict['context_type']
if 'context' not in args_dict:
context = ''
else:
context = args_dict['context']
# ALGORITHM_MANAGEMENT_MODE FORMATTING CHECK
if 'algorithm_management_settings' not in args_dict:
params = {}
params['proportions'] = []
for algorithm in alg_list:
params['proportions'].append({
'alg_label':
algorithm['alg_label'],
'proportion':
1. / len(alg_list)
})
algorithm_management_settings = {}
algorithm_management_settings['mode'] = 'fixed_proportions'
algorithm_management_settings['params'] = params
else:
algorithm_management_settings = args_dict[
'algorithm_management_settings']
try:
mode = algorithm_management_settings['mode']
params = algorithm_management_settings['params']
except:
error = "%s.initExp algorithm_management_settings must be a dictionary with fields 'mode' and 'params'" % (
self.app_id)
return '{}', False, error
if mode == 'fixed_proportions':
try:
algorithm_proportions_list = params['proportions']
except:
error = "%s.initExp algorithm_management_settings['params'] must be a dictionary with field 'proportions'" % (
self.app_id)
return '{}', False, error
# check if alg_labels are properly labeled
for proportion_item in algorithm_proportions_list:
proportion = proportion_item['proportion']
target_alg_label = proportion_item['alg_label']
target_alg_label_in_alg_list = False
for algorithm in alg_list:
if algorithm['alg_label'] == target_alg_label:
target_alg_label_in_alg_list = True
if not target_alg_label_in_alg_list:
error = "%s.initExp algorithm_management_settings['params']['proportions'] must be a list of dictionaries, each dictionary containing the fields 'alg_label' and 'proportion'. The 'alg_label' value must be one of the alg_labels in a provided alg_list and 'proportion' must be nonnegative and sum to 1 : '%s' not in provided alg_list" % (
self.app_id, target_alg_label)
return '{}', False, error
elif mode == 'pure_exploration':
error = "%s.initExp Sorry, '%s' is not yet supported." % (
self.app_id, mode)
return '{}', False, error
elif mode == 'explore_exploit':
error = "%s.initExp Sorry, '%s' is not yet supported." % (
self.app_id, mode)
return '{}', False, error
else:
error = "%s.initExp unsupported algorithm_management_mode: '%s'. Must be in {'pure_exploration','explore_exploit','fixed_proportions'}" % (
self.app_id, algorithm_management_mode)
return '{}', False, error
# ALGORITHM_MANAGEMENT_MODE FORMATTING CHECK
if 'participant_to_algorithm_management' not in args_dict:
participant_to_algorithm_management = 'one_to_many'
else:
participant_to_algorithm_management = args_dict[
'participant_to_algorithm_management']
if participant_to_algorithm_management not in [
'one_to_many', 'one_to_one'
]:
error = "%s.initExp unsupported participant_to_algorithm_management: '%s'. Must be in {'one_to_many','one_to_one'}" % (
self.app_id, participant_to_algorithm_management)
return '{}', False, error
# assign uid to each algorithm and save it
for algorithm in alg_list:
alg_uid = utils.getNewUID()
algorithm['alg_uid'] = alg_uid
db.set(app_id + ':algorithms', alg_uid, 'alg_uid', alg_uid)
db.set(app_id + ':algorithms', alg_uid, 'exp_uid', exp_uid)
db.set(app_id + ':experiments', exp_uid, 'exp_uid', exp_uid)
db.set(app_id + ':experiments', exp_uid, 'app_id', app_id)
db.set(app_id + ':experiments', exp_uid, 'n', n)
db.set(app_id + ':experiments', exp_uid, 'failure_probability',
delta)
db.set(app_id + ':experiments', exp_uid, 'alg_list', alg_list)
db.set(app_id + ':experiments', exp_uid,
'algorithm_management_settings',
algorithm_management_settings)
db.set(app_id + ':experiments', exp_uid,
'participant_to_algorithm_management',
participant_to_algorithm_management)
db.set(app_id + ':experiments', exp_uid, 'instructions',
instructions)
db.set(app_id + ':experiments', exp_uid, 'debrief', debrief)
db.set(app_id + ':experiments', exp_uid, 'context_type',
context_type)
db.set(app_id + ':experiments', exp_uid, 'context', context)
db.set(app_id + ':experiments', exp_uid, 'num_tries', num_tries)
# now create intitialize each algorithm
for algorithm in alg_list:
alg_id = algorithm['alg_id']
alg_uid = algorithm['alg_uid']
db.set(app_id + ':algorithms', alg_uid, 'alg_id', alg_id)
db.set(app_id + ':algorithms', alg_uid, 'alg_uid', alg_uid)
db.set(app_id + ':algorithms', alg_uid, 'exp_uid', exp_uid)
# get sandboxed database for the specific app_id,alg_id,exp_uid - closing off the rest of the database to the algorithm
rc = ResourceClient(app_id, exp_uid, alg_uid, db)
# get specific algorithm to make calls to
alg = utils.get_app_alg(self.app_id, alg_id)
# call initExp
didSucceed, dt = utils.timeit(alg.initExp)(
resource=rc, n=n, failure_probability=delta)
log_entry = {
'exp_uid': exp_uid,
'alg_uid': alg_uid,
'task': 'initExp',
'duration': dt,
'timestamp': utils.datetimeNow()
}
ell.log(app_id + ':ALG-DURATION', log_entry)
response_json = '{}'
log_entry = {
'exp_uid': exp_uid,
'task': 'initExp',
'json': response_json,
'timestamp': utils.datetimeNow()
}
ell.log(app_id + ':APP-RESPONSE', log_entry)
return response_json, True, ''
except Exception, err:
error = traceback.format_exc()
log_entry = {
'exp_uid': exp_uid,
'task': 'initExp',
'error': error,
'timestamp': utils.datetimeNow(),
'args_json': args_json
}
ell.log(app_id + ':APP-EXCEPTION', log_entry)
return '{}', False, error
#!/usr/bin/python
import time
import traceback
import next.utils as utils
import next.broker.broker
broker = next.broker.broker.JobBroker()
while (1):
timestamp = utils.datetime2str(utils.datetimeNow())
try:
broker.refresh_domain_hashes()
except Exception, err:
error = traceback.format_exc()
print error
# wait
time.sleep(2)
def processAnswer(self, exp_uid, args_json):
try:
args_dict = self.helper.convert_json(args_json)
args_dict = verifier.verify(args_dict, self.reference_dict['processAnswer']['args'])
# Update timing info in query
query = self.butler.queries.get(uid=args_dict['args']['query_uid'])
timestamp_answer_received = args_dict['args'].get('timestamp_answer_received', None)
delta_datetime = utils.str2datetime(timestamp_answer_received) - \
utils.str2datetime(query['timestamp_query_generated'])
round_trip_time = delta_datetime.total_seconds()
response_time = float(args_dict['args'].get('response_time',0.))
query_update = self.call_app_fn(query['alg_label'], query['alg_id'], 'processAnswer', args_dict)
query_update.update({'response_time':response_time,
'network_delay':round_trip_time - response_time,
'timestamp_answer_received': timestamp_answer_received
})
self.butler.queries.set_many(uid=args_dict['args']['query_uid'],key_value_dict=query_update)
return json.dumps({'args': {}, 'meta': {'log_entry_durations':self.log_entry_durations}}), True, ''
except Exception, error:
exc_type, exc_value, exc_traceback = sys.exc_info()
full_error = str(traceback.format_exc())+'\n'+str(error)
utils.debug_print("processAnswer Exception: " + full_error, color='red')
log_entry = { 'exp_uid':exp_uid,'task':'processAnswer','error':full_error,'timestamp':utils.datetimeNow(),'args_json':args_json }
self.butler.ell.log( self.app_id+':APP-EXCEPTION', log_entry )
traceback.print_tb(exc_traceback)
raise Exception(error)
def getQuery(self,exp_uid,args_json,db,ell):
"""
A request to ask which k arms to duel next
Expected input (in jsonstructure with string keys):
(int) k : number of objects to display
[optional] (string) participant_uid : unique identifier of session for a participant answering questions (that is, an email address is not good enough as the participant could participate in multiple exp_uids so it would not be unique against all experiments), if key non-existant particpant_uid is assigned as exp_uid.
Expected output (in json structure with string keys):
(list) target_indices : list of k target indexes e.g. [ (int) target_index_1, ... , (int) target_index_k ]
(str) query_uid : unique identifier of query (used to look up for reportAnswer)
Usage:
getQuery_response_json,didSucceed,message = app.getQuery(exp_uid,getQuery_args_json)
Example input:
getQuery_args_json = {"k": 3, "participant_uid": "0077110d03cf06b8f77d11acc399e8a7"}
Example output:
getQuery_response_json = {"query_uid": "4d02a9924f92138287edd17ca5feb6e1", "target_indices": [ 3, 6, 9 ]
"""
try:
app_id = self.app_id
log_entry = { 'exp_uid':exp_uid,'task':'getQuery','json':args_json,'timestamp':utils.datetimeNow() }
ell.log( app_id+':APP-CALL', log_entry )
# convert args_json to args_dict
try:
args_dict = json.loads(args_json)
except:
error = "%s.initExp input args_json is in improper format" % self.app_id
return '{}',False,error
# get list of algorithms associated with project
alg_list,didSucceed,message = db.get(app_id+':experiments',exp_uid,'alg_list')
alg_label_to_alg_id = {}
alg_label_to_alg_uid = {}
for algorithm in alg_list:
alg_label_to_alg_id[ algorithm['alg_label'] ] = algorithm['alg_id']
alg_label_to_alg_uid[ algorithm['alg_label'] ] = algorithm['alg_uid']
algorithm_management_settings,didSucceed,message = db.get(app_id+':experiments',exp_uid,'algorithm_management_settings')
# ASSIGN ALGORITHM TO PARTICIPANT
if 'participant_uid' in args_dict:
participant_uid = args_dict['participant_uid']
else:
participant_uid = exp_uid
participant_doc_exists,didSucceed,message = db.exists(app_id+':participants',participant_uid,'participant_uid')
first_participant_query = not participant_doc_exists
if first_participant_query:
db.set(app_id+':participants',participant_uid,'participant_uid',participant_uid)
db.set(app_id+':participants',participant_uid,'exp_uid',exp_uid)
participant_to_algorithm_management,didSucceed,message = db.get(app_id+':experiments',exp_uid,'participant_to_algorithm_management')
if (participant_uid==exp_uid) or (participant_to_algorithm_management=='one_to_many') or (first_participant_query):
if algorithm_management_settings['mode']=='fixed_proportions':
proportions_list = algorithm_management_settings['params']['proportions']
prop = [ prop_item['proportion'] for prop_item in proportions_list ]
prop_item = numpy.random.choice(alg_list,p=prop)
else:
raise Exception('algorithm_management_mode : '+algorithm_management_settings['mode']+' not implemented')
alg_id = alg_label_to_alg_id[ prop_item['alg_label'] ]
alg_uid = alg_label_to_alg_uid[ prop_item['alg_label'] ]
alg_label = prop_item['alg_label']
if (first_participant_query) and (participant_to_algorithm_management=='one_to_one'):
db.set(app_id+':participants',participant_uid,'alg_id',alg_id)
db.set(app_id+':participants',participant_uid,'alg_uid',alg_uid)
elif (participant_to_algorithm_management=='one_to_one'):
# If here, then alg_uid should already be assigned in participant doc
alg_id,didSucceed,message = db.get(app_id+':participants',participant_uid,'alg_id')
alg_uid,didSucceed,message = db.get(app_id+':participants',participant_uid,'alg_uid')
else:
raise Exception('participant_to_algorithm_management : '+participant_to_algorithm_management+' not implemented')
# get sandboxed database for the specific app_id,alg_id,exp_uid - closing off the rest of the database to the algorithm
rc = ResourceClient(app_id,exp_uid,alg_uid,db)
# get specific algorithm to make calls to
alg = utils.get_app_alg(self.app_id,alg_id)
# call getQuery
targets,dt = utils.timeit(alg.getQuery)(resource=rc)
# update targets
target_list = []
for target in targets:
target_list.append({'index':target})
# check for context
context_type,didSucceed,message = db.get(app_id+':experiments',exp_uid,'context_type')
context,didSucceed,message = db.get(app_id+':experiments',exp_uid,'context')
# log
log_entry_durations = { 'exp_uid':exp_uid,'alg_uid':alg_uid,'task':'getQuery','duration':dt }
log_entry_durations.update( rc.getDurations() )
meta = {'log_entry_durations':log_entry_durations}
# create JSON query payload
timestamp = str(utils.datetimeNow())
query_uid = utils.getNewUID()
query = {}
query['query_uid'] = query_uid
query['target_indices'] = target_list
# save query data to database
query_doc = {}
query_doc.update(query)
query_doc['participant_uid'] = participant_uid
query_doc['alg_uid'] = alg_uid
query_doc['exp_uid'] = exp_uid
query_doc['alg_label'] = alg_label
query_doc['timestamp_query_generated'] = timestamp
for field in query_doc:
db.set(app_id+':queries',query_uid,field,query_doc[field])
# add context after updating query doc to avoid redundant information
query['context_type'] = context_type
query['context'] = context
args_out = {'args':query,'meta':meta}
response_json = json.dumps(args_out)
log_entry = { 'exp_uid':exp_uid,'task':'getQuery','json':response_json,'timestamp':utils.datetimeNow() }
ell.log( app_id+':APP-RESPONSE', log_entry )
return response_json,True,''
except Exception, err:
error = traceback.format_exc()
log_entry = { 'exp_uid':exp_uid,'task':'getQuery','error':error,'timestamp':utils.datetimeNow(),'args_json':args_json }
ell.log( app_id+':APP-EXCEPTION', log_entry )
return '{}',False,error
def init_alg(self, exp_uid, algorithm, alg_args):
butler = Butler(self.app_id, exp_uid, self.myApp.TargetManager, self.butler.db, self.butler.ell, algorithm['alg_label'], algorithm['alg_id'])
alg = utils.get_app_alg(self.app_id, algorithm['alg_id'])
if 'args' in self.algs_reference_dict['initExp']:
alg_args = verifier.verify(alg_args, self.algs_reference_dict['initExp']['args'])
# I got rid of a timeit function here; it wasn't handling the
# argument unpacking correctly? --Scott, 2016-3-7
# TODO: put dt back in and change log_entry to relfect that
alg_response = alg.initExp(butler, **alg_args)
alg_response = verifier.verify({'rets':alg_response}, {'rets':self.algs_reference_dict['initExp']['rets']})
log_entry = {'exp_uid':exp_uid, 'alg_label':algorithm['alg_label'], 'task':'initExp', 'duration':-1, 'timestamp':utils.datetimeNow()}
self.butler.log('ALG-DURATION', log_entry)
def predict(self,exp_uid,args_json,db,ell):
"""
Description: uses current model empirical estimates to forecast the ranking of the arms in order of likelhood of being the best from most to least likely
Expected input:
(string) predict_id : 'arm_ranking'
(dict) params : dictionary with fields
(string) alg_label : describes target algorithm to use
Expected output (in json structure):
(list of dicts with fields):
(int) index : index of target
(int) rank : rank that the algorithm assigns to index (rank 0 is most likely the best arm)
"""
try:
app_id = self.app_id
log_entry = { 'exp_uid':exp_uid,'task':'predict','json':args_json,'timestamp':utils.datetimeNow() }
ell.log( app_id+':APP-CALL', log_entry )
# convert args_json to args_dict
try:
args_dict = json.loads(args_json)
except:
error = "%s.predict failed to convert input args_json due to improper format" %(self.app_id)
return '{}',False,error
# check for the fields that must be contained in args or error occurs
necessary_fields = ['predict_id','params']
for field in necessary_fields:
try:
args_dict[field]
except KeyError:
error = "%s.predict input arguments missing field: %s" % (self.app_id,str(field))
return '{}',False,error
predict_id = args_dict['predict_id']
params = args_dict['params']
if predict_id == "arm_ranking":
alg_label = params['alg_label']
# get list of algorithms associated with project
alg_list,didSucceed,message = db.get(app_id+':experiments',exp_uid,'alg_list')
# get alg_id
for algorithm in alg_list:
if alg_label == algorithm['alg_label']:
alg_id = algorithm['alg_id']
alg_uid = algorithm['alg_uid']
num_reported_answers,didSucceed,message = db.get(app_id+':experiments',exp_uid,'num_reported_answers_for_'+alg_uid)
if type(num_reported_answers)!=int:
num_reported_answers=0
# get sandboxed database for the specific app_id,alg_id,exp_uid - closing off the rest of the database to the algorithm
rc = ResourceClient(app_id,exp_uid,alg_uid,db)
# get specific algorithm to make calls to
alg = utils.get_app_alg(self.app_id,alg_id)
# call getQuery
scores,precisions,dt = utils.timeit(alg.predict)(resource=rc)
log_entry_durations = { 'exp_uid':exp_uid,'alg_uid':alg_uid,'task':'predict','duration':dt }
log_entry_durations.update( rc.getDurations() )
meta = {'log_entry_durations':log_entry_durations}
import numpy
ranks = (-numpy.array(scores)).argsort().tolist()
n = len(scores)
indexes = numpy.array(range(n))[ranks]
scores = numpy.array(scores)[ranks]
precisions = numpy.array(precisions)[ranks]
ranks = range(n)
targets = []
for index in range(n):
targets.append( {'index':indexes[index],'rank':ranks[index],'score':scores[index],'precision':precisions[index]} )
log_entry = { 'exp_uid':exp_uid,'alg_uid':alg_uid,'timestamp':utils.datetimeNow() }
log_entry.update( {'targets':targets,'num_reported_answers':num_reported_answers} )
ell.log( app_id+':ALG-EVALUATION', log_entry )
response_args_dict = {}
args_out = {'args':response_args_dict,'meta':meta}
predict_json = json.dumps(args_out)
log_entry = { 'exp_uid':exp_uid,'task':'predict','json':predict_json,'timestamp':utils.datetimeNow() }
ell.log( app_id+':APP-RESPONSE', log_entry )
return predict_json,True,''
except Exception, err:
error = traceback.format_exc()
log_entry = { 'exp_uid':exp_uid,'task':'predict','error':str(error),'timestamp':utils.datetimeNow(),'args_json':args_json }
didSucceed,message = ell.log( app_id+':APP-EXCEPTION', log_entry )
return '{}',False,error
def getQuery(self, exp_uid, args_json, db, ell):
"""
A request to ask which two arms to duel next
Expected input (in jsonstructure with string keys):
[optional] (string) participant_uid : unique identifier of session for a participant answering questions (that is, an email address is not good enough as the participant could participate in multiple exp_uids so it would not be unique against all experiments), if key non-existant particpant_uid is assigned as exp_uid.
Expected output (in json structure with string keys):
(list) target_indices : list that stores dictionary of targets with fields:
{
(int) index : the index of the target of relevance
(str) label : in {'left','right'} for display
(int) flag : integer for algorithm's use
}
(str) query_uid : unique identifier of query (used to look up for processAnswer)
Usage:
getQuery_response_json,didSucceed,message = app.getQuery(db_API,exp_uid,getQuery_args_json)
Example input:
getQuery_args_json = {"participant_uid": "0077110d03cf06b8f77d11acc399e8a7"}
Example output:
getQuery_response_json = {"query_uid": "4d02a9924f92138287edd17ca5feb6e1", "target_indices": [{"index": 9, "flag": 0, "label": "left"}, {"index": 8, "flag": 1, "label": "right"}]}
"""
try:
app_id = self.app_id
log_entry = {
'exp_uid': exp_uid,
'task': 'getQuery',
'json': args_json,
'timestamp': utils.datetimeNow()
}
ell.log(app_id + ':APP-CALL', log_entry)
# convert args_json to args_dict
try:
args_dict = json.loads(args_json)
except:
error = "%s.initExp input args_json is in improper format" % self.app_id
return '{}', False, error
# get list of algorithms associated with project
alg_list, didSucceed, message = db.get(app_id + ':experiments',
exp_uid, 'alg_list')
alg_label_to_alg_id = {}
alg_label_to_alg_uid = {}
for algorithm in alg_list:
alg_label_to_alg_id[
algorithm['alg_label']] = algorithm['alg_id']
alg_label_to_alg_uid[
algorithm['alg_label']] = algorithm['alg_uid']
algorithm_management_settings, didSucceed, message = db.get(
app_id + ':experiments', exp_uid,
'algorithm_management_settings')
# ASSIGN ALGORITHM TO PARTICIPANT
if 'participant_uid' in args_dict:
participant_uid = args_dict['participant_uid']
else:
participant_uid = exp_uid
participant_doc_exists, didSucceed, message = db.exists(
app_id + ':participants', participant_uid, 'participant_uid')
first_participant_query = not participant_doc_exists
if first_participant_query:
db.set(app_id + ':participants', participant_uid,
'participant_uid', participant_uid)
db.set(app_id + ':participants', participant_uid, 'exp_uid',
exp_uid)
participant_to_algorithm_management, didSucceed, message = db.get(
app_id + ':experiments', exp_uid,
'participant_to_algorithm_management')
if (participant_uid == exp_uid) or (
participant_to_algorithm_management
== 'one_to_many') or (first_participant_query):
if algorithm_management_settings[
'mode'] == 'fixed_proportions':
proportions_list = algorithm_management_settings['params'][
'proportions']
prop = [
prop_item['proportion']
for prop_item in proportions_list
]
prop_item = numpy.random.choice(alg_list, p=prop)
else:
raise Exception('algorithm_management_mode : ' +
algorithm_management_settings['mode'] +
' not implemented')
alg_id = alg_label_to_alg_id[prop_item['alg_label']]
alg_uid = alg_label_to_alg_uid[prop_item['alg_label']]
alg_label = prop_item['alg_label']
if (first_participant_query) and (
participant_to_algorithm_management == 'one_to_one'):
db.set(app_id + ':participants', participant_uid, 'alg_id',
alg_id)
db.set(app_id + ':participants', participant_uid,
'alg_uid', alg_uid)
elif (participant_to_algorithm_management == 'one_to_one'):
# If here, then alg_uid should already be assigned in participant doc
alg_id, didSucceed, message = db.get(app_id + ':participants',
participant_uid, 'alg_id')
alg_uid, didSucceed, message = db.get(app_id + ':participants',
participant_uid,
'alg_uid')
else:
raise Exception('participant_to_algorithm_management : ' +
participant_to_algorithm_management +
' not implemented')
# get sandboxed database for the specific app_id,alg_id,exp_uid - closing off the rest of the database to the algorithm
rc = ResourceClient(app_id, exp_uid, alg_uid, db)
# get specific algorithm to make calls to
alg = utils.get_app_alg(self.app_id, alg_id)
# call getQuery
index_left, index_right, index_painted, dt = utils.timeit(
alg.getQuery)(resource=rc)
# check for context
context_type, didSucceed, message = db.get(app_id + ':experiments',
exp_uid, 'context_type')
context, didSucceed, message = db.get(app_id + ':experiments',
exp_uid, 'context')
# log
log_entry_durations = {
'exp_uid': exp_uid,
'alg_uid': alg_uid,
'task': 'getQuery',
'duration': dt
}
log_entry_durations.update(rc.getDurations())
meta = {'log_entry_durations': log_entry_durations}
# create JSON query payload
if index_left == index_painted:
targets = [{
'index': index_left,
'label': 'left',
'flag': 1
}, {
'index': index_right,
'label': 'right',
'flag': 0
}]
else:
targets = [{
'index': index_left,
'label': 'left',
'flag': 0
}, {
'index': index_right,
'label': 'right',
'flag': 1
}]
timestamp = str(utils.datetimeNow())
query_uid = utils.getNewUID()
query = {}
query['query_uid'] = query_uid
query['target_indices'] = targets
# save query data to database
query_doc = {}
query_doc.update(query)
query_doc['participant_uid'] = participant_uid
query_doc['alg_uid'] = alg_uid
query_doc['exp_uid'] = exp_uid
query_doc['alg_label'] = alg_label
query_doc['timestamp_query_generated'] = timestamp
db.set_doc(app_id + ':queries', query_uid, query_doc)
# add context after updating query doc to avoid redundant information
query['context_type'] = context_type
query['context'] = context
args_out = {'args': query, 'meta': meta}
response_json = json.dumps(args_out)
log_entry = {
'exp_uid': exp_uid,
'task': 'getQuery',
'json': response_json,
'timestamp': utils.datetimeNow()
}
ell.log(app_id + ':APP-RESPONSE', log_entry)
return response_json, True, ''
except Exception, err:
error = traceback.format_exc()
log_entry = {
'exp_uid': exp_uid,
'task': 'getQuery',
'error': error,
'timestamp': utils.datetimeNow(),
'args_json': args_json
}
ell.log(app_id + ':APP-EXCEPTION', log_entry)
return '{}', False, error
def initExp(self,exp_uid,args_json,db,ell):
"""
initialize the project and necessary experiments
Expected input (in json structure with string keys):
(int) n: number of arms
(int) k: number of objects to display
(float) failure_probability : confidence
[optional] (list of dicts) alg_list : with fields (Defaults given by Info.get_app_default_alg_list)
(string) alg_id : valid alg_id for this app_id
(string) alg_label : unique identifier for algorithm (e.g. may have experiment with repeated alg_id's, but alg_labels must be unqiue, will also be used for plot legends
[optional] (string) test_alg_label : must be one of the alg_label's in alg_list (Default is self)
[optional] (dict) algorithm_management_settings : dictionary with fields (string) 'mode' and (dict) 'params'. mode in {'pure_exploration','explore_exploit','fixed_proportions'}. Default is 'fixed_proportions' and allocates uniform probability to each algorithm. If mode=fixed_proportions then params is a dictionary that contains the field 'proportions' which is a list of dictionaries with fields 'alg_label' and 'proportion' for all algorithms in alg_list. All proportions must be positive and sum to 1 over all algs in alg_list
[optional] (string) participant_to_algorithm_management : in {'one_to_one','one_to_many'}. Default is 'one_to_many'.
[optional] (string) instructions
[optional] (string) debrief
Expected output:
if error:
return (JSON) '{}', (bool) False, (str) error_str
else:
return (JSON) '{}', (bool) True,''
Usage:
initExp_response_json,didSucceed,message = app.initExp(exp_uid,initExp_args_json)
Example input:
initExp_args_json = {"participant_to_algorithm_management": "one_to_many", "alg_list": [{"alg_label": "BR_LilUCB", "alg_id": "BR_LilUCB", "params": {}}], "algorithm_management_settings": {"params": {"proportions": [{"alg_label": "BR_LilUCB", "proportion": 1.0}]}, "mode": "fixed_proportions"}, "failure_probability": 0.01, "n": 10}
Example output:
initExp_response_json = {}
"""
try:
app_id = self.app_id
# remove any reminants of an experiment if it exists
didSucceed,message = db.delete_docs_with_filter('experiments_admin',{'exp_uid':exp_uid})
didSucceed,message = db.delete_docs_with_filter(app_id+':experiments',{'exp_uid':exp_uid})
didSucceed,message = db.delete_docs_with_filter(app_id+':queries',{'exp_uid':exp_uid})
didSucceed,message = db.delete_docs_with_filter(app_id+':participants',{'exp_uid':exp_uid})
didSucceed,message = db.delete_docs_with_filter(app_id+':algorithms',{'exp_uid':exp_uid})
didSucceed,message = ell.delete_logs_with_filter(app_id+':APP-CALL',{'exp_uid':exp_uid})
didSucceed,message = ell.delete_logs_with_filter(app_id+':APP-RESPONSE',{'exp_uid':exp_uid})
didSucceed,message = ell.delete_logs_with_filter(app_id+':APP-EXCEPTION',{'exp_uid':exp_uid})
didSucceed,message = ell.delete_logs_with_filter(app_id+':ALG-DURATION',{'exp_uid':exp_uid})
didSucceed,message = ell.delete_logs_with_filter(app_id+':ALG-EVALUATION',{'exp_uid':exp_uid})
# add indexes (only adds them if they do not already exist)
didSucceed,message = db.ensure_index('experiments_admin',{'exp_uid':1})
didSucceed,message = db.ensure_index(app_id+':experiments',{'exp_uid':1})
didSucceed,message = db.ensure_index(app_id+':queries',{'query_uid':1})
didSucceed,message = db.ensure_index(app_id+':queries',{'exp_uid':1})
didSucceed,message = db.ensure_index(app_id+':queries',{'alg_uid':1})
didSucceed,message = db.ensure_index(app_id+':queries',{'participant_uid':1})
didSucceed,message = db.ensure_index(app_id+':participants',{'participant_uid':1})
didSucceed,message = db.ensure_index(app_id+':participants',{'exp_uid':1})
didSucceed,message = db.ensure_index(app_id+':algorithms',{'alg_uid':1})
didSucceed,message = db.ensure_index(app_id+':algorithms',{'exp_uid':1})
didSucceed,message = ell.ensure_index(app_id+':APP-CALL',{'exp_uid':1})
didSucceed,message = ell.ensure_index(app_id+':APP-CALL',{'timestamp':1})
didSucceed,message = ell.ensure_index(app_id+':APP-CALL',{'exp_uid':1,'timestamp':1})
didSucceed,message = ell.ensure_index(app_id+':APP-CALL',{'exp_uid':1,'task':1})
didSucceed,message = ell.ensure_index(app_id+':APP-RESPONSE',{'exp_uid':1})
didSucceed,message = ell.ensure_index(app_id+':APP-RESPONSE',{'timestamp':1})
didSucceed,message = ell.ensure_index(app_id+':APP-RESPONSE',{'exp_uid':1,'timestamp':1})
didSucceed,message = ell.ensure_index(app_id+':APP-RESPONSE',{'exp_uid':1,'task':1})
didSucceed,message = ell.ensure_index(app_id+':APP-EXCEPTION',{'exp_uid':1})
didSucceed,message = ell.ensure_index(app_id+':APP-EXCEPTION',{'timestamp':1})
didSucceed,message = ell.ensure_index(app_id+':APP-EXCEPTION',{'exp_uid':1,'timestamp':1})
didSucceed,message = ell.ensure_index(app_id+':APP-EXCEPTION',{'exp_uid':1,'task':1})
didSucceed,message = ell.ensure_index(app_id+':ALG-DURATION',{'exp_uid':1})
didSucceed,message = ell.ensure_index(app_id+':ALG-DURATION',{'alg_uid':1})
didSucceed,message = ell.ensure_index(app_id+':ALG-DURATION',{'timestamp':1})
didSucceed,message = ell.ensure_index(app_id+':ALG-DURATION',{'exp_uid':1,'timestamp':1})
didSucceed,message = ell.ensure_index(app_id+':ALG-DURATION',{'alg_uid':1,'task':1})
didSucceed,message = ell.ensure_index(app_id+':ALG-EVALUATION',{'exp_uid':1})
didSucceed,message = ell.ensure_index(app_id+':ALG-EVALUATION',{'alg_uid':1})
didSucceed,message = ell.ensure_index(app_id+':ALG-EVALUATION',{'timestamp':1})
didSucceed,message = ell.ensure_index(app_id+':ALG-EVALUATION',{'exp_uid':1,'timestamp':1})
db.set('experiments_admin',exp_uid,'exp_uid',exp_uid)
db.set('experiments_admin',exp_uid,'app_id',app_id)
db.set('experiments_admin',exp_uid,'start_date',utils.datetime2str(utils.datetimeNow()))
log_entry = { 'exp_uid':exp_uid,'task':'initExp','json':args_json,'timestamp':utils.datetimeNow() }
ell.log( app_id+':APP-CALL', log_entry )
# convert args_json to args_dict
try:
args_dict = json.loads(args_json)
except:
error = "%s.initExp input args_json is in improper format" % self.app_id
return '{}',False,error
# check for the fields that must be contained in args or error occurs
necessary_fields = ['n','k','failure_probability']
for field in necessary_fields:
try:
args_dict[field]
except KeyError:
error = "%s.initExp input arguments missing field: %s" % (self.app_id,str(field))
return '{}',False,error
n = args_dict['n']
k = args_dict['k']
delta = args_dict['failure_probability']
if 'alg_list' in args_dict:
alg_list = args_dict['alg_list']
supportedAlgs = utils.get_app_supported_algs(self.app_id)
for algorithm in alg_list:
if algorithm['alg_id'] not in supportedAlgs:
error = "%s.initExp unsupported algorithm '%s' in alg_list" % (self.app_id,alg_id)
return '{}',False,error
else:
alg_list = utils.get_app_default_alg_list(self.app_id)
if 'instructions' not in args_dict:
instructions = utils.get_app_default_instructions(app_id)
else:
instructions = args_dict['instructions']
if 'debrief' not in args_dict:
debrief = utils.get_app_default_instructions(app_id)
else:
debrief = args_dict['debrief']
if 'num_tries' not in args_dict:
num_tries = utils.get_app_default_num_tries(app_id)
else:
num_tries = args_dict['num_tries']
if 'context_type' not in args_dict:
context_type = 'none'
else:
context_type = args_dict['context_type']
if 'context' not in args_dict:
context = ''
else:
context = args_dict['context']
# ALGORITHM_MANAGEMENT_MODE FORMATTING CHECK
if 'algorithm_management_settings' not in args_dict:
params = {}
params['proportions'] = []
for algorithm in alg_list:
params['proportions'].append( { 'alg_label': algorithm['alg_label'] , 'proportion':1./len(alg_list)} )
algorithm_management_settings = {}
algorithm_management_settings['mode'] = 'fixed_proportions'
algorithm_management_settings['params'] = params
else:
algorithm_management_settings = args_dict['algorithm_management_settings']
try:
mode = algorithm_management_settings['mode']
params = algorithm_management_settings['params']
except:
error = "%s.initExp algorithm_management_settings must be a dictionary with fields 'mode' and 'params'" % (self.app_id)
return '{}',False,error
if mode == 'fixed_proportions':
try:
algorithm_proportions_list = params['proportions']
except:
error = "%s.initExp algorithm_management_settings['params'] must be a dictionary with field 'proportions'" % (self.app_id)
return '{}',False,error
# check if alg_labels are properly labeled
for proportion_item in algorithm_proportions_list:
proportion = proportion_item['proportion']
target_alg_label = proportion_item['alg_label']
target_alg_label_in_alg_list = False
for algorithm in alg_list:
if algorithm['alg_label']==target_alg_label:
target_alg_label_in_alg_list = True
if not target_alg_label_in_alg_list:
error = "%s.initExp algorithm_management_settings['params']['proportions'] must be a list of dictionaries, each dictionary containing the fields 'alg_label' and 'proportion'. The 'alg_label' value must be one of the alg_labels in a provided alg_list and 'proportion' must be nonnegative and sum to 1 : '%s' not in provided alg_list" % (self.app_id,target_alg_label)
return '{}',False,error
elif mode == 'pure_exploration':
error = "%s.initExp Sorry, '%s' is not yet supported." % (self.app_id,mode)
return '{}',False,error
elif mode == 'explore_exploit':
error = "%s.initExp Sorry, '%s' is not yet supported." % (self.app_id,mode)
return '{}',False,error
else:
error = "%s.initExp unsupported algorithm_management_mode: '%s'. Must be in {'pure_exploration','explore_exploit','fixed_proportions'}" % (self.app_id,algorithm_management_mode)
return '{}',False,error
# ALGORITHM_MANAGEMENT_MODE FORMATTING CHECK
if 'participant_to_algorithm_management' not in args_dict:
participant_to_algorithm_management = 'one_to_many'
else:
participant_to_algorithm_management = args_dict['participant_to_algorithm_management']
if participant_to_algorithm_management not in ['one_to_many','one_to_one']:
error = "%s.initExp unsupported participant_to_algorithm_management: '%s'. Must be in {'one_to_many','one_to_one'}" % (self.app_id,participant_to_algorithm_management)
return '{}',False,error
# assign uid to each algorithm and save it
for algorithm in alg_list:
alg_uid = utils.getNewUID()
algorithm['alg_uid'] = alg_uid
db.set(app_id+':algorithms',alg_uid,'alg_uid',alg_uid)
db.set(app_id+':algorithms',alg_uid,'exp_uid',exp_uid)
db.set(app_id+':experiments',exp_uid,'exp_uid',exp_uid)
db.set(app_id+':experiments',exp_uid,'app_id',app_id)
db.set(app_id+':experiments',exp_uid,'n',n)
db.set(app_id+':experiments',exp_uid,'k',k)
db.set(app_id+':experiments',exp_uid,'failure_probability',delta)
db.set(app_id+':experiments',exp_uid,'alg_list',alg_list)
db.set(app_id+':experiments',exp_uid,'algorithm_management_settings',algorithm_management_settings)
db.set(app_id+':experiments',exp_uid,'participant_to_algorithm_management',participant_to_algorithm_management)
db.set(app_id+':experiments',exp_uid,'instructions',instructions)
db.set(app_id+':experiments',exp_uid,'debrief',debrief)
db.set(app_id+':experiments',exp_uid,'context_type',context_type)
db.set(app_id+':experiments',exp_uid,'context',context)
db.set(app_id+':experiments',exp_uid,'num_tries',num_tries)
# now create intitialize each algorithm
for algorithm in alg_list:
alg_id = algorithm['alg_id']
alg_uid = algorithm['alg_uid']
db.set(app_id+':algorithms',alg_uid,'alg_id',alg_id)
db.set(app_id+':algorithms',alg_uid,'alg_uid',alg_uid)
db.set(app_id+':algorithms',alg_uid,'exp_uid',exp_uid)
# get sandboxed database for the specific app_id,alg_id,exp_uid - closing off the rest of the database to the algorithm
rc = ResourceClient(app_id,exp_uid,alg_uid,db)
# get specific algorithm to make calls to
alg = utils.get_app_alg(self.app_id,alg_id)
# call initExp
didSucceed,dt = utils.timeit(alg.initExp)(resource=rc,n=n,k=k,failure_probability=delta)
log_entry = { 'exp_uid':exp_uid,'alg_uid':alg_uid,'task':'initExp','duration':dt,'timestamp':utils.datetimeNow() }
ell.log( app_id+':ALG-DURATION', log_entry )
response_json = '{}'
log_entry = { 'exp_uid':exp_uid,'task':'initExp','json':response_json,'timestamp':utils.datetimeNow() }
ell.log( app_id+':APP-RESPONSE', log_entry )
return response_json,True,''
except Exception, err:
error = traceback.format_exc()
log_entry = { 'exp_uid':exp_uid,'task':'initExp','error':error,'timestamp':utils.datetimeNow(),'args_json':args_json }
ell.log( app_id+':APP-EXCEPTION', log_entry )
return '{}',False,error
def predict(self,exp_uid,args_json,db,ell):
"""
Have the model learned by some particular algorithm predict a variety of stuff
Expected input (in json structure with string keys):
(string) predict_id : identifier for the desired prediction
(list) params : dictionary of stat_id specific fields.
##########
Description: Each algorithm (with an associated alg_label) has a test_alg_label associated with it.
Expected input:
(string) predict_id : 'evaluate_on_test'
(dict) params : dictionary with fields
(string) alg_label : describes target algorithm to test
(string) test_alg_label : describes the algorithm that whos triplets are evaluated on the target algorithm
Expected output (in json structure):
(float) num_reported_answers : number of reported answers after which the calculation was made
(float) error : 0/1 loss on test set
"""
try:
app_id = self.app_id
log_entry = { 'exp_uid':exp_uid,'task':'predict','json':args_json,'timestamp':utils.datetimeNow() }
ell.log( app_id+':APP-CALL', log_entry )
# convert args_json to args_dict
try:
args_dict = json.loads(args_json)
except:
error = "%s.predict input args_json is in improper format" % self.app_id
return '{}',False,error
# check for the fields that must be contained in args or error occurs
necessary_fields = ['predict_id','params']
for field in necessary_fields:
try:
args_dict[field]
except KeyError:
error = "%s.predict input arguments missing field: %s" % (self.app_id,str(field))
return '{}',False,error
predict_id = args_dict['predict_id']
params = args_dict['params']
if predict_id == "evaluate_on_test":
alg_label = params['alg_label']
test_alg_label = params['test_alg_label']
# get list of algorithms associated with project
alg_list,didSucceed,message = db.get(app_id+':experiments',exp_uid,'alg_list')
# get alg_id
for algorithm in alg_list:
if alg_label == algorithm['alg_label']:
alg_id = algorithm['alg_id']
alg_uid = algorithm['alg_uid']
num_reported_answers,didSucceed,message = db.get(app_id+':experiments',exp_uid,'num_reported_answers_for_'+alg_uid)
if type(num_reported_answers)!=int:
num_reported_answers=0
if test_alg_label == algorithm['alg_label']:
test_alg_id = algorithm['alg_id']
test_alg_uid = algorithm['alg_uid']
# get list of triplets from test
queries,didSucceed,message = db.get_docs_with_filter(app_id+':queries',{'alg_uid':test_alg_uid})
test_S = []
for query in queries:
if 'q' in query.keys():
q = query['q']
test_S.append(q)
# get sandboxed database for the specific app_id,alg_id,exp_uid - closing off the rest of the database to the algorithm
rc = ResourceClient(app_id,exp_uid,alg_uid,db)
# get specific algorithm to make calls to
alg = utils.get_app_alg(self.app_id,alg_id)
##### Triplet Predict #####
# call predict
if len(test_S)>0:
labels = []
for idx,q in enumerate(test_S):
labels.append(1)
R = numpy.random.randn()
if R < 0:
test_S[idx] = [ q[1], q[0], q[2] ]
labels[idx] = -1
test_y,dt = utils.timeit(alg.predict)(rc,test_S)
log_entry_durations = { 'exp_uid':exp_uid,'alg_uid':alg_uid,'task':'predict','duration':dt }
log_entry_durations.update( rc.getDurations() )
meta = {'log_entry_durations':log_entry_durations}
# compute error rate: test_y should match labels
number_correct = 0.
for i in range(len(test_S)):
if test_y[i]*labels[i] > 0:
number_correct += 1.0
accuracy = number_correct/len(test_S)
err = 1.0-accuracy
else:
err = 0.5
labels = []
test_y = []
params['num_reported_answers'] = num_reported_answers
params['error'] = err
params['num_test_triplets'] = len(test_S)
params['labels'] = labels
params['test_y'] = test_y
##### Get Embedding #####
Xd,X2 = alg.getStats(rc)
params['Xd'] = Xd
params['X2'] = X2
log_entry = { 'exp_uid':exp_uid,'alg_uid':alg_uid,'timestamp':utils.datetimeNow() }
log_entry.update( params )
ell.log( app_id+':ALG-EVALUATION', log_entry )
data = params
response_args_dict = data
args_out = {'args':response_args_dict,'meta':meta}
predict_json = json.dumps(args_out)
log_entry = { 'exp_uid':exp_uid,'task':'predict','json':predict_json,'timestamp':utils.datetimeNow() }
ell.log( app_id+':APP-RESPONSE', log_entry )
return predict_json,True,''
except Exception, err:
error = traceback.format_exc()
log_entry = { 'exp_uid':exp_uid,'task':'predict','error':str(error),'timestamp':utils.datetimeNow() }
didSucceed,message = ell.log( app_id+':APP-EXCEPTION', log_entry )
return '{}',False,error
where {hostname} and {port} are as they are below
"""
import sys
sys.path.append("/next_backend")
import time
import traceback
import next.utils as utils
import subprocess
import next.constants as constants
import os
while(1):
timestamp = utils.datetimeNow()
print "[ %s ] Calling database daemon..." % str(timestamp)
subprocess.call('python ./next/database/database_backup.py',shell=True)
time.sleep(3600*6) # once every 6 hours
def getQuery(self, exp_uid, args_json):
try:
args_dict = self.helper.convert_json(args_json)
args_dict = verifier.verify(args_dict, self.reference_dict['getQuery']['args'])
experiment_dict = self.butler.experiment.get()
alg_list = experiment_dict['args']['alg_list']
participant_to_algorithm_management = experiment_dict['args']['participant_to_algorithm_management']
algorithm_management_settings = experiment_dict['args']['algorithm_management_settings']
# Create the participant dictionary in participants bucket if needed. Also pull out label and id for this algorithm
participant_uid = args_dict['args'].get('participant_uid', args_dict['exp_uid'])
# Check to see if the first participant has come by and if not, save to db
participant_doc = self.butler.participants.get(uid=participant_uid)
first_participant_query = participant_doc==None
if first_participant_query:
participant_doc = {}
self.butler.participants.set(uid=participant_uid, value={'exp_uid':exp_uid, 'participant_uid':participant_uid})
if (participant_uid == exp_uid) or (participant_to_algorithm_management == 'one_to_many') or (first_participant_query):
if algorithm_management_settings['mode'] == 'fixed_proportions':
labels = [alg['alg_label'] for alg in algorithm_management_settings['params']]
prop = [prop_item['proportion'] for prop_item in algorithm_management_settings['params']]
# reorder prop and alg_list to have same order
new_alg_list = []
broken = False
for label in labels:
broken = False
for alg in alg_list:
if label == alg['alg_label']:
new_alg_list += [alg]
broken = True
break
if not broken:
raise Exception('alg_label not present for both porportions and labels')
chosen_alg = numpy.random.choice(new_alg_list, p=prop)
elif algorithm_management_settings['mode'] == 'custom' :
chosen_alg = self.myApp.chooseAlg(self.butler, alg_list, args_dict['args'])
else:
chosen_alg = numpy.random.choice(alg_list)
alg_id = chosen_alg['alg_id']
alg_label = chosen_alg['alg_label']
if (first_participant_query) and (participant_to_algorithm_management=='one_to_one'):
self.butler.participants.set(uid=participant_uid, key='alg_id',value=alg_id)
self.butler.participants.set(uid=participant_uid, key='alg_label',value=alg_label)
elif (participant_to_algorithm_management=='one_to_one'):
alg_id = participant_doc['alg_id']
alg_label = participant_doc['alg_label']
query_uid = utils.getNewUID()
args_dict['args'].update(query_uid=query_uid)
query_doc = self.call_app_fn(alg_label, alg_id, 'getQuery', args_dict)
query_doc.update({'participant_uid':participant_uid,
'alg_id':alg_id,
'exp_uid':exp_uid,
'alg_label':alg_label,
'timestamp_query_generated':str(utils.datetimeNow()),
'query_uid':query_uid})
self.butler.queries.set(uid=query_uid, value=query_doc)
return json.dumps({'args':query_doc,'meta':{'log_entry_durations':self.log_entry_durations}}), True,''
except Exception, error:
exc_type, exc_value, exc_traceback = sys.exc_info()
full_error = str(traceback.format_exc())+'\n'+str(error)
utils.debug_print("getQuery Exception: " + full_error, color='red')
log_entry = { 'exp_uid':exp_uid,'task':'getQuery','error':full_error,'timestamp':utils.datetimeNow(),'args_json':args_json }
self.butler.ell.log( self.app_id+':APP-EXCEPTION', log_entry )
traceback.print_tb(exc_traceback)
return '{}', False, str(error)
def most_current_embedding(self, app, butler, alg_label):
"""
Description: Returns embedding in the form of a list of dictionaries, which is conveneint for downstream applications
Expected input:
(string) alg_label : must be a valid alg_label contained in alg_list list of dicts
Expected output (in dict):
plot_type : 'scatter2d_noaxis'
(float) x_min : minimum x-value to display in viewing box
(float) x_max : maximum x-value to display in viewing box
(float) y_min : minimum y-value to display in viewing box
(float) y_max : maximum y-value to display in viewing box
(list of dicts with fields) data :
(int) index : index of target
(float) x : x-value of target
(float) y : y-value of target
"""
TargetManager = butler.targets
item = app.getModel(
json.dumps({
'exp_uid': app.exp_uid,
'args': {
'alg_label': alg_label
}
}))
embedding = item['X']
data = []
x_min = numpy.float('inf')
x_max = -numpy.float('inf')
y_min = numpy.float('inf')
y_max = -numpy.float('inf')
for idx, target in enumerate(embedding):
target_dict = {}
target_dict['target'] = TargetManager.get_target_item(
app.exp_uid, idx)
target_dict['x'] = target[
0] # this is what will actually be plotted,
try:
target_dict['y'] = target[
1] # takes first two components, (could be replaced by PCA)
except:
target_dict['y'] = 0.
target_dict['darray'] = target
x_min = min(x_min, target[0])
x_max = max(x_max, target[0])
y_min = min(y_min, target[1])
y_max = max(y_max, target[1])
data.append(target_dict)
return_dict = {
'timestamp': str(utils.datetimeNow()),
'x_min': x_min,
'x_max': x_max,
'y_min': y_min,
'y_max': y_max,
'data': data,
'plot_type': 'scatter2d_noaxis'
}
return return_dict
def getQuery(self,exp_uid,args_json,db,ell):
"""
A request to ask the query: "is {center} more similar to {left} or {right}?"
Expected input (in jsonstructure with string keys):
[optional] (string) participant_uid : unique identifier of session for a participant answering questions (that is, an email address is not good enough as the participant could participate in multiple exp_uids so it would not be unique against all experiments), if key non-existant particpant_uid is assigned as exp_uid.
Expected output (in json structure with string keys):
(list) target_indices : list that stores dictionary of targets with fields:
{
(int) index : the index of the target of relevance
(str) label : in {'left','right','center'}
(int) flag : integer for algorithm's use
}
(str) query_uid : unique identifier of query (used to look up for reportAnswer)
Usage:
getQuery_response_json,didSucceed,message = app.getQuery(db_API,exp_uid,getQuery_args_json)
Example input:
getQuery_args_json = {"participant_uid": "ecaf1d60ab995b3c57afb3a1f3f288f0"}
Example output:
getQuery_response_json = {"query_uid": "a061ce00742603afc540d23e08ab77b3", "target_indices": [{"index": 19, "flag": 0, "label": "center"}, {"index": 8, "flag": 0, "label": "left"}, {"index": 15, "flag": 0, "label": "right"}]}
"""
try:
app_id = self.app_id
log_entry = { 'exp_uid':exp_uid,'task':'getQuery','json':args_json,'timestamp':utils.datetimeNow() }
ell.log( app_id+':APP-CALL', log_entry )
# convert args_json to args_dict
try:
args_dict = json.loads(args_json)
except:
error = "%s.initExp input args_json is in improper format" % self.app_id
return '{}',False,error
# get list of algorithms associated with project
alg_list,didSucceed,message = db.get(app_id+':experiments',exp_uid,'alg_list')
alg_label_to_alg_id = {}
alg_label_to_alg_uid = {}
for algorithm in alg_list:
alg_label_to_alg_id[ algorithm['alg_label'] ] = algorithm['alg_id']
alg_label_to_alg_uid[ algorithm['alg_label'] ] = algorithm['alg_uid']
algorithm_management_settings,didSucceed,message = db.get(app_id+':experiments',exp_uid,'algorithm_management_settings')
# ASSIGN ALGORITHM TO PARTICIPANT
if 'participant_uid' in args_dict:
participant_uid = args_dict['participant_uid']
else:
participant_uid = exp_uid
participant_doc_exists,didSucceed,message = db.exists(app_id+':participants',participant_uid,'participant_uid')
first_participant_query = not participant_doc_exists
if first_participant_query:
db.set(app_id+':participants',participant_uid,'participant_uid',participant_uid)
db.set(app_id+':participants',participant_uid,'exp_uid',exp_uid)
participant_to_algorithm_management,didSucceed,message = db.get(app_id+':experiments',exp_uid,'participant_to_algorithm_management')
if (participant_uid==exp_uid) or (participant_to_algorithm_management=='one_to_many') or (first_participant_query):
if algorithm_management_settings['mode']=='fixed_proportions':
proportions_list = algorithm_management_settings['params']['proportions']
prop = [ prop_item['proportion'] for prop_item in proportions_list ]
prop_item = numpy.random.choice(alg_list,p=prop)
else:
raise Exception('algorithm_management_mode : '+algorithm_management_settings['mode']+' not implemented')
alg_id = alg_label_to_alg_id[ prop_item['alg_label'] ]
alg_uid = alg_label_to_alg_uid[ prop_item['alg_label'] ]
alg_label = prop_item['alg_label']
if (first_participant_query) and (participant_to_algorithm_management=='one_to_one'):
db.set(app_id+':participants',participant_uid,'alg_id',alg_id)
db.set(app_id+':participants',participant_uid,'alg_uid',alg_uid)
elif (participant_to_algorithm_management=='one_to_one'):
# If here, then alg_uid should already be assigned in participant doc
alg_id,didSucceed,message = db.get(app_id+':participants',participant_uid,'alg_id')
alg_uid,didSucceed,message = db.get(app_id+':participants',participant_uid,'alg_uid')
else:
raise Exception('participant_to_algorithm_management : '+participant_to_algorithm_management+' not implemented')
# get sandboxed database for the specific app_id,alg_id,exp_uid - closing off the rest of the database to the algorithm
rc = ResourceClient(app_id,exp_uid,alg_uid,db)
# get specific algorithm to make calls to
alg = utils.get_app_alg(self.app_id,alg_id)
# call getQuery
index_center,index_left,index_right,dt = utils.timeit(alg.getQuery)(resource=rc)
log_entry_durations = { 'exp_uid':exp_uid,'alg_uid':alg_uid,'task':'getQuery','duration':dt }
log_entry_durations.update( rc.getDurations() )
meta = {'log_entry_durations':log_entry_durations}
# create JSON query payload
timestamp = str(utils.datetimeNow())
query_uid = utils.getNewUID()
query = {}
query['query_uid'] = query_uid
query['target_indices'] = [ {'index':index_center,'label':'center','flag':0},{'index':index_left,'label':'left','flag':0},{'index':index_right,'label':'right','flag':0} ]
# save query data to database
query_doc = {}
query_doc.update(query)
query_doc['participant_uid'] = participant_uid
query_doc['alg_uid'] = alg_uid
query_doc['exp_uid'] = exp_uid
query_doc['alg_label'] = alg_label
query_doc['timestamp_query_generated'] = timestamp
for field in query_doc:
db.set(app_id+':queries',query_uid,field,query_doc[field])
args_out = {'args':query,'meta':meta}
response_json = json.dumps(args_out)
log_entry = { 'exp_uid':exp_uid,'task':'getQuery','json':response_json,'timestamp':utils.datetimeNow() }
ell.log( app_id+':APP-RESPONSE', log_entry )
return response_json,True,''
except Exception, err:
error = traceback.format_exc()
log_entry = { 'exp_uid':exp_uid,'task':'getQuery','error':error,'timestamp':utils.datetimeNow() }
ell.log( app_id+':APP-EXCEPTION', log_entry )
return '{}',False,error
