def getModel(self, exp_uid, args_json): try: args_dict = self.helper.convert_json(args_json) args_dict = verifier.verify(args_dict, self.reference_dict['getModel']['args']) alg_label = args_dict['args']['alg_label'] args = self.butler.experiment.get(key='args') for algorithm in args['alg_list']: if alg_label == algorithm['alg_label']: alg_id = algorithm['alg_id'] myapp_response = self.call_app_fn(alg_label, alg_id, 'getModel', args_dict) myapp_response['exp_uid'] = exp_uid myapp_response['alg_label'] = alg_label # Log the response of the getModel in ALG-EVALUATION if args_dict['args']['logging']: alg_log_entry = {'exp_uid': exp_uid, 'alg_label':alg_label, 'task': 'getModel', 'timestamp': str(utils.datetimeNow())} alg_log_entry.update(myapp_response) self.butler.log('ALG-EVALUATION', alg_log_entry) return json.dumps({'args': myapp_response, 'meta': {'log_entry_durations':self.log_entry_durations, 'timestamp': str(utils.datetimeNow())}}), True, '' except Exception, error: exc_type, exc_value, exc_traceback = sys.exc_info() full_error = str(traceback.format_exc())+'\n'+str(error) utils.debug_print("getModel Exception: " + full_error, color='red') log_entry = { 'exp_uid':exp_uid,'task':'getModel','error':full_error,'timestamp':utils.datetimeNow(),'args_json':args_json } self.butler.ell.log( self.app_id+':APP-EXCEPTION', log_entry ) traceback.print_tb(exc_traceback) return Exception(error)
def dashboardAsync(self, app_id, exp_uid, args, ignore_result=False): """ Run a task (task_name) on a set of args with a given app_id, and exp_uid. Waits for computation to finish and returns the answer unless ignore_result=True in which case its a non-blocking call. No guarantees about order of execution. Inputs: ::\n (string) app_id, (string) exp_id, (string) task_name, (json) args Outputs: ::\n task_name(app_id, exp_id, args) """ submit_timestamp = utils.datetimeNow('string') domain = self.__get_domain_for_job(app_id + "_" + exp_uid) if next.constants.CELERY_ON: result = tasks.apply_dashboard.apply_async( args=[app_id, exp_uid, args, submit_timestamp], exchange='dashboard@' + domain, routing_key='dashboard@' + domain) if ignore_result: return True else: return result.get(interval=0.001) else: result = tasks.apply_dashboard(app_id, exp_uid, args, submit_timestamp) if ignore_result: return True else: return result
def refresh_domain_hashes(self): # see what workers are out there worker_pings = None while worker_pings == None: try: # one could also use app.control.inspect().active_queues() worker_pings = app.control.inspect().ping() except: worker_pings = None worker_names = worker_pings.keys() domain_names_with_dups = [item.split('@')[1] for item in worker_names] domain_names = list(set(domain_names_with_dups)) # remove duplicates! timestamp = utils.datetime2str(utils.datetimeNow()) print "[ %s ] domains with active workers = %s" % (timestamp, str(domain_names)) replicated_domains_hash = [] for domain in domain_names: for i in range(self.num_replicas): replicated_domains_hash.append( (domain, self.hash(domain + '_replica_' + str(i)), i)) replicated_domains_hash = sorted(replicated_domains_hash, key=lambda x: x[1]) self.r.set('replicated_domains_hash', json.dumps(replicated_domains_hash))
def init_alg(self, exp_uid, algorithm, alg_args): butler = Butler(self.app_id, exp_uid, self.myApp.TargetManager, self.butler.db, self.butler.ell, algorithm['alg_label'], algorithm['alg_id']) alg = utils.get_app_alg(self.app_id, algorithm['alg_id']) if 'args' in self.algs_reference_dict['initExp']: alg_args = verifier.verify( alg_args, self.algs_reference_dict['initExp']['args']) # I got rid of a timeit function here; it wasn't handling the # argument unpacking correctly? --Scott, 2016-3-7 # TODO: put dt back in and change log_entry to relfect that alg_response = alg.initExp(butler, **alg_args) alg_response = verifier.verify( {'rets': alg_response}, {'rets': self.algs_reference_dict['initExp']['rets']}) log_entry = { 'exp_uid': exp_uid, 'alg_label': algorithm['alg_label'], 'task': 'initExp', 'duration': -1, 'timestamp': utils.datetimeNow() } self.butler.log('ALG-DURATION', log_entry)
def applySyncByNamespace(self, app_id, exp_uid, task_name, args, namespace=None, ignore_result=False,time_limit=0): """ Run a task (task_name) on a set of args with a given app_id, and exp_uid asynchronously. Waits for computation to finish and returns the answer unless ignore_result=True in which case its a non-blocking call. If this method is called a sequence of times with the same namespace (defaults to exp_uid if not set) it is guaranteed that they will execute in order, each job finishing before the next begins Inputs: ::\n (string) app_id, (string) exp_id, (string) task_name, (json) args """ if namespace==None: namespace=exp_uid domain = self.__get_domain_for_job(app_id+"_"+exp_uid) job_uid = utils.getNewUID() submit_timestamp = utils.datetimeNow('string') if time_limit == 0: soft_time_limit = None hard_time_limit = None else: soft_time_limit = time_limit hard_time_limit = time_limit + .01 result = tasks.apply_sync_by_namespace.apply_async(args=[app_id,exp_uid,task_name, args, namespace, job_uid, submit_timestamp, time_limit], exchange='sync@'+domain, routing_key='sync@'+domain,soft_time_limit=soft_time_limit,time_limit=hard_time_limit) if ignore_result: return True else: return result.get(interval=.001)
def dashboardAsync(self, app_id, exp_uid, args, ignore_result=False): """ Run a task (task_name) on a set of args with a given app_id, and exp_uid. Waits for computation to finish and returns the answer unless ignore_result=True in which case its a non-blocking call. No guarantees about order of execution. Inputs: ::\n (string) app_id, (string) exp_id, (string) task_name, (json) args Outputs: ::\n task_name(app_id, exp_id, args) """ submit_timestamp = utils.datetimeNow('string') domain = self.__get_domain_for_job(app_id+"_"+exp_uid) if next.constants.CELERY_ON: result = tasks.apply_dashboard.apply_async(args=[app_id, exp_uid, args, submit_timestamp], exchange='dashboard@'+domain, routing_key='dashboard@'+domain) if ignore_result: return True else: return result.get(interval=0.001) else: result = tasks.apply_dashboard(app_id,exp_uid, args, submit_timestamp) if ignore_result: return True else: return result
def test_log(lapi): B = 'test_log' now = utils.datetimeNow() log_entry = {'a': 2, 'timestamp': now} lapi.log(B, log_entry) assert lapi._bucket(B).find_one({'timestamp': now}).get('a') == 2
def applySyncByNamespace(self, app_id, exp_uid, task_name, args, namespace=None, ignore_result=False,time_limit=0): """ Run a task (task_name) on a set of args with a given app_id, and exp_uid asynchronously. Waits for computation to finish and returns the answer unless ignore_result=True in which case its a non-blocking call. If this method is called a sequence of times with the same namespace (defaults to exp_uid if not set) it is guaranteed that they will execute in order, each job finishing before the next begins Inputs: ::\n (string) app_id, (string) exp_id, (string) task_name, (json) args """ submit_timestamp = utils.datetimeNow('string') if namespace==None: namespace=exp_uid domain = self.__get_domain_for_job(app_id+"_"+exp_uid) num_queues = next.constants.CELERY_SYNC_WORKER_COUNT # assign namespaces to queues (with worker of concurrency 1) in round-robbin try: namespace_cnt = int(self.r.get(namespace+"_cnt")) except: pipe = self.r.pipeline(True) while 1: try: pipe.watch(namespace+"_cnt","namespace_counter") if not pipe.exists(namespace+"_cnt"): if not pipe.exists('namespace_counter'): namespace_counter = 0 else: namespace_counter = pipe.get('namespace_counter') pipe.multi() pipe.set(namespace+"_cnt",int(namespace_counter)+1) pipe.set('namespace_counter',int(namespace_counter)+1) pipe.execute() else: pipe.unwatch() break except redis.exceptions.WatchError: continue finally: pipe.reset() namespace_cnt = int(self.r.get(namespace+"_cnt")) queue_number = (namespace_cnt % num_queues) + 1 queue_name = 'sync_queue_'+str(queue_number)+'@'+domain job_uid = utils.getNewUID() if time_limit == 0: soft_time_limit = None hard_time_limit = None else: soft_time_limit = time_limit hard_time_limit = time_limit + .01 result = tasks.apply_sync_by_namespace.apply_async(args=[app_id,exp_uid,task_name, args, namespace, job_uid, submit_timestamp, time_limit], queue=queue_name,soft_time_limit=soft_time_limit,time_limit=hard_time_limit) if ignore_result: return True else: return result.get(interval=.001)
def test_get_logs_with_filter(lapi): B = 'test_get_logs_with_filter' now = utils.datetimeNow() log_entry = {'a': 2, 'timestamp': now} lapi.log(B, log_entry) retrieved_entry = lapi.get_logs_with_filter(B, {'timestamp': now}) assert len(retrieved_entry) == 1 assert retrieved_entry[0].get('a') == 2
def getQuery(self, exp_uid, args_json): try: args_dict = self.helper.convert_json(args_json) args_dict = verifier.verify(args_dict, self.reference_dict['getQuery']['args']) experiment_dict = self.butler.experiment.get() alg_list = experiment_dict['args']['alg_list'] participant_to_algorithm_management = experiment_dict['args']['participant_to_algorithm_management'] algorithm_management_settings = experiment_dict['args']['algorithm_management_settings'] # Create the participant dictionary in participants bucket if needed. Also pull out label and id for this algorithm participant_uid = args_dict['args'].get('participant_uid', args_dict['exp_uid']) # Check to see if the first participant has come by and if not, save to db participant_doc = self.butler.participants.get(uid=participant_uid) first_participant_query = participant_doc==None if first_participant_query: participant_doc = {} self.butler.participants.set(uid=participant_uid, value={'exp_uid':exp_uid, 'participant_uid':participant_uid}) if (participant_uid == exp_uid) or (participant_to_algorithm_management == 'one_to_many') or (first_participant_query): if algorithm_management_settings['mode'] == 'fixed_proportions': prop = [prop_item['proportion'] for prop_item in algorithm_management_settings['params']] chosen_alg = numpy.random.choice(alg_list, p=prop) elif algorithm_management_settings['mode'] == 'custom' : chosen_alg = self.myApp.chooseAlg(self.butler, alg_list, args_dict['args']) else: chosen_alg = numpy.random.choice(alg_list) alg_id = chosen_alg['alg_id'] alg_label = chosen_alg['alg_label'] if (first_participant_query) and (participant_to_algorithm_management=='one_to_one'): self.butler.participants.set(uid=participant_uid, key='alg_id',value=alg_id) self.butler.participants.set(uid=participant_uid, key='alg_label',value=alg_label) elif (participant_to_algorithm_management=='one_to_one'): alg_id = participant_doc['alg_id'] alg_label = participant_doc['alg_label'] query_uid = utils.getNewUID() args_dict['args'].update(query_uid=query_uid) query_doc = self.call_app_fn(alg_label, alg_id, 'getQuery', args_dict) query_doc.update({'participant_uid':participant_uid, 'alg_id':alg_id, 'exp_uid':exp_uid, 'alg_label':alg_label, 'timestamp_query_generated':str(utils.datetimeNow()), 'query_uid':query_uid}) self.butler.queries.set(uid=query_uid, value=query_doc) return json.dumps({'args':query_doc,'meta':{'log_entry_durations':self.log_entry_durations}}), True,'' except Exception, error: exc_type, exc_value, exc_traceback = sys.exc_info() full_error = str(traceback.format_exc())+'\n'+str(error) utils.debug_print("getQuery Exception: " + full_error, color='red') log_entry = { 'exp_uid':exp_uid,'task':'getQuery','error':full_error,'timestamp':utils.datetimeNow(),'args_json':args_json } self.butler.ell.log( self.app_id+':APP-EXCEPTION', log_entry ) traceback.print_tb(exc_traceback) return '{}', False, str(error)
def initExp(self, exp_uid, args_json): try: self.helper.ensure_indices(self.app_id, self.butler.db, self.butler.ell) args_dict = self.helper.convert_json(args_json) args_dict = verifier.verify(args_dict, self.reference_dict['initExp']['args']) args_dict['exp_uid'] = exp_uid # to get doc from db args_dict['start_date'] = utils.datetime2str(utils.datetimeNow()) self.butler.admin.set(uid=exp_uid, value={ 'exp_uid': exp_uid, 'app_id': self.app_id, 'start_date': str(utils.datetimeNow()) }) utils.debug_print( "Launching app with arguments: {}".format(args_dict)) self.butler.experiment.set(value={'exp_uid': exp_uid}) args_dict['args'] = self.init_app(exp_uid, args_dict['args']['alg_list'], args_dict['args']) args_dict['git_hash'] = git_hash self.butler.experiment.set_many(key_value_dict=args_dict) return '{}', True, '' except Exception, error: exc_type, exc_value, exc_traceback = sys.exc_info() full_error = str(traceback.format_exc()) + '\n' + str(error) utils.debug_print("initExp Exception: " + full_error, color='red') log_entry = { 'exp_uid': exp_uid, 'task': 'initExp', 'error': full_error, 'timestamp': utils.datetimeNow(), 'args_json': args_json } self.butler.ell.log(self.app_id + ':APP-EXCEPTION', log_entry) traceback.print_tb(exc_traceback) return '{}', False, str(error)
def processAnswer(self, exp_uid, args_json): try: args_dict = self.helper.convert_json(args_json) args_dict = verifier.verify( args_dict, self.reference_dict['processAnswer']['args']) # Update timing info in query query = self.butler.queries.get(uid=args_dict['args']['query_uid']) timestamp_answer_received = args_dict['args'].get( 'timestamp_answer_received', None) delta_datetime = utils.str2datetime(timestamp_answer_received) - \ utils.str2datetime(query['timestamp_query_generated']) round_trip_time = delta_datetime.total_seconds() response_time = float(args_dict['args'].get('response_time', 0.)) query_update = self.call_app_fn(query['alg_label'], query['alg_id'], 'processAnswer', args_dict) query_update.update({ 'response_time': response_time, 'network_delay': round_trip_time - response_time, 'timestamp_answer_received': timestamp_answer_received }) self.butler.queries.set_many(uid=args_dict['args']['query_uid'], key_value_dict=query_update) return json.dumps({ 'args': {}, 'meta': { 'log_entry_durations': self.log_entry_durations } }), True, '' except Exception, error: exc_type, exc_value, exc_traceback = sys.exc_info() full_error = str(traceback.format_exc()) + '\n' + str(error) utils.debug_print("processAnswer Exception: " + full_error, color='red') log_entry = { 'exp_uid': exp_uid, 'task': 'processAnswer', 'error': full_error, 'timestamp': utils.datetimeNow(), 'args_json': args_json } self.butler.ell.log(self.app_id + ':APP-EXCEPTION', log_entry) traceback.print_tb(exc_traceback) raise Exception(error)
def most_current_embedding(self,app,butler,alg_label): """ Description: Returns embedding in the form of a list of dictionaries, which is conveneint for downstream applications Expected input: (string) alg_label : must be a valid alg_label contained in alg_list list of dicts Expected output (in dict): plot_type : 'scatter2d_noaxis' (float) x_min : minimum x-value to display in viewing box (float) x_max : maximum x-value to display in viewing box (float) y_min : minimum y-value to display in viewing box (float) y_max : maximum y-value to display in viewing box (list of dicts with fields) data : (int) index : index of target (float) x : x-value of target (float) y : y-value of target """ TargetManager = butler.targets item = app.getModel(json.dumps({'exp_uid':app.exp_uid, 'args':{'alg_label':alg_label}})) embedding = item['X'] data = [] x_min = numpy.float('inf') x_max = -numpy.float('inf') y_min = numpy.float('inf') y_max = -numpy.float('inf') for idx,target in enumerate(embedding): target_dict = {} target_dict['target'] = TargetManager.get_target_item(app.exp_uid, idx) target_dict['x'] = target[0] # this is what will actually be plotted, try: target_dict['y'] = target[1] # takes first two components, (could be replaced by PCA) except: target_dict['y'] = 0. target_dict['darray'] = target x_min = min(x_min,target[0]) x_max = max(x_max,target[0]) y_min = min(y_min,target[1]) y_max = max(y_max,target[1]) data.append(target_dict) return_dict = {'timestamp':str(utils.datetimeNow()), 'x_min':x_min, 'x_max':x_max, 'y_min':y_min, 'y_max':y_max, 'data':data, 'plot_type':'scatter2d_noaxis'} return return_dict
def log(self,bucket_id,log_dict): """ Saves log_dict to PermStore as an individual document for later recall. Inputs: (string) bucket_id, (dict with string values) log_dict Outputs: (bool) didSucceed, (string) message Usage: ::\n didSucceed,message = db.log(bucket_id,doc_uid) """ timestamp = utils.datetimeNow() log_dict.update({ 'timestamp':timestamp }) return self.permStore.setDoc(constants.logs_database_id,bucket_id,None,log_dict)
def applySyncByNamespace(self, app_id, exp_uid, task_name, args, namespace=None, ignore_result=False, time_limit=0): """ Run a task (task_name) on a set of args with a given app_id, and exp_uid asynchronously. Waits for computation to finish and returns the answer unless ignore_result=True in which case its a non-blocking call. If this method is called a sequence of times with the same namespace (defaults to exp_uid if not set) it is guaranteed that they will execute in order, each job finishing before the next begins Inputs: ::\n (string) app_id, (string) exp_id, (string) task_name, (json) args """ if namespace == None: namespace = exp_uid domain = self.__get_domain_for_job(app_id + "_" + exp_uid) job_uid = utils.getNewUID() submit_timestamp = utils.datetimeNow('string') if time_limit == 0: soft_time_limit = None hard_time_limit = None else: soft_time_limit = time_limit hard_time_limit = time_limit + .01 result = tasks.apply_sync_by_namespace.apply_async( args=[ app_id, exp_uid, task_name, args, namespace, job_uid, submit_timestamp, time_limit ], exchange='sync@' + domain, routing_key='sync@' + domain, soft_time_limit=soft_time_limit, time_limit=hard_time_limit) if ignore_result: return True else: return result.get(interval=.001)
def initExp(self, exp_uid, args_json): try: self.helper.ensure_indices(self.app_id,self.butler.db, self.butler.ell) args_dict = self.helper.convert_json(args_json) args_dict = verifier.verify(args_dict, self.reference_dict['initExp']['args']) args_dict['exp_uid'] = exp_uid # to get doc from db args_dict['start_date'] = utils.datetime2str(utils.datetimeNow()) self.butler.admin.set(uid=exp_uid,value={'exp_uid': exp_uid, 'app_id':self.app_id, 'start_date':str(utils.datetimeNow())}) self.butler.experiment.set(value={'exp_uid': exp_uid}) args_dict['args'] = self.init_app(exp_uid, args_dict['args']['alg_list'], args_dict['args']) args_dict['git_hash'] = git_hash self.butler.experiment.set_many(key_value_dict=args_dict) return '{}', True, '' except Exception, error: exc_type, exc_value, exc_traceback = sys.exc_info() full_error = str(traceback.format_exc())+'\n'+str(error) utils.debug_print("initExp Exception: " + full_error, color='red') log_entry = { 'exp_uid':exp_uid,'task':'initExp','error':full_error,'timestamp':utils.datetimeNow(),'args_json':args_json } self.butler.ell.log( self.app_id+':APP-EXCEPTION', log_entry ) traceback.print_tb(exc_traceback) return '{}', False, str(error)
def refresh_domain_hashes(self): # see what workers are out there worker_pings = None while worker_pings==None: try: # one could also use app.control.inspect().active_queues() worker_pings = app.control.inspect().ping() except: worker_pings = None worker_names = worker_pings.keys() domain_names_with_dups = [ item.split('@')[1] for item in worker_names] domain_names = list(set(domain_names_with_dups)) # remove duplicates! timestamp = utils.datetime2str(utils.datetimeNow()) print "[ %s ] domains with active workers = %s" % (timestamp,str(domain_names)) replicated_domains_hash = [] for domain in domain_names: for i in range(self.num_replicas): replicated_domains_hash.append((domain, self.hash(domain+'_replica_'+str(i)), i)) replicated_domains_hash = sorted( replicated_domains_hash, key = lambda x: x[1]) self.r.set('replicated_domains_hash',json.dumps(replicated_domains_hash))
def predict(self, exp_uid, args_json, db, ell): """ Description: uses current model empirical estimates to forecast the ranking of the arms in order of likelhood of being the best from most to least likely Expected input: (string) predict_id : 'predict_labels' (dict) params : dictionary with fields (string) alg_label : describes target algorithm to use Expected output (in json structure): (list of dicts with fields): (int) index : index of target (int) rank : rank that the algorithm assigns to index (rank 0 is most likely the best arm) """ try: app_id = self.app_id log_entry = { 'exp_uid': exp_uid, 'task': 'predict', 'json': args_json, 'timestamp': utils.datetimeNow() } ell.log(app_id + ':APP-CALL', log_entry) # convert args_json to args_dict try: args_dict = json.loads(args_json) except: error = "%s.predict failed to convert input args_json due to improper format" % ( self.app_id) return '{}', False, error # check for the fields that must be contained in args or error occurs necessary_fields = ['predict_id', 'params'] for field in necessary_fields: try: args_dict[field] except KeyError: error = "%s.predict input arguments missing field: %s" % ( self.app_id, str(field)) return '{}', False, error predict_id = args_dict['predict_id'] params = args_dict['params'] if predict_id == "predict_labels": alg_label = params['alg_label'] # get list of algorithms associated with project alg_list, didSucceed, message = db.get(app_id + ':experiments', exp_uid, 'alg_list') # get alg_id for algorithm in alg_list: if alg_label == algorithm['alg_label']: alg_id = algorithm['alg_id'] alg_uid = algorithm['alg_uid'] num_reported_answers, didSucceed, message = db.get( app_id + ':experiments', exp_uid, 'num_reported_answers_for_' + alg_uid) if type(num_reported_answers) != int: num_reported_answers = 0 # get sandboxed database for the specific app_id,alg_id,exp_uid - closing off the rest of the database to the algorithm rc = ResourceClient(app_id, exp_uid, alg_uid, db) # get specific algorithm to make calls to alg = utils.get_app_alg(self.app_id, alg_id) # call getQuery predicted_labels, dt = utils.timeit(alg.predict)(resource=rc) log_entry_durations = { 'exp_uid': exp_uid, 'alg_uid': alg_uid, 'task': 'predict', 'duration': dt } log_entry_durations.update(rc.getDurations()) meta = {'log_entry_durations': log_entry_durations} test_labels, didSucceed, message = db.get( app_id + ':experiments', exp_uid, 'test_labels') num_errors = 0 for answer in test_labels: target_index, target_label = answer if target_label * predicted_labels[target_index] < 0: num_errors += 1. error = num_errors / float(max(1., len(test_labels))) log_entry = { 'exp_uid': exp_uid, 'alg_uid': alg_uid, 'timestamp': utils.datetimeNow() } log_entry.update({ 'error': error, 'num_reported_answers': num_reported_answers }) ell.log(app_id + ':ALG-EVALUATION', log_entry) response_args_dict = { 'exp_uid': exp_uid, 'alg_uid': alg_uid, 'error': error, 'num_reported_answers': num_reported_answers } args_out = {'args': response_args_dict, 'meta': meta} predict_json = json.dumps(args_out) log_entry = { 'exp_uid': exp_uid, 'task': 'predict', 'json': predict_json, 'timestamp': utils.datetimeNow() } ell.log(app_id + ':APP-RESPONSE', log_entry) return predict_json, True, '' except Exception, err: error = traceback.format_exc() log_entry = { 'exp_uid': exp_uid, 'task': 'predict', 'error': str(error), 'timestamp': utils.datetimeNow(), 'args_json': args_json } didSucceed, message = ell.log(app_id + ':APP-EXCEPTION', log_entry) return '{}', False, error
def applySyncByNamespace(self, app_id, exp_uid, alg_id, alg_label, task_name, args, namespace=None, ignore_result=False, time_limit=0): """ Run a task (task_name) on a set of args with a given app_id, and exp_uid asynchronously. Waits for computation to finish and returns the answer unless ignore_result=True in which case its a non-blocking call. If this method is called a sequence of times with the same namespace (defaults to exp_uid if not set) it is guaranteed that they will execute in order, each job finishing before the next begins Inputs: ::\n (string) app_id, (string) exp_id, (string) task_name, (json) args """ submit_timestamp = utils.datetimeNow('string') if namespace == None: namespace = exp_uid domain = self.__get_domain_for_job(app_id + "_" + exp_uid) num_queues = next.constants.CELERY_SYNC_WORKER_COUNT # assign namespaces to queues (with worker of concurrency 1) in round-robbin try: namespace_cnt = int(self.r.get(namespace + "_cnt")) except: pipe = self.r.pipeline(True) while 1: try: pipe.watch(namespace + "_cnt", "namespace_counter") if not pipe.exists(namespace + "_cnt"): if not pipe.exists('namespace_counter'): namespace_counter = 0 else: namespace_counter = pipe.get('namespace_counter') pipe.multi() pipe.set(namespace + "_cnt", int(namespace_counter) + 1) pipe.set('namespace_counter', int(namespace_counter) + 1) pipe.execute() else: pipe.unwatch() break except redis.exceptions.WatchError: continue finally: pipe.reset() namespace_cnt = int(self.r.get(namespace + "_cnt")) queue_number = (namespace_cnt % num_queues) + 1 queue_name = 'sync_queue_' + str(queue_number) + '@' + domain job_uid = utils.getNewUID() if time_limit == 0: soft_time_limit = None hard_time_limit = None else: soft_time_limit = time_limit hard_time_limit = time_limit + .01 if next.constants.CELERY_ON: result = tasks.apply_sync_by_namespace.apply_async( args=[ app_id, exp_uid, alg_id, alg_label, task_name, args, namespace, job_uid, submit_timestamp, time_limit ], queue=queue_name, soft_time_limit=soft_time_limit, time_limit=hard_time_limit) if ignore_result: return True else: return result.get(interval=.001) else: result = tasks.apply_sync_by_namespace(app_id, exp_uid, alg_id, alg_label, task_name, args, namespace, job_uid, submit_timestamp, time_limit) if ignore_result: return True else: return result
def getQuery(self, exp_uid, args_json, db, ell): """ A request to ask which k arms to duel next Expected input (in jsonstructure with string keys): (int) k : number of objects to display [optional] (string) participant_uid : unique identifier of session for a participant answering questions (that is, an email address is not good enough as the participant could participate in multiple exp_uids so it would not be unique against all experiments), if key non-existant particpant_uid is assigned as exp_uid. Expected output (in json structure with string keys): (list) target_indices : list of k target indexes e.g. [ (int) target_index_1, ... , (int) target_index_k ] (str) query_uid : unique identifier of query (used to look up for processAnswer) """ try: app_id = self.app_id log_entry = { 'exp_uid': exp_uid, 'task': 'getQuery', 'json': args_json, 'timestamp': utils.datetimeNow() } ell.log(app_id + ':APP-CALL', log_entry) # convert args_json to args_dict try: args_dict = json.loads(args_json) except: error = "%s.initExp input args_json is in improper format" % self.app_id return '{}', False, error # get list of algorithms associated with project alg_list, didSucceed, message = db.get(app_id + ':experiments', exp_uid, 'alg_list') alg_label_to_alg_id = {} alg_label_to_alg_uid = {} for algorithm in alg_list: alg_label_to_alg_id[ algorithm['alg_label']] = algorithm['alg_id'] alg_label_to_alg_uid[ algorithm['alg_label']] = algorithm['alg_uid'] algorithm_management_settings, didSucceed, message = db.get( app_id + ':experiments', exp_uid, 'algorithm_management_settings') # ASSIGN ALGORITHM TO PARTICIPANT if 'participant_uid' in args_dict: participant_uid = args_dict['participant_uid'] else: participant_uid = exp_uid participant_doc_exists, didSucceed, message = db.exists( app_id + ':participants', participant_uid, 'participant_uid') first_participant_query = not participant_doc_exists if first_participant_query: db.set(app_id + ':participants', participant_uid, 'participant_uid', participant_uid) db.set(app_id + ':participants', participant_uid, 'exp_uid', exp_uid) participant_to_algorithm_management, didSucceed, message = db.get( app_id + ':experiments', exp_uid, 'participant_to_algorithm_management') if (participant_uid == exp_uid) or ( participant_to_algorithm_management == 'one_to_many') or (first_participant_query): if algorithm_management_settings[ 'mode'] == 'fixed_proportions': proportions_list = algorithm_management_settings['params'][ 'proportions'] prop = [ prop_item['proportion'] for prop_item in proportions_list ] prop_item = numpy.random.choice(alg_list, p=prop) else: raise Exception('algorithm_management_mode : ' + algorithm_management_settings['mode'] + ' not implemented') alg_id = alg_label_to_alg_id[prop_item['alg_label']] alg_uid = alg_label_to_alg_uid[prop_item['alg_label']] alg_label = prop_item['alg_label'] if (first_participant_query) and ( participant_to_algorithm_management == 'one_to_one'): db.set(app_id + ':participants', participant_uid, 'alg_id', alg_id) db.set(app_id + ':participants', participant_uid, 'alg_uid', alg_uid) elif (participant_to_algorithm_management == 'one_to_one'): # If here, then alg_uid should already be assigned in participant doc alg_id, didSucceed, message = db.get(app_id + ':participants', participant_uid, 'alg_id') alg_uid, didSucceed, message = db.get(app_id + ':participants', participant_uid, 'alg_uid') else: raise Exception('participant_to_algorithm_management : ' + participant_to_algorithm_management + ' not implemented') # get sandboxed database for the specific app_id,alg_id,exp_uid - closing off the rest of the database to the algorithm rc = ResourceClient(app_id, exp_uid, alg_uid, db) # get specific algorithm to make calls to alg = utils.get_app_alg(self.app_id, alg_id) # call getQuery targets, dt = utils.timeit(alg.getQuery)(resource=rc) # update targets target_list = [] for target in targets: target_list.append({'index': target}) # check for context context_type, didSucceed, message = db.get(app_id + ':experiments', exp_uid, 'context_type') context, didSucceed, message = db.get(app_id + ':experiments', exp_uid, 'context') # log log_entry_durations = { 'exp_uid': exp_uid, 'alg_uid': alg_uid, 'task': 'getQuery', 'duration': dt } log_entry_durations.update(rc.getDurations()) meta = {'log_entry_durations': log_entry_durations} # create JSON query payload timestamp = str(utils.datetimeNow()) query_uid = utils.getNewUID() query = {} query['query_uid'] = query_uid query['target_indices'] = target_list # save query data to database query_doc = {} query_doc.update(query) query_doc['participant_uid'] = participant_uid query_doc['alg_uid'] = alg_uid query_doc['exp_uid'] = exp_uid query_doc['alg_label'] = alg_label query_doc['timestamp_query_generated'] = timestamp for field in query_doc: db.set(app_id + ':queries', query_uid, field, query_doc[field]) # add context after updating query doc to avoid redundant information query['context_type'] = context_type query['context'] = context args_out = {'args': query, 'meta': meta} response_json = json.dumps(args_out) log_entry = { 'exp_uid': exp_uid, 'task': 'getQuery', 'json': response_json, 'timestamp': utils.datetimeNow() } ell.log(app_id + ':APP-RESPONSE', log_entry) return response_json, True, '' except Exception, err: error = traceback.format_exc() log_entry = { 'exp_uid': exp_uid, 'task': 'getQuery', 'error': error, 'timestamp': utils.datetimeNow(), 'args_json': args_json } ell.log(app_id + ':APP-EXCEPTION', log_entry) return '{}', False, error
def getQuery(self,exp_uid,args_json,db,ell): """ A request to ask which two arms to duel next Expected input (in jsonstructure with string keys): [optional] (string) participant_uid : unique identifier of session for a participant answering questions (that is, an email address is not good enough as the participant could participate in multiple exp_uids so it would not be unique against all experiments), if key non-existant particpant_uid is assigned as exp_uid. Expected output (in json structure with string keys): (list) target_indices : list that stores dictionary of targets with fields: { (int) index : the index of the target of relevance (str) label : in {'left','right'} for display (int) flag : integer for algorithm's use } (str) query_uid : unique identifier of query (used to look up for processAnswer) """ try: app_id = self.app_id log_entry = { 'exp_uid':exp_uid,'task':'getQuery','json':args_json,'timestamp':utils.datetimeNow() } ell.log( app_id+':APP-CALL', log_entry ) # convert args_json to args_dict try: args_dict = json.loads(args_json) except: error = "%s.initExp input args_json is in improper format" % self.app_id return '{}',False,error # get list of algorithms associated with project alg_list,didSucceed,message = db.get(app_id+':experiments',exp_uid,'alg_list') alg_label_to_alg_id = {} alg_label_to_alg_uid = {} for algorithm in alg_list: alg_label_to_alg_id[ algorithm['alg_label'] ] = algorithm['alg_id'] alg_label_to_alg_uid[ algorithm['alg_label'] ] = algorithm['alg_uid'] algorithm_management_settings,didSucceed,message = db.get(app_id+':experiments',exp_uid,'algorithm_management_settings') # ASSIGN ALGORITHM TO PARTICIPANT if 'participant_uid' in args_dict: participant_uid = args_dict['participant_uid'] else: participant_uid = exp_uid participant_doc_exists,didSucceed,message = db.exists(app_id+':participants',participant_uid,'participant_uid') first_participant_query = not participant_doc_exists if first_participant_query: db.set(app_id+':participants',participant_uid,'participant_uid',participant_uid) db.set(app_id+':participants',participant_uid,'exp_uid',exp_uid) participant_to_algorithm_management,didSucceed,message = db.get(app_id+':experiments',exp_uid,'participant_to_algorithm_management') if (participant_uid==exp_uid) or (participant_to_algorithm_management=='one_to_many') or (first_participant_query): if algorithm_management_settings['mode']=='fixed_proportions': proportions_list = algorithm_management_settings['params']['proportions'] prop = [ prop_item['proportion'] for prop_item in proportions_list ] prop_item = numpy.random.choice(alg_list,p=prop) else: raise Exception('algorithm_management_mode : '+algorithm_management_settings['mode']+' not implemented') alg_id = alg_label_to_alg_id[ prop_item['alg_label'] ] alg_uid = alg_label_to_alg_uid[ prop_item['alg_label'] ] alg_label = prop_item['alg_label'] if (first_participant_query) and (participant_to_algorithm_management=='one_to_one'): db.set(app_id+':participants',participant_uid,'alg_id',alg_id) db.set(app_id+':participants',participant_uid,'alg_uid',alg_uid) elif (participant_to_algorithm_management=='one_to_one'): # If here, then alg_uid should already be assigned in participant doc alg_id,didSucceed,message = db.get(app_id+':participants',participant_uid,'alg_id') alg_uid,didSucceed,message = db.get(app_id+':participants',participant_uid,'alg_uid') else: raise Exception('participant_to_algorithm_management : '+participant_to_algorithm_management+' not implemented') # get sandboxed database for the specific app_id,alg_id,exp_uid - closing off the rest of the database to the algorithm rc = ResourceClient(app_id,exp_uid,alg_uid,db) # get specific algorithm to make calls to alg = utils.get_app_alg(self.app_id,alg_id) # call getQuery index_left,index_right,index_painted,dt = utils.timeit(alg.getQuery)(resource=rc) # check for context context_type,didSucceed,message = db.get(app_id+':experiments',exp_uid,'context_type') context,didSucceed,message = db.get(app_id+':experiments',exp_uid,'context') # log log_entry_durations = { 'exp_uid':exp_uid,'alg_uid':alg_uid,'task':'getQuery','duration':dt } log_entry_durations.update( rc.getDurations() ) meta = {'log_entry_durations':log_entry_durations} # create JSON query payload if index_left==index_painted: targets = [ {'index':index_left,'label':'left','flag':1}, {'index':index_right,'label':'right','flag':0} ] else: targets = [ {'index':index_left,'label':'left','flag':0}, {'index':index_right,'label':'right','flag':1} ] timestamp = str(utils.datetimeNow()) query_uid = utils.getNewUID() query = {} query['query_uid'] = query_uid query['target_indices'] = targets # save query data to database query_doc = {} query_doc.update(query) query_doc['participant_uid'] = participant_uid query_doc['alg_uid'] = alg_uid query_doc['exp_uid'] = exp_uid query_doc['alg_label'] = alg_label query_doc['timestamp_query_generated'] = timestamp db.set_doc(app_id+':queries',query_uid,query_doc) # add context after updating query doc to avoid redundant information query['context_type'] = context_type query['context'] = context args_out = {'args':query,'meta':meta} response_json = json.dumps(args_out) log_entry = { 'exp_uid':exp_uid,'task':'getQuery','json':response_json,'timestamp':utils.datetimeNow() } ell.log( app_id+':APP-RESPONSE', log_entry ) return response_json,True,'' except Exception, err: error = traceback.format_exc() log_entry = { 'exp_uid':exp_uid,'task':'getQuery','error':error,'timestamp':utils.datetimeNow(),'args_json':args_json } ell.log( app_id+':APP-EXCEPTION', log_entry ) return '{}',False,error
def predict(self, exp_uid, args_json, db, ell): """ uses current model empirical estimates to forecast which index has the highest mean Expected input (in json structure with string keys): (string) predict_id : identifier for the desired prediction (list) params : dictionary of stat_id specific fields. ########## Description: uses current model empirical estimates to forecast the ranking of the arms in order of likelhood of being the best from most to least likely Expected input: (string) predict_id : 'arm_ranking' (dict) params : dictionary with fields (string) alg_label : describes target algorithm to use Expected output (in json structure): (list of dicts with fields): (int) index : index of target (int) rank : rank that the algorithm assigns to index (rank 0 is most likely the best arm) """ try: app_id = self.app_id log_entry = { 'exp_uid': exp_uid, 'task': 'predict', 'json': args_json, 'timestamp': utils.datetimeNow() } ell.log(app_id + ':APP-CALL', log_entry) # convert args_json to args_dict try: args_dict = json.loads(args_json) except: error = "%s.predict failed to convert input args_json due to improper format" % ( self.app_id) return '{}', False, error # check for the fields that must be contained in args or error occurs necessary_fields = ['predict_id', 'params'] for field in necessary_fields: try: args_dict[field] except KeyError: error = "%s.predict input arguments missing field: %s" % ( self.app_id, str(field)) return '{}', False, error predict_id = args_dict['predict_id'] params = args_dict['params'] if predict_id == "arm_ranking": alg_label = params['alg_label'] # get list of algorithms associated with project alg_list, didSucceed, message = db.get(app_id + ':experiments', exp_uid, 'alg_list') # get alg_id for algorithm in alg_list: if alg_label == algorithm['alg_label']: alg_id = algorithm['alg_id'] alg_uid = algorithm['alg_uid'] num_reported_answers, didSucceed, message = db.get( app_id + ':experiments', exp_uid, 'num_reported_answers_for_' + alg_uid) if type(num_reported_answers) != int: num_reported_answers = 0 # get sandboxed database for the specific app_id,alg_id,exp_uid - closing off the rest of the database to the algorithm rc = ResourceClient(app_id, exp_uid, alg_uid, db) # get specific algorithm to make calls to alg = utils.get_app_alg(self.app_id, alg_id) # call getQuery scores, precisions, dt = utils.timeit(alg.predict)(resource=rc) log_entry_durations = { 'exp_uid': exp_uid, 'alg_uid': alg_uid, 'task': 'predict', 'duration': dt } log_entry_durations.update(rc.getDurations()) meta = {'log_entry_durations': log_entry_durations} import numpy ranks = (-numpy.array(scores)).argsort().tolist() n = len(scores) indexes = numpy.array(range(n))[ranks] scores = numpy.array(scores)[ranks] precisions = numpy.array(precisions)[ranks] ranks = range(n) targets = [] for index in range(n): targets.append({ 'index': indexes[index], 'rank': ranks[index], 'score': scores[index], 'precision': precisions[index] }) log_entry = { 'exp_uid': exp_uid, 'alg_uid': alg_uid, 'timestamp': utils.datetimeNow() } log_entry.update({ 'targets': targets, 'num_reported_answers': num_reported_answers }) ell.log(app_id + ':ALG-EVALUATION', log_entry) response_args_dict = {} args_out = {'args': response_args_dict, 'meta': meta} predict_json = json.dumps(args_out) log_entry = { 'exp_uid': exp_uid, 'task': 'predict', 'json': predict_json, 'timestamp': utils.datetimeNow() } ell.log(app_id + ':APP-RESPONSE', log_entry) return predict_json, True, '' except Exception, err: error = traceback.format_exc() log_entry = { 'exp_uid': exp_uid, 'task': 'predict', 'error': str(error), 'timestamp': utils.datetimeNow(), 'args_json': args_json } didSucceed, message = ell.log(app_id + ':APP-EXCEPTION', log_entry) return '{}', False, error
def processAnswer(self,exp_uid,args_json,db,ell): """ reporting back the reward of pulling the arm suggested by getQuery Expected input (in json structure with string keys): (index) index_winner : index of the winner in (must be index of left or right target in target_indices) (str) query_uid : unique identifier of query Expected output (comma separated): if error: return (JSON) '{}', (bool) False, (str) error else: return (JSON) '{}', (bool) True,'' """ try: app_id = self.app_id log_entry = { 'exp_uid':exp_uid,'task':'processAnswer','json':args_json,'timestamp':utils.datetimeNow() } ell.log( app_id+':APP-CALL', log_entry ) # convert args_json to args_dict try: args_dict = json.loads(args_json) except: error = "%s.processAnswer input args_json is in improper format" % self.app_id return '{}',False,error # check for the fields that must be contained in args or error occurs necessary_fields = ['index_winner','query_uid'] for field in necessary_fields: try: args_dict[field] except KeyError: error = "%s.processAnswer input arguments missing field: %s" % (self.app_id,str(field)) return '{}',False,error # get list of algorithms associated with project alg_list,didSucceed,message = db.get(app_id+':experiments',exp_uid,'alg_list') # get alg_id query_uid = args_dict['query_uid'] alg_uid,didSucceed,message = db.get(app_id+':queries',query_uid,'alg_uid') if not didSucceed: raise Exception("Failed to retrieve query with query_uid="+query_uid) for algorithm in alg_list: if alg_uid == algorithm['alg_uid']: alg_id = algorithm['alg_id'] alg_label = algorithm['alg_label'] test_alg_label = algorithm['test_alg_label'] num_reported_answers,didSucceed,message = db.increment(app_id+':experiments',exp_uid,'num_reported_answers_for_'+alg_uid) # get sandboxed database for the specific app_id,alg_id,exp_uid - closing off the rest of the database to the algorithm rc = ResourceClient(app_id,exp_uid,alg_uid,db) # get specific algorithm to make calls to alg = utils.get_app_alg(self.app_id,alg_id) targets,didSucceed,message = db.get(app_id+':queries',query_uid,'target_indices') for target in targets: if target['label'] == 'center': index_center = target['index'] elif target['label'] == 'left': index_left = target['index'] elif target['label'] == 'right': index_right = target['index'] index_winner = args_dict['index_winner'] # update query doc timestamp_query_generated,didSucceed,message = db.get(app_id+':queries',query_uid,'timestamp_query_generated') datetime_query_generated = utils.str2datetime(timestamp_query_generated) timestamp_answer_received = args_dict.get('meta',{}).get('timestamp_answer_received',None) if timestamp_answer_received == None: datetime_answer_received = datetime_query_generated else: datetime_answer_received = utils.str2datetime(timestamp_answer_received) delta_datetime = datetime_answer_received - datetime_query_generated round_trip_time = delta_datetime.seconds + delta_datetime.microseconds/1000000. response_time = float(args_dict.get('response_time',0.)) db.set(app_id+':queries',query_uid,'response_time',response_time) db.set(app_id+':queries',query_uid,'network_delay',round_trip_time-response_time) db.set(app_id+':queries',query_uid,'index_winner',index_winner) q = [index_left,index_right,index_center] if index_winner==index_right: q = [index_right,index_left,index_center] db.set(app_id+':queries',query_uid,'q',q) # call processAnswer didSucceed,dt = utils.timeit(alg.processAnswer)(resource=rc,index_center=index_center,index_left=index_left,index_right=index_right,index_winner=index_winner) log_entry_durations = { 'exp_uid':exp_uid,'alg_uid':alg_uid,'task':'processAnswer','duration':dt } log_entry_durations.update( rc.getDurations() ) meta = {'log_entry_durations':log_entry_durations} # check if we're going to evaluate this loss n,didSucceed,message = db.get(app_id+':experiments',exp_uid,'n') if num_reported_answers % ((n+4)/4) == 0: predict_id = 'get_embedding' params = {'alg_label':alg_label} predict_args_dict = {'predict_id':predict_id,'params':params} predict_args_json = json.dumps(predict_args_dict) db.submit_job(app_id,exp_uid,'predict',predict_args_json,ignore_result=True) ############### response_args_dict = {} args_out = {'args':response_args_dict,'meta':meta} response_json = json.dumps(args_out) log_entry = { 'exp_uid':exp_uid,'task':'processAnswer','json':response_json,'timestamp':utils.datetimeNow() } ell.log( app_id+':APP-RESPONSE', log_entry ) return response_json,True,"" except Exception, err: error = traceback.format_exc() log_entry = { 'exp_uid':exp_uid,'task':'processAnswer','error':error,'timestamp':utils.datetimeNow() } ell.log( app_id+':APP-EXCEPTION', log_entry ) return '{}',False,error
def predict(self,exp_uid,args_json,db,ell): """ Have the model learned by some particular algorithm predict a variety of stuff Expected input (in json structure with string keys): (string) predict_id : identifier for the desired prediction (list) params : dictionary of stat_id specific fields. ########## Description: Each algorithm (with an associated alg_label) has a test_alg_label associated with it. Expected input: (string) predict_id : 'evaluate_on_test' (dict) params : dictionary with fields (string) alg_label : describes target algorithm to test (string) test_alg_label : describes the algorithm that whos triplets are evaluated on the target algorithm Expected output (in json structure): (float) num_reported_answers : number of reported answers after which the calculation was made (float) error : 0/1 loss on test set """ try: app_id = self.app_id log_entry = { 'exp_uid':exp_uid,'task':'predict','json':args_json,'timestamp':utils.datetimeNow() } ell.log( app_id+':APP-CALL', log_entry ) # convert args_json to args_dict try: args_dict = json.loads(args_json) except: error = "%s.predict input args_json is in improper format" % self.app_id return '{}',False,error # check for the fields that must be contained in args or error occurs necessary_fields = ['predict_id','params'] for field in necessary_fields: try: args_dict[field] except KeyError: error = "%s.predict input arguments missing field: %s" % (self.app_id,str(field)) return '{}',False,error predict_id = args_dict['predict_id'] params = args_dict['params'] alg_label = params['alg_label'] # get list of algorithms associated with project alg_list,didSucceed,message = db.get(app_id+':experiments',exp_uid,'alg_list') # get alg_id for algorithm in alg_list: if alg_label == algorithm['alg_label']: alg_id = algorithm['alg_id'] alg_uid = algorithm['alg_uid'] meta = {} if predict_id=='get_embedding': # get sandboxed database for the specific app_id,alg_id,exp_uid - closing off the rest of the database to the algorithm rc = ResourceClient(app_id,exp_uid,alg_uid,db) # get specific algorithm to make calls to alg = utils.get_app_alg(self.app_id,alg_id) ##### Get Embedding ##### Xd,num_reported_answers,dt = utils.timeit(alg.predict)(rc) log_entry_durations = { 'exp_uid':exp_uid,'alg_uid':alg_uid,'task':'predict','duration':dt } log_entry_durations.update( rc.getDurations() ) meta = {'log_entry_durations':log_entry_durations} params['Xd'] = Xd params['num_reported_answers'] = num_reported_answers log_entry = { 'exp_uid':exp_uid,'alg_uid':alg_uid,'timestamp':utils.datetimeNow() } log_entry.update( params ) ell.log( app_id+':ALG-EVALUATION', log_entry ) params['timestamp'] = str(log_entry['timestamp']) response_args_dict = params elif predict_id=='get_queries': # get list of triplets from test queries,didSucceed,message = db.get_docs_with_filter(app_id+':queries',{'alg_uid':alg_uid}) S = [] for query in queries: if 'q' in query.keys(): q = query['q'] S.append(q) params['queries'] = S params['num_reported_answers'] = len(S) response_args_dict = params args_out = {'args':response_args_dict,'meta':meta} predict_json = json.dumps(args_out) log_entry = { 'exp_uid':exp_uid,'task':'predict','json':predict_json,'timestamp':utils.datetimeNow() } ell.log( app_id+':APP-RESPONSE', log_entry ) return predict_json,True,'' except Exception, err: error = traceback.format_exc() log_entry = { 'exp_uid':exp_uid,'task':'predict','error':str(error),'timestamp':utils.datetimeNow() } didSucceed,message = ell.log( app_id+':APP-EXCEPTION', log_entry ) return '{}',False,error
def processAnswer(self, exp_uid, args_json, db, ell): """ reporting back the reward of pulling the arm suggested by getQuery Expected input (in json structure with string keys): (str) query_uid : unique identifier of query (int) index_winner : index of arm must be {index_left,index_right} Expected output (comma separated): if error: return (JSON) '{}', (bool) False, (str) error else: return (JSON) '{}', (bool) True,'' Usage: processAnswer_args_json,didSucceed,message = app.processAnswer(db_API,exp_uid,processAnswer_args_json) Example input: processAnswer_args_json = {"query_uid": "4d02a9924f92138287edd17ca5feb6e1", "index_winner": 8} Example output: processAnswer_response_json = {} """ try: app_id = self.app_id log_entry = { 'exp_uid': exp_uid, 'task': 'processAnswer', 'json': args_json, 'timestamp': utils.datetimeNow() } ell.log(app_id + ':APP-CALL', log_entry) # convert args_json to args_dict try: args_dict = json.loads(args_json) except: error = "%s.processAnswer input args_json is in improper format" % self.app_id return '{}', False, error # check for the fields that must be contained in args or error occurs necessary_fields = ['index_winner', 'query_uid'] for field in necessary_fields: try: args_dict[field] except KeyError: error = "%s.processAnswer input arguments missing field: %s" % ( self.app_id, str(field)) return '{}', False, error # get list of algorithms associated with project alg_list, didSucceed, message = db.get(app_id + ':experiments', exp_uid, 'alg_list') # get alg_id query_uid = args_dict['query_uid'] alg_uid, didSucceed, message = db.get(app_id + ':queries', query_uid, 'alg_uid') for algorithm in alg_list: if alg_uid == algorithm['alg_uid']: alg_id = algorithm['alg_id'] alg_label = algorithm['alg_label'] num_reported_answers, didSucceed, message = db.increment( app_id + ':experiments', exp_uid, 'num_reported_answers_for_' + alg_uid) # get sandboxed database for the specific app_id,alg_id,exp_uid - closing off the rest of the database to the algorithm rc = ResourceClient(app_id, exp_uid, alg_uid, db) # get specific algorithm to make calls to alg = utils.get_app_alg(self.app_id, alg_id) targets, didSucceed, message = db.get(app_id + ':queries', query_uid, 'target_indices') for target in targets: if target['label'] == 'left': index_left = target['index'] if target['label'] == 'right': index_right = target['index'] if target['flag'] == 1: index_painted = target['index'] index_winner = args_dict['index_winner'] # update query doc timestamp_query_generated, didSucceed, message = db.get( app_id + ':queries', query_uid, 'timestamp_query_generated') datetime_query_generated = utils.str2datetime( timestamp_query_generated) timestamp_answer_received = args_dict.get('meta', {}).get( 'timestamp_answer_received', None) if timestamp_answer_received == None: datetime_answer_received = datetime_query_generated else: datetime_answer_received = utils.str2datetime( timestamp_answer_received) delta_datetime = datetime_answer_received - datetime_query_generated round_trip_time = delta_datetime.seconds + delta_datetime.microseconds / 1000000. response_time = float(args_dict.get('response_time', 0.)) db.set(app_id + ':queries', query_uid, 'response_time', response_time) db.set(app_id + ':queries', query_uid, 'network_delay', round_trip_time - response_time) db.set(app_id + ':queries', query_uid, 'index_winner', index_winner) # call processAnswer didSucceed, dt = utils.timeit(alg.processAnswer)( resource=rc, index_left=index_left, index_right=index_right, index_painted=index_painted, index_winner=index_winner) log_entry_durations = { 'exp_uid': exp_uid, 'alg_uid': alg_uid, 'task': 'processAnswer', 'duration': dt } log_entry_durations.update(rc.getDurations()) meta = {'log_entry_durations': log_entry_durations} ############### predict_id = 'arm_ranking' params = {'alg_label': alg_label} predict_args_dict = {'predict_id': predict_id, 'params': params} predict_args_json = json.dumps(predict_args_dict) db.submit_job(app_id, exp_uid, 'predict', predict_args_json, ignore_result=True) ############### response_args_dict = {} args_out = {'args': response_args_dict, 'meta': meta} response_json = json.dumps(args_out) log_entry = { 'exp_uid': exp_uid, 'task': 'processAnswer', 'json': response_json, 'timestamp': utils.datetimeNow() } ell.log(app_id + ':APP-RESPONSE', log_entry) return response_json, True, "" except Exception, err: error = traceback.format_exc() log_entry = { 'exp_uid': exp_uid, 'task': 'processAnswer', 'error': error, 'timestamp': utils.datetimeNow(), 'args_json': args_json } ell.log(app_id + ':APP-EXCEPTION', log_entry) return '{}', False, error
def getStats(self, exp_uid, args_json, db, ell): """ Get statistics for the experiment and its algorithms Expected input (in json structure with string keys): (string) stat_id : identifier for the desired statistic (dict) params : dictionary of stat_id specific fields. See Next.utils.get_app_supported_stats(app_id) ResourceManager.get_app_supported_stats(app_id) for a description of each of the available stats and their inputs and outputs """ try: app_id = self.app_id log_entry = { 'exp_uid': exp_uid, 'task': 'getStats', 'json': args_json, 'timestamp': utils.datetimeNow() } ell.log(app_id + ':APP-CALL', log_entry) # convert args_json to args_dict try: args_dict = json.loads(args_json) except: error = "%s.getStats input args_json is in improper format" % self.app_id return '{}', False, error # check for the fields that must be contained in args or error occurs necessary_fields = ['stat_id', 'params'] for field in necessary_fields: try: args_dict[field] except KeyError: error = "%s.getStats input arguments missing field: %s" % ( self.app_id, str(field)) return '{}', False, error stat_id = args_dict['stat_id'] params = args_dict['params'] dashboard = DuelingBanditsPureExplorationDashboard(db, ell) # input task if stat_id == "api_activity_histogram": task = params['task'] activity_stats = dashboard.api_activity_histogram( self.app_id, exp_uid, task) stats = activity_stats # input Noneokay elif stat_id == "api_processAnswer_activity_stacked_histogram": activity_stats = dashboard.api_processAnswer_activity_stacked_histogram( self.app_id, exp_uid) stats = activity_stats # input task elif stat_id == "compute_duration_multiline_plot": task = params['task'] compute_stats = dashboard.compute_duration_multiline_plot( self.app_id, exp_uid, task) stats = compute_stats # input task, alg_label elif stat_id == "compute_duration_detailed_stacked_area_plot": task = params['task'] alg_label = params['alg_label'] compute_detailed_stats = dashboard.compute_duration_detailed_stacked_area_plot( self.app_id, exp_uid, task, alg_label) stats = compute_detailed_stats # input alg_label elif stat_id == "response_time_histogram": alg_label = params['alg_label'] response_time_stats = dashboard.response_time_histogram( self.app_id, exp_uid, alg_label) stats = response_time_stats # input alg_label elif stat_id == "network_delay_histogram": alg_label = params['alg_label'] network_delay_stats = dashboard.network_delay_histogram( self.app_id, exp_uid, alg_label) stats = network_delay_stats elif stat_id == "most_current_ranking": alg_label = params['alg_label'] stats = dashboard.most_current_ranking(self.app_id, exp_uid, alg_label) response_json = json.dumps(stats) log_entry = { 'exp_uid': exp_uid, 'task': 'getStats', 'json': response_json, 'timestamp': utils.datetimeNow() } ell.log(app_id + ':APP-RESPONSE', log_entry) return response_json, True, '' except Exception, err: error = traceback.format_exc() log_entry = { 'exp_uid': exp_uid, 'task': 'getStats', 'error': error, 'timestamp': utils.datetimeNow(), 'args_json': args_json } ell.log(app_id + ':APP-EXCEPTION', log_entry) return '{}', False, error
def reportAnswer(self,exp_uid,args_json,db,ell): """ reporting back the reward of pulling the arm suggested by getQuery Expected input: (str) query_uid : unique identifier of query (int) index_winner : index of arm must be one of the indices given by getQuery Expected output (comma separated): if error: return (JSON) '{}', (bool) False, (str) error else: return (JSON) '{}', (bool) True,'' Usage: reportAnswer_args_json,didSucceed,message = app.reportAnswer(exp_uid,reportAnswer_args_json) Example input: reportAnswer_args_json = {"query_uid": "4d02a9924f92138287edd17ca5feb6e1", "index_winner": 8} Example output: reportAnswer_response_json = {} """ try: app_id = self.app_id log_entry = { 'exp_uid':exp_uid,'task':'reportAnswer','json':args_json,'timestamp':utils.datetimeNow() } ell.log( app_id+':APP-CALL', log_entry ) # convert args_json to args_dict try: args_dict = json.loads(args_json) except: error = "%s.reportAnswer input args_json is in improper format" % self.app_id return '{}',False,error # check for the fields that must be contained in args or error occurs necessary_fields = ['index_winner','query_uid'] for field in necessary_fields: try: args_dict[field] except KeyError: error = "%s.reportAnswer input arguments missing field: %s" % (self.app_id,str(field)) return '{}',False,error # get list of algorithms associated with project alg_list,didSucceed,message = db.get(app_id+':experiments',exp_uid,'alg_list') # get alg_id query_uid = args_dict['query_uid'] alg_uid,didSucceed,message = db.get(app_id+':queries',query_uid,'alg_uid') for algorithm in alg_list: if alg_uid == algorithm['alg_uid']: alg_id = algorithm['alg_id'] alg_label = algorithm['alg_label'] num_reported_answers,didSucceed,message = db.increment(app_id+':experiments',exp_uid,'num_reported_answers_for_'+alg_uid) # get sandboxed database for the specific app_id,alg_id,exp_uid - closing off the rest of the database to the algorithm rc = ResourceClient(app_id,exp_uid,alg_uid,db) # get specific algorithm to make calls to alg = utils.get_app_alg(self.app_id,alg_id) # get targets associated with the specific query targets,didSucceed,message = db.get(app_id+':queries',query_uid,'target_indices') target_indices = [] for target in targets: target_indices.append(target['index']) # get the index winner index_winner = args_dict['index_winner'] # update query doc timestamp_query_generated,didSucceed,message = db.get(app_id+':queries',query_uid,'timestamp_query_generated') datetime_query_generated = utils.str2datetime(timestamp_query_generated) timestamp_answer_received = args_dict.get('meta',{}).get('timestamp_answer_received',None) if timestamp_answer_received == None: datetime_answer_received = datetime_query_generated else: datetime_answer_received = utils.str2datetime(timestamp_answer_received) delta_datetime = datetime_answer_received - datetime_query_generated round_trip_time = delta_datetime.seconds + delta_datetime.microseconds/1000000. response_time = float(args_dict.get('response_time',0.)) db.set(app_id+':queries',query_uid,'response_time',response_time) db.set(app_id+':queries',query_uid,'network_delay',round_trip_time-response_time) db.set(app_id+':queries',query_uid,'index_winner',index_winner) # call reportAnswer didSucceed,dt = utils.timeit(alg.reportAnswer)(resource=rc,targets=target_indices,index_winner=index_winner) log_entry_durations = { 'exp_uid':exp_uid,'alg_uid':alg_uid,'task':'reportAnswer','duration':dt } log_entry_durations.update( rc.getDurations() ) meta = {'log_entry_durations':log_entry_durations} # calling predict ############### predict_id = 'arm_ranking' params = {'alg_label':alg_label} predict_args_dict = {'predict_id':predict_id,'params':params} predict_args_json = json.dumps(predict_args_dict) db.submit_job(app_id,exp_uid,'predict',predict_args_json,ignore_result=True) ############### response_args_dict = {} args_out = {'args':response_args_dict,'meta':meta} response_json = json.dumps(args_out) log_entry = { 'exp_uid':exp_uid,'task':'reportAnswer','json':response_json,'timestamp':utils.datetimeNow() } ell.log( app_id+':APP-RESPONSE', log_entry ) return response_json,True,"" except Exception, err: error = traceback.format_exc() log_entry = { 'exp_uid':exp_uid,'task':'reportAnswer','error':error,'timestamp':utils.datetimeNow(),'args_json':args_json } ell.log( app_id+':APP-EXCEPTION', log_entry ) return '{}',False,error
def daemonProcess(self,exp_uid,args_json,db,ell): try: app_id = self.app_id log_entry = { 'exp_uid':exp_uid,'task':'daemonProcess','json':args_json,'timestamp':utils.datetimeNow() } ell.log( app_id+':APP-CALL', log_entry ) # convert args_json to args_dict try: args_dict = json.loads(args_json) except: error = "%s.daemonProcess input args_json is in improper format" % self.app_id return '{}',False,error # check for the fields that must be contained in args or error occurs necessary_fields = ['alg_uid','daemon_args'] for field in necessary_fields: try: args_dict[field] except KeyError: error = "%s.daemonProcess input arguments missing field: %s" % (self.app_id,str(field)) return '{}',False,error alg_daemon_args = args_dict['daemon_args'] alg_uid = args_dict['alg_uid'] alg_id,didSucceed,message = db.get(app_id+':algorithms',alg_uid,'alg_id') # get sandboxed database for the specific app_id,alg_id,exp_uid - closing off the rest of the database to the algorithm rc = ResourceClient(app_id,exp_uid,alg_uid,db) # get specific algorithm to make calls to alg = utils.get_app_alg(self.app_id,alg_id) didSucceed,dt = utils.timeit(alg.daemonProcess)(resource=rc,daemon_args_dict=alg_daemon_args) log_entry_durations = { 'exp_uid':exp_uid,'alg_uid':alg_uid,'task':'daemonProcess','duration':dt } log_entry_durations.update( rc.getDurations() ) meta = {'log_entry_durations':log_entry_durations} daemon_message = {} args_out = {'args':daemon_message,'meta':meta} response_json = json.dumps(args_out) log_entry = { 'exp_uid':exp_uid,'task':'daemonProcess','json':response_json,'timestamp':utils.datetimeNow() } ell.log( app_id+':APP-RESPONSE', log_entry ) return response_json,True,'' except Exception, err: error = traceback.format_exc() log_entry = { 'exp_uid':exp_uid,'task':'daemonProcess','error':error,'timestamp':utils.datetimeNow(),'args_json':args_json } ell.log( app_id+':APP-EXCEPTION', log_entry ) return '{}',False,error
def daemonProcess(self, exp_uid, args_json, db, ell): try: app_id = self.app_id log_entry = { 'exp_uid': exp_uid, 'task': 'daemonProcess', 'json': args_json, 'timestamp': utils.datetimeNow() } ell.log(app_id + ':APP-CALL', log_entry) # convert args_json to args_dict try: args_dict = json.loads(args_json) except: error = "%s.daemonProcess input args_json is in improper format" % self.app_id return '{}', False, error # check for the fields that must be contained in args or error occurs necessary_fields = ['alg_uid', 'daemon_args'] for field in necessary_fields: try: args_dict[field] except KeyError: error = "%s.daemonProcess input arguments missing field: %s" % ( self.app_id, str(field)) return '{}', False, error alg_daemon_args = args_dict['daemon_args'] alg_uid = args_dict['alg_uid'] alg_id, didSucceed, message = db.get(app_id + ':algorithms', alg_uid, 'alg_id') # get sandboxed database for the specific app_id,alg_id,exp_uid - closing off the rest of the database to the algorithm # rc = ResourceClient(app_id,exp_uid,alg_uid,db) rc = ResourceClient(app_id, exp_uid, alg_uid, db) # get specific algorithm to make calls to alg = utils.get_app_alg(self.app_id, alg_id) didSucceed, dt = utils.timeit(alg.daemonProcess)( resource=rc, daemon_args_dict=alg_daemon_args) log_entry_durations = { 'exp_uid': exp_uid, 'alg_uid': alg_uid, 'task': 'daemonProcess', 'duration': dt } log_entry_durations.update(rc.getDurations()) meta = {'log_entry_durations': log_entry_durations} daemon_message = {} args_out = {'args': daemon_message, 'meta': meta} response_json = json.dumps(args_out) log_entry = { 'exp_uid': exp_uid, 'task': 'daemonProcess', 'json': response_json, 'timestamp': utils.datetimeNow() } ell.log(app_id + ':APP-RESPONSE', log_entry) return response_json, True, '' except Exception, err: error = traceback.format_exc() log_entry = { 'exp_uid': exp_uid, 'task': 'daemonProcess', 'error': error, 'timestamp': utils.datetimeNow(), 'args_json': args_json } ell.log(app_id + ':APP-EXCEPTION', log_entry) return '{}', False, error
def getStats(self,exp_uid,args_json,db,ell): """ Get statistics for the experiment and its algorithms Expected input (in json structure with string keys): (string) stat_id : identifier for the desired statistic (dict) params : dictionary of stat_id specific fields. See Next.utils.get_app_supported_stats(app_id) ResourceManager.get_app_supported_stats(app_id) for a description of each of the available stats and their inputs and outputs """ try: app_id = self.app_id log_entry = { 'exp_uid':exp_uid,'task':'getStats','json':args_json,'timestamp':utils.datetimeNow() } ell.log( app_id+':APP-CALL', log_entry ) # convert args_json to args_dict try: args_dict = json.loads(args_json) except: error = "%s.getStats input args_json is in improper format" % self.app_id return '{}',False,error # check for the fields that must be contained in args or error occurs necessary_fields = ['stat_id','params'] for field in necessary_fields: try: args_dict[field] except KeyError: error = "%s.getStats input arguments missing field: %s" % (self.app_id,str(field)) return '{}',False,error stat_id = args_dict['stat_id'] params = args_dict['params'] dashboard = TupleBanditsPureExplorationDashboard(db,ell) # input task if stat_id == "api_activity_histogram": task = params['task'] activity_stats = dashboard.api_activity_histogram(self.app_id,exp_uid,task) stats = activity_stats # input Noneokay elif stat_id == "api_reportAnswer_activity_stacked_histogram": activity_stats = dashboard.api_reportAnswer_activity_stacked_histogram(self.app_id,exp_uid) stats = activity_stats # input task elif stat_id == "compute_duration_multiline_plot": task = params['task'] compute_stats = dashboard.compute_duration_multiline_plot(self.app_id,exp_uid,task) stats = compute_stats # input task, alg_label elif stat_id == "compute_duration_detailed_stacked_area_plot": task = params['task'] alg_label = params['alg_label'] compute_detailed_stats = dashboard.compute_duration_detailed_stacked_area_plot(self.app_id,exp_uid,task,alg_label) stats = compute_detailed_stats # input alg_label elif stat_id == "response_time_histogram": alg_label = params['alg_label'] response_time_stats = dashboard.response_time_histogram(self.app_id,exp_uid,alg_label) stats = response_time_stats # input alg_label elif stat_id == "network_delay_histogram": alg_label = params['alg_label'] network_delay_stats = dashboard.network_delay_histogram(self.app_id,exp_uid,alg_label) stats = network_delay_stats elif stat_id == "most_current_ranking": alg_label = params['alg_label'] stats = dashboard.most_current_ranking(self.app_id,exp_uid,alg_label) response_json = json.dumps(stats) log_entry = { 'exp_uid':exp_uid,'task':'getStats','json':response_json,'timestamp':utils.datetimeNow() } ell.log( app_id+':APP-RESPONSE', log_entry ) return response_json,True,'' except Exception, err: error = traceback.format_exc() log_entry = { 'exp_uid':exp_uid,'task':'getStats','error':error,'timestamp':utils.datetimeNow(),'args_json':args_json } ell.log( app_id+':APP-EXCEPTION', log_entry ) return '{}',False,error
def initExp(self, exp_uid, args_json, db, ell): """ initialize the project and necessary experiments Expected input (in json structure with string keys): (int) n: number of arms (float) failure_probability : confidence [optional] (list of dicts) alg_list : with fields (Defaults given by Info.get_app_default_alg_list) (string) alg_id : valid alg_id for this app_id (string) alg_label : unique identifier for algorithm (e.g. may have experiment with repeated alg_id's, but alg_labels must be unqiue, will also be used for plot legends [optional] (string) test_alg_label : must be one of the alg_label's in alg_list (Default is self) [optional] (dict) algorithm_management_settings : dictionary with fields (string) 'mode' and (dict) 'params'. mode in {'pure_exploration','explore_exploit','fixed_proportions'}. Default is 'fixed_proportions' and allocates uniform probability to each algorithm. If mode=fixed_proportions then params is a dictionary that contains the field 'proportions' which is a list of dictionaries with fields 'alg_label' and 'proportion' for all algorithms in alg_list. All proportions must be positive and sum to 1 over all algs in alg_list [optional] (string) participant_to_algorithm_management : in {'one_to_one','one_to_many'}. Default is 'one_to_many'. [optional] (string) instructions [optional] (string) debrief [optional] (int) num_tries Expected output: if error: return (JSON) '{}', (bool) False, (str) error_str else: return (JSON) '{}', (bool) True,'' Usage: initExp_response_json,didSucceed,message = app.initExp(db_API,exp_uid,initExp_args_json) Example input: initExp_args_json = {"participant_to_algorithm_management": "one_to_many", "alg_list": [{"alg_label": "BR_LilUCB", "alg_id": "BR_LilUCB", "params": {}}], "algorithm_management_settings": {"params": {"proportions": [{"alg_label": "BR_LilUCB", "proportion": 1.0}]}, "mode": "fixed_proportions"}, "failure_probability": 0.01, "n": 10} Example output: initExp_response_json = {} """ try: app_id = self.app_id # remove any reminants of an experiment if it exists didSucceed, message = db.delete_docs_with_filter( 'experiments_admin', {'exp_uid': exp_uid}) didSucceed, message = db.delete_docs_with_filter( app_id + ':experiments', {'exp_uid': exp_uid}) didSucceed, message = db.delete_docs_with_filter( app_id + ':queries', {'exp_uid': exp_uid}) didSucceed, message = db.delete_docs_with_filter( app_id + ':participants', {'exp_uid': exp_uid}) didSucceed, message = db.delete_docs_with_filter( app_id + ':algorithms', {'exp_uid': exp_uid}) didSucceed, message = ell.delete_logs_with_filter( app_id + ':APP-CALL', {'exp_uid': exp_uid}) didSucceed, message = ell.delete_logs_with_filter( app_id + ':APP-RESPONSE', {'exp_uid': exp_uid}) didSucceed, message = ell.delete_logs_with_filter( app_id + ':APP-EXCEPTION', {'exp_uid': exp_uid}) didSucceed, message = ell.delete_logs_with_filter( app_id + ':ALG-DURATION', {'exp_uid': exp_uid}) didSucceed, message = ell.delete_logs_with_filter( app_id + ':ALG-EVALUATION', {'exp_uid': exp_uid}) # add indexes (only adds them if they do not already exist) didSucceed, message = db.ensure_index('experiments_admin', {'exp_uid': 1}) didSucceed, message = db.ensure_index(app_id + ':experiments', {'exp_uid': 1}) didSucceed, message = db.ensure_index(app_id + ':queries', {'query_uid': 1}) didSucceed, message = db.ensure_index(app_id + ':queries', {'exp_uid': 1}) didSucceed, message = db.ensure_index(app_id + ':queries', {'alg_uid': 1}) didSucceed, message = db.ensure_index(app_id + ':queries', {'participant_uid': 1}) didSucceed, message = db.ensure_index(app_id + ':participants', {'participant_uid': 1}) didSucceed, message = db.ensure_index(app_id + ':participants', {'exp_uid': 1}) didSucceed, message = db.ensure_index(app_id + ':algorithms', {'alg_uid': 1}) didSucceed, message = db.ensure_index(app_id + ':algorithms', {'exp_uid': 1}) didSucceed, message = ell.ensure_index(app_id + ':APP-CALL', {'exp_uid': 1}) didSucceed, message = ell.ensure_index(app_id + ':APP-CALL', {'timestamp': 1}) didSucceed, message = ell.ensure_index(app_id + ':APP-CALL', { 'exp_uid': 1, 'timestamp': 1 }) didSucceed, message = ell.ensure_index(app_id + ':APP-CALL', { 'exp_uid': 1, 'task': 1 }) didSucceed, message = ell.ensure_index(app_id + ':APP-RESPONSE', {'exp_uid': 1}) didSucceed, message = ell.ensure_index(app_id + ':APP-RESPONSE', {'timestamp': 1}) didSucceed, message = ell.ensure_index(app_id + ':APP-RESPONSE', { 'exp_uid': 1, 'timestamp': 1 }) didSucceed, message = ell.ensure_index(app_id + ':APP-RESPONSE', { 'exp_uid': 1, 'task': 1 }) didSucceed, message = ell.ensure_index(app_id + ':APP-EXCEPTION', {'exp_uid': 1}) didSucceed, message = ell.ensure_index(app_id + ':APP-EXCEPTION', {'timestamp': 1}) didSucceed, message = ell.ensure_index(app_id + ':APP-EXCEPTION', { 'exp_uid': 1, 'timestamp': 1 }) didSucceed, message = ell.ensure_index(app_id + ':APP-EXCEPTION', { 'exp_uid': 1, 'task': 1 }) didSucceed, message = ell.ensure_index(app_id + ':ALG-DURATION', {'exp_uid': 1}) didSucceed, message = ell.ensure_index(app_id + ':ALG-DURATION', {'alg_uid': 1}) didSucceed, message = ell.ensure_index(app_id + ':ALG-DURATION', {'timestamp': 1}) didSucceed, message = ell.ensure_index(app_id + ':ALG-DURATION', { 'exp_uid': 1, 'timestamp': 1 }) didSucceed, message = ell.ensure_index(app_id + ':ALG-DURATION', { 'alg_uid': 1, 'task': 1 }) didSucceed, message = ell.ensure_index(app_id + ':ALG-EVALUATION', {'exp_uid': 1}) didSucceed, message = ell.ensure_index(app_id + ':ALG-EVALUATION', {'alg_uid': 1}) didSucceed, message = ell.ensure_index(app_id + ':ALG-EVALUATION', {'timestamp': 1}) didSucceed, message = ell.ensure_index(app_id + ':ALG-EVALUATION', { 'exp_uid': 1, 'timestamp': 1 }) db.set('experiments_admin', exp_uid, 'exp_uid', exp_uid) db.set('experiments_admin', exp_uid, 'app_id', app_id) db.set('experiments_admin', exp_uid, 'start_date', utils.datetime2str(utils.datetimeNow())) log_entry = { 'exp_uid': exp_uid, 'task': 'initExp', 'json': args_json, 'timestamp': utils.datetimeNow() } ell.log(app_id + ':APP-CALL', log_entry) # convert args_json to args_dict try: args_dict = json.loads(args_json) except: error = "%s.initExp input args_json is in improper format" % self.app_id return '{}', False, error # check for the fields that must be contained in args or error occurs necessary_fields = ['n', 'failure_probability'] for field in necessary_fields: try: args_dict[field] except KeyError: error = "%s.initExp input arguments missing field: %s" % ( self.app_id, str(field)) return '{}', False, error n = args_dict['n'] delta = args_dict['failure_probability'] if 'alg_list' in args_dict: alg_list = args_dict['alg_list'] supportedAlgs = utils.get_app_supported_algs(self.app_id) for algorithm in alg_list: if algorithm['alg_id'] not in supportedAlgs: error = "%s.initExp unsupported algorithm '%s' in alg_list" % ( self.app_id, alg_id) return '{}', False, error else: alg_list = utils.get_app_default_alg_list(self.app_id) if 'instructions' not in args_dict: instructions = utils.get_app_default_instructions(app_id) else: instructions = args_dict['instructions'] if 'debrief' not in args_dict: debrief = utils.get_app_default_instructions(app_id) else: debrief = args_dict['debrief'] if 'num_tries' not in args_dict: num_tries = utils.get_app_default_num_tries(app_id) else: num_tries = args_dict['num_tries'] if 'context_type' not in args_dict: context_type = 'none' else: context_type = args_dict['context_type'] if 'context' not in args_dict: context = '' else: context = args_dict['context'] # ALGORITHM_MANAGEMENT_MODE FORMATTING CHECK if 'algorithm_management_settings' not in args_dict: params = {} params['proportions'] = [] for algorithm in alg_list: params['proportions'].append({ 'alg_label': algorithm['alg_label'], 'proportion': 1. / len(alg_list) }) algorithm_management_settings = {} algorithm_management_settings['mode'] = 'fixed_proportions' algorithm_management_settings['params'] = params else: algorithm_management_settings = args_dict[ 'algorithm_management_settings'] try: mode = algorithm_management_settings['mode'] params = algorithm_management_settings['params'] except: error = "%s.initExp algorithm_management_settings must be a dictionary with fields 'mode' and 'params'" % ( self.app_id) return '{}', False, error if mode == 'fixed_proportions': try: algorithm_proportions_list = params['proportions'] except: error = "%s.initExp algorithm_management_settings['params'] must be a dictionary with field 'proportions'" % ( self.app_id) return '{}', False, error # check if alg_labels are properly labeled for proportion_item in algorithm_proportions_list: proportion = proportion_item['proportion'] target_alg_label = proportion_item['alg_label'] target_alg_label_in_alg_list = False for algorithm in alg_list: if algorithm['alg_label'] == target_alg_label: target_alg_label_in_alg_list = True if not target_alg_label_in_alg_list: error = "%s.initExp algorithm_management_settings['params']['proportions'] must be a list of dictionaries, each dictionary containing the fields 'alg_label' and 'proportion'. The 'alg_label' value must be one of the alg_labels in a provided alg_list and 'proportion' must be nonnegative and sum to 1 : '%s' not in provided alg_list" % ( self.app_id, target_alg_label) return '{}', False, error elif mode == 'pure_exploration': error = "%s.initExp Sorry, '%s' is not yet supported." % ( self.app_id, mode) return '{}', False, error elif mode == 'explore_exploit': error = "%s.initExp Sorry, '%s' is not yet supported." % ( self.app_id, mode) return '{}', False, error else: error = "%s.initExp unsupported algorithm_management_mode: '%s'. Must be in {'pure_exploration','explore_exploit','fixed_proportions'}" % ( self.app_id, algorithm_management_mode) return '{}', False, error # ALGORITHM_MANAGEMENT_MODE FORMATTING CHECK if 'participant_to_algorithm_management' not in args_dict: participant_to_algorithm_management = 'one_to_many' else: participant_to_algorithm_management = args_dict[ 'participant_to_algorithm_management'] if participant_to_algorithm_management not in [ 'one_to_many', 'one_to_one' ]: error = "%s.initExp unsupported participant_to_algorithm_management: '%s'. Must be in {'one_to_many','one_to_one'}" % ( self.app_id, participant_to_algorithm_management) return '{}', False, error # assign uid to each algorithm and save it for algorithm in alg_list: alg_uid = utils.getNewUID() algorithm['alg_uid'] = alg_uid db.set(app_id + ':algorithms', alg_uid, 'alg_uid', alg_uid) db.set(app_id + ':algorithms', alg_uid, 'exp_uid', exp_uid) db.set(app_id + ':experiments', exp_uid, 'exp_uid', exp_uid) db.set(app_id + ':experiments', exp_uid, 'app_id', app_id) db.set(app_id + ':experiments', exp_uid, 'n', n) db.set(app_id + ':experiments', exp_uid, 'failure_probability', delta) db.set(app_id + ':experiments', exp_uid, 'alg_list', alg_list) db.set(app_id + ':experiments', exp_uid, 'algorithm_management_settings', algorithm_management_settings) db.set(app_id + ':experiments', exp_uid, 'participant_to_algorithm_management', participant_to_algorithm_management) db.set(app_id + ':experiments', exp_uid, 'instructions', instructions) db.set(app_id + ':experiments', exp_uid, 'debrief', debrief) db.set(app_id + ':experiments', exp_uid, 'context_type', context_type) db.set(app_id + ':experiments', exp_uid, 'context', context) db.set(app_id + ':experiments', exp_uid, 'num_tries', num_tries) # now create intitialize each algorithm for algorithm in alg_list: alg_id = algorithm['alg_id'] alg_uid = algorithm['alg_uid'] db.set(app_id + ':algorithms', alg_uid, 'alg_id', alg_id) db.set(app_id + ':algorithms', alg_uid, 'alg_uid', alg_uid) db.set(app_id + ':algorithms', alg_uid, 'exp_uid', exp_uid) # get sandboxed database for the specific app_id,alg_id,exp_uid - closing off the rest of the database to the algorithm rc = ResourceClient(app_id, exp_uid, alg_uid, db) # get specific algorithm to make calls to alg = utils.get_app_alg(self.app_id, alg_id) # call initExp didSucceed, dt = utils.timeit(alg.initExp)( resource=rc, n=n, failure_probability=delta) log_entry = { 'exp_uid': exp_uid, 'alg_uid': alg_uid, 'task': 'initExp', 'duration': dt, 'timestamp': utils.datetimeNow() } ell.log(app_id + ':ALG-DURATION', log_entry) response_json = '{}' log_entry = { 'exp_uid': exp_uid, 'task': 'initExp', 'json': response_json, 'timestamp': utils.datetimeNow() } ell.log(app_id + ':APP-RESPONSE', log_entry) return response_json, True, '' except Exception, err: error = traceback.format_exc() log_entry = { 'exp_uid': exp_uid, 'task': 'initExp', 'error': error, 'timestamp': utils.datetimeNow(), 'args_json': args_json } ell.log(app_id + ':APP-EXCEPTION', log_entry) return '{}', False, error
#!/usr/bin/python import time import traceback import next.utils as utils import next.broker.broker broker = next.broker.broker.JobBroker() while (1): timestamp = utils.datetime2str(utils.datetimeNow()) try: broker.refresh_domain_hashes() except Exception, err: error = traceback.format_exc() print error # wait time.sleep(2)
def processAnswer(self, exp_uid, args_json): try: args_dict = self.helper.convert_json(args_json) args_dict = verifier.verify(args_dict, self.reference_dict['processAnswer']['args']) # Update timing info in query query = self.butler.queries.get(uid=args_dict['args']['query_uid']) timestamp_answer_received = args_dict['args'].get('timestamp_answer_received', None) delta_datetime = utils.str2datetime(timestamp_answer_received) - \ utils.str2datetime(query['timestamp_query_generated']) round_trip_time = delta_datetime.total_seconds() response_time = float(args_dict['args'].get('response_time',0.)) query_update = self.call_app_fn(query['alg_label'], query['alg_id'], 'processAnswer', args_dict) query_update.update({'response_time':response_time, 'network_delay':round_trip_time - response_time, 'timestamp_answer_received': timestamp_answer_received }) self.butler.queries.set_many(uid=args_dict['args']['query_uid'],key_value_dict=query_update) return json.dumps({'args': {}, 'meta': {'log_entry_durations':self.log_entry_durations}}), True, '' except Exception, error: exc_type, exc_value, exc_traceback = sys.exc_info() full_error = str(traceback.format_exc())+'\n'+str(error) utils.debug_print("processAnswer Exception: " + full_error, color='red') log_entry = { 'exp_uid':exp_uid,'task':'processAnswer','error':full_error,'timestamp':utils.datetimeNow(),'args_json':args_json } self.butler.ell.log( self.app_id+':APP-EXCEPTION', log_entry ) traceback.print_tb(exc_traceback) raise Exception(error)
def getQuery(self,exp_uid,args_json,db,ell): """ A request to ask which k arms to duel next Expected input (in jsonstructure with string keys): (int) k : number of objects to display [optional] (string) participant_uid : unique identifier of session for a participant answering questions (that is, an email address is not good enough as the participant could participate in multiple exp_uids so it would not be unique against all experiments), if key non-existant particpant_uid is assigned as exp_uid. Expected output (in json structure with string keys): (list) target_indices : list of k target indexes e.g. [ (int) target_index_1, ... , (int) target_index_k ] (str) query_uid : unique identifier of query (used to look up for reportAnswer) Usage: getQuery_response_json,didSucceed,message = app.getQuery(exp_uid,getQuery_args_json) Example input: getQuery_args_json = {"k": 3, "participant_uid": "0077110d03cf06b8f77d11acc399e8a7"} Example output: getQuery_response_json = {"query_uid": "4d02a9924f92138287edd17ca5feb6e1", "target_indices": [ 3, 6, 9 ] """ try: app_id = self.app_id log_entry = { 'exp_uid':exp_uid,'task':'getQuery','json':args_json,'timestamp':utils.datetimeNow() } ell.log( app_id+':APP-CALL', log_entry ) # convert args_json to args_dict try: args_dict = json.loads(args_json) except: error = "%s.initExp input args_json is in improper format" % self.app_id return '{}',False,error # get list of algorithms associated with project alg_list,didSucceed,message = db.get(app_id+':experiments',exp_uid,'alg_list') alg_label_to_alg_id = {} alg_label_to_alg_uid = {} for algorithm in alg_list: alg_label_to_alg_id[ algorithm['alg_label'] ] = algorithm['alg_id'] alg_label_to_alg_uid[ algorithm['alg_label'] ] = algorithm['alg_uid'] algorithm_management_settings,didSucceed,message = db.get(app_id+':experiments',exp_uid,'algorithm_management_settings') # ASSIGN ALGORITHM TO PARTICIPANT if 'participant_uid' in args_dict: participant_uid = args_dict['participant_uid'] else: participant_uid = exp_uid participant_doc_exists,didSucceed,message = db.exists(app_id+':participants',participant_uid,'participant_uid') first_participant_query = not participant_doc_exists if first_participant_query: db.set(app_id+':participants',participant_uid,'participant_uid',participant_uid) db.set(app_id+':participants',participant_uid,'exp_uid',exp_uid) participant_to_algorithm_management,didSucceed,message = db.get(app_id+':experiments',exp_uid,'participant_to_algorithm_management') if (participant_uid==exp_uid) or (participant_to_algorithm_management=='one_to_many') or (first_participant_query): if algorithm_management_settings['mode']=='fixed_proportions': proportions_list = algorithm_management_settings['params']['proportions'] prop = [ prop_item['proportion'] for prop_item in proportions_list ] prop_item = numpy.random.choice(alg_list,p=prop) else: raise Exception('algorithm_management_mode : '+algorithm_management_settings['mode']+' not implemented') alg_id = alg_label_to_alg_id[ prop_item['alg_label'] ] alg_uid = alg_label_to_alg_uid[ prop_item['alg_label'] ] alg_label = prop_item['alg_label'] if (first_participant_query) and (participant_to_algorithm_management=='one_to_one'): db.set(app_id+':participants',participant_uid,'alg_id',alg_id) db.set(app_id+':participants',participant_uid,'alg_uid',alg_uid) elif (participant_to_algorithm_management=='one_to_one'): # If here, then alg_uid should already be assigned in participant doc alg_id,didSucceed,message = db.get(app_id+':participants',participant_uid,'alg_id') alg_uid,didSucceed,message = db.get(app_id+':participants',participant_uid,'alg_uid') else: raise Exception('participant_to_algorithm_management : '+participant_to_algorithm_management+' not implemented') # get sandboxed database for the specific app_id,alg_id,exp_uid - closing off the rest of the database to the algorithm rc = ResourceClient(app_id,exp_uid,alg_uid,db) # get specific algorithm to make calls to alg = utils.get_app_alg(self.app_id,alg_id) # call getQuery targets,dt = utils.timeit(alg.getQuery)(resource=rc) # update targets target_list = [] for target in targets: target_list.append({'index':target}) # check for context context_type,didSucceed,message = db.get(app_id+':experiments',exp_uid,'context_type') context,didSucceed,message = db.get(app_id+':experiments',exp_uid,'context') # log log_entry_durations = { 'exp_uid':exp_uid,'alg_uid':alg_uid,'task':'getQuery','duration':dt } log_entry_durations.update( rc.getDurations() ) meta = {'log_entry_durations':log_entry_durations} # create JSON query payload timestamp = str(utils.datetimeNow()) query_uid = utils.getNewUID() query = {} query['query_uid'] = query_uid query['target_indices'] = target_list # save query data to database query_doc = {} query_doc.update(query) query_doc['participant_uid'] = participant_uid query_doc['alg_uid'] = alg_uid query_doc['exp_uid'] = exp_uid query_doc['alg_label'] = alg_label query_doc['timestamp_query_generated'] = timestamp for field in query_doc: db.set(app_id+':queries',query_uid,field,query_doc[field]) # add context after updating query doc to avoid redundant information query['context_type'] = context_type query['context'] = context args_out = {'args':query,'meta':meta} response_json = json.dumps(args_out) log_entry = { 'exp_uid':exp_uid,'task':'getQuery','json':response_json,'timestamp':utils.datetimeNow() } ell.log( app_id+':APP-RESPONSE', log_entry ) return response_json,True,'' except Exception, err: error = traceback.format_exc() log_entry = { 'exp_uid':exp_uid,'task':'getQuery','error':error,'timestamp':utils.datetimeNow(),'args_json':args_json } ell.log( app_id+':APP-EXCEPTION', log_entry ) return '{}',False,error
def init_alg(self, exp_uid, algorithm, alg_args): butler = Butler(self.app_id, exp_uid, self.myApp.TargetManager, self.butler.db, self.butler.ell, algorithm['alg_label'], algorithm['alg_id']) alg = utils.get_app_alg(self.app_id, algorithm['alg_id']) if 'args' in self.algs_reference_dict['initExp']: alg_args = verifier.verify(alg_args, self.algs_reference_dict['initExp']['args']) # I got rid of a timeit function here; it wasn't handling the # argument unpacking correctly? --Scott, 2016-3-7 # TODO: put dt back in and change log_entry to relfect that alg_response = alg.initExp(butler, **alg_args) alg_response = verifier.verify({'rets':alg_response}, {'rets':self.algs_reference_dict['initExp']['rets']}) log_entry = {'exp_uid':exp_uid, 'alg_label':algorithm['alg_label'], 'task':'initExp', 'duration':-1, 'timestamp':utils.datetimeNow()} self.butler.log('ALG-DURATION', log_entry)
def predict(self,exp_uid,args_json,db,ell): """ Description: uses current model empirical estimates to forecast the ranking of the arms in order of likelhood of being the best from most to least likely Expected input: (string) predict_id : 'arm_ranking' (dict) params : dictionary with fields (string) alg_label : describes target algorithm to use Expected output (in json structure): (list of dicts with fields): (int) index : index of target (int) rank : rank that the algorithm assigns to index (rank 0 is most likely the best arm) """ try: app_id = self.app_id log_entry = { 'exp_uid':exp_uid,'task':'predict','json':args_json,'timestamp':utils.datetimeNow() } ell.log( app_id+':APP-CALL', log_entry ) # convert args_json to args_dict try: args_dict = json.loads(args_json) except: error = "%s.predict failed to convert input args_json due to improper format" %(self.app_id) return '{}',False,error # check for the fields that must be contained in args or error occurs necessary_fields = ['predict_id','params'] for field in necessary_fields: try: args_dict[field] except KeyError: error = "%s.predict input arguments missing field: %s" % (self.app_id,str(field)) return '{}',False,error predict_id = args_dict['predict_id'] params = args_dict['params'] if predict_id == "arm_ranking": alg_label = params['alg_label'] # get list of algorithms associated with project alg_list,didSucceed,message = db.get(app_id+':experiments',exp_uid,'alg_list') # get alg_id for algorithm in alg_list: if alg_label == algorithm['alg_label']: alg_id = algorithm['alg_id'] alg_uid = algorithm['alg_uid'] num_reported_answers,didSucceed,message = db.get(app_id+':experiments',exp_uid,'num_reported_answers_for_'+alg_uid) if type(num_reported_answers)!=int: num_reported_answers=0 # get sandboxed database for the specific app_id,alg_id,exp_uid - closing off the rest of the database to the algorithm rc = ResourceClient(app_id,exp_uid,alg_uid,db) # get specific algorithm to make calls to alg = utils.get_app_alg(self.app_id,alg_id) # call getQuery scores,precisions,dt = utils.timeit(alg.predict)(resource=rc) log_entry_durations = { 'exp_uid':exp_uid,'alg_uid':alg_uid,'task':'predict','duration':dt } log_entry_durations.update( rc.getDurations() ) meta = {'log_entry_durations':log_entry_durations} import numpy ranks = (-numpy.array(scores)).argsort().tolist() n = len(scores) indexes = numpy.array(range(n))[ranks] scores = numpy.array(scores)[ranks] precisions = numpy.array(precisions)[ranks] ranks = range(n) targets = [] for index in range(n): targets.append( {'index':indexes[index],'rank':ranks[index],'score':scores[index],'precision':precisions[index]} ) log_entry = { 'exp_uid':exp_uid,'alg_uid':alg_uid,'timestamp':utils.datetimeNow() } log_entry.update( {'targets':targets,'num_reported_answers':num_reported_answers} ) ell.log( app_id+':ALG-EVALUATION', log_entry ) response_args_dict = {} args_out = {'args':response_args_dict,'meta':meta} predict_json = json.dumps(args_out) log_entry = { 'exp_uid':exp_uid,'task':'predict','json':predict_json,'timestamp':utils.datetimeNow() } ell.log( app_id+':APP-RESPONSE', log_entry ) return predict_json,True,'' except Exception, err: error = traceback.format_exc() log_entry = { 'exp_uid':exp_uid,'task':'predict','error':str(error),'timestamp':utils.datetimeNow(),'args_json':args_json } didSucceed,message = ell.log( app_id+':APP-EXCEPTION', log_entry ) return '{}',False,error
def getQuery(self, exp_uid, args_json, db, ell): """ A request to ask which two arms to duel next Expected input (in jsonstructure with string keys): [optional] (string) participant_uid : unique identifier of session for a participant answering questions (that is, an email address is not good enough as the participant could participate in multiple exp_uids so it would not be unique against all experiments), if key non-existant particpant_uid is assigned as exp_uid. Expected output (in json structure with string keys): (list) target_indices : list that stores dictionary of targets with fields: { (int) index : the index of the target of relevance (str) label : in {'left','right'} for display (int) flag : integer for algorithm's use } (str) query_uid : unique identifier of query (used to look up for processAnswer) Usage: getQuery_response_json,didSucceed,message = app.getQuery(db_API,exp_uid,getQuery_args_json) Example input: getQuery_args_json = {"participant_uid": "0077110d03cf06b8f77d11acc399e8a7"} Example output: getQuery_response_json = {"query_uid": "4d02a9924f92138287edd17ca5feb6e1", "target_indices": [{"index": 9, "flag": 0, "label": "left"}, {"index": 8, "flag": 1, "label": "right"}]} """ try: app_id = self.app_id log_entry = { 'exp_uid': exp_uid, 'task': 'getQuery', 'json': args_json, 'timestamp': utils.datetimeNow() } ell.log(app_id + ':APP-CALL', log_entry) # convert args_json to args_dict try: args_dict = json.loads(args_json) except: error = "%s.initExp input args_json is in improper format" % self.app_id return '{}', False, error # get list of algorithms associated with project alg_list, didSucceed, message = db.get(app_id + ':experiments', exp_uid, 'alg_list') alg_label_to_alg_id = {} alg_label_to_alg_uid = {} for algorithm in alg_list: alg_label_to_alg_id[ algorithm['alg_label']] = algorithm['alg_id'] alg_label_to_alg_uid[ algorithm['alg_label']] = algorithm['alg_uid'] algorithm_management_settings, didSucceed, message = db.get( app_id + ':experiments', exp_uid, 'algorithm_management_settings') # ASSIGN ALGORITHM TO PARTICIPANT if 'participant_uid' in args_dict: participant_uid = args_dict['participant_uid'] else: participant_uid = exp_uid participant_doc_exists, didSucceed, message = db.exists( app_id + ':participants', participant_uid, 'participant_uid') first_participant_query = not participant_doc_exists if first_participant_query: db.set(app_id + ':participants', participant_uid, 'participant_uid', participant_uid) db.set(app_id + ':participants', participant_uid, 'exp_uid', exp_uid) participant_to_algorithm_management, didSucceed, message = db.get( app_id + ':experiments', exp_uid, 'participant_to_algorithm_management') if (participant_uid == exp_uid) or ( participant_to_algorithm_management == 'one_to_many') or (first_participant_query): if algorithm_management_settings[ 'mode'] == 'fixed_proportions': proportions_list = algorithm_management_settings['params'][ 'proportions'] prop = [ prop_item['proportion'] for prop_item in proportions_list ] prop_item = numpy.random.choice(alg_list, p=prop) else: raise Exception('algorithm_management_mode : ' + algorithm_management_settings['mode'] + ' not implemented') alg_id = alg_label_to_alg_id[prop_item['alg_label']] alg_uid = alg_label_to_alg_uid[prop_item['alg_label']] alg_label = prop_item['alg_label'] if (first_participant_query) and ( participant_to_algorithm_management == 'one_to_one'): db.set(app_id + ':participants', participant_uid, 'alg_id', alg_id) db.set(app_id + ':participants', participant_uid, 'alg_uid', alg_uid) elif (participant_to_algorithm_management == 'one_to_one'): # If here, then alg_uid should already be assigned in participant doc alg_id, didSucceed, message = db.get(app_id + ':participants', participant_uid, 'alg_id') alg_uid, didSucceed, message = db.get(app_id + ':participants', participant_uid, 'alg_uid') else: raise Exception('participant_to_algorithm_management : ' + participant_to_algorithm_management + ' not implemented') # get sandboxed database for the specific app_id,alg_id,exp_uid - closing off the rest of the database to the algorithm rc = ResourceClient(app_id, exp_uid, alg_uid, db) # get specific algorithm to make calls to alg = utils.get_app_alg(self.app_id, alg_id) # call getQuery index_left, index_right, index_painted, dt = utils.timeit( alg.getQuery)(resource=rc) # check for context context_type, didSucceed, message = db.get(app_id + ':experiments', exp_uid, 'context_type') context, didSucceed, message = db.get(app_id + ':experiments', exp_uid, 'context') # log log_entry_durations = { 'exp_uid': exp_uid, 'alg_uid': alg_uid, 'task': 'getQuery', 'duration': dt } log_entry_durations.update(rc.getDurations()) meta = {'log_entry_durations': log_entry_durations} # create JSON query payload if index_left == index_painted: targets = [{ 'index': index_left, 'label': 'left', 'flag': 1 }, { 'index': index_right, 'label': 'right', 'flag': 0 }] else: targets = [{ 'index': index_left, 'label': 'left', 'flag': 0 }, { 'index': index_right, 'label': 'right', 'flag': 1 }] timestamp = str(utils.datetimeNow()) query_uid = utils.getNewUID() query = {} query['query_uid'] = query_uid query['target_indices'] = targets # save query data to database query_doc = {} query_doc.update(query) query_doc['participant_uid'] = participant_uid query_doc['alg_uid'] = alg_uid query_doc['exp_uid'] = exp_uid query_doc['alg_label'] = alg_label query_doc['timestamp_query_generated'] = timestamp db.set_doc(app_id + ':queries', query_uid, query_doc) # add context after updating query doc to avoid redundant information query['context_type'] = context_type query['context'] = context args_out = {'args': query, 'meta': meta} response_json = json.dumps(args_out) log_entry = { 'exp_uid': exp_uid, 'task': 'getQuery', 'json': response_json, 'timestamp': utils.datetimeNow() } ell.log(app_id + ':APP-RESPONSE', log_entry) return response_json, True, '' except Exception, err: error = traceback.format_exc() log_entry = { 'exp_uid': exp_uid, 'task': 'getQuery', 'error': error, 'timestamp': utils.datetimeNow(), 'args_json': args_json } ell.log(app_id + ':APP-EXCEPTION', log_entry) return '{}', False, error
def initExp(self,exp_uid,args_json,db,ell): """ initialize the project and necessary experiments Expected input (in json structure with string keys): (int) n: number of arms (int) k: number of objects to display (float) failure_probability : confidence [optional] (list of dicts) alg_list : with fields (Defaults given by Info.get_app_default_alg_list) (string) alg_id : valid alg_id for this app_id (string) alg_label : unique identifier for algorithm (e.g. may have experiment with repeated alg_id's, but alg_labels must be unqiue, will also be used for plot legends [optional] (string) test_alg_label : must be one of the alg_label's in alg_list (Default is self) [optional] (dict) algorithm_management_settings : dictionary with fields (string) 'mode' and (dict) 'params'. mode in {'pure_exploration','explore_exploit','fixed_proportions'}. Default is 'fixed_proportions' and allocates uniform probability to each algorithm. If mode=fixed_proportions then params is a dictionary that contains the field 'proportions' which is a list of dictionaries with fields 'alg_label' and 'proportion' for all algorithms in alg_list. All proportions must be positive and sum to 1 over all algs in alg_list [optional] (string) participant_to_algorithm_management : in {'one_to_one','one_to_many'}. Default is 'one_to_many'. [optional] (string) instructions [optional] (string) debrief Expected output: if error: return (JSON) '{}', (bool) False, (str) error_str else: return (JSON) '{}', (bool) True,'' Usage: initExp_response_json,didSucceed,message = app.initExp(exp_uid,initExp_args_json) Example input: initExp_args_json = {"participant_to_algorithm_management": "one_to_many", "alg_list": [{"alg_label": "BR_LilUCB", "alg_id": "BR_LilUCB", "params": {}}], "algorithm_management_settings": {"params": {"proportions": [{"alg_label": "BR_LilUCB", "proportion": 1.0}]}, "mode": "fixed_proportions"}, "failure_probability": 0.01, "n": 10} Example output: initExp_response_json = {} """ try: app_id = self.app_id # remove any reminants of an experiment if it exists didSucceed,message = db.delete_docs_with_filter('experiments_admin',{'exp_uid':exp_uid}) didSucceed,message = db.delete_docs_with_filter(app_id+':experiments',{'exp_uid':exp_uid}) didSucceed,message = db.delete_docs_with_filter(app_id+':queries',{'exp_uid':exp_uid}) didSucceed,message = db.delete_docs_with_filter(app_id+':participants',{'exp_uid':exp_uid}) didSucceed,message = db.delete_docs_with_filter(app_id+':algorithms',{'exp_uid':exp_uid}) didSucceed,message = ell.delete_logs_with_filter(app_id+':APP-CALL',{'exp_uid':exp_uid}) didSucceed,message = ell.delete_logs_with_filter(app_id+':APP-RESPONSE',{'exp_uid':exp_uid}) didSucceed,message = ell.delete_logs_with_filter(app_id+':APP-EXCEPTION',{'exp_uid':exp_uid}) didSucceed,message = ell.delete_logs_with_filter(app_id+':ALG-DURATION',{'exp_uid':exp_uid}) didSucceed,message = ell.delete_logs_with_filter(app_id+':ALG-EVALUATION',{'exp_uid':exp_uid}) # add indexes (only adds them if they do not already exist) didSucceed,message = db.ensure_index('experiments_admin',{'exp_uid':1}) didSucceed,message = db.ensure_index(app_id+':experiments',{'exp_uid':1}) didSucceed,message = db.ensure_index(app_id+':queries',{'query_uid':1}) didSucceed,message = db.ensure_index(app_id+':queries',{'exp_uid':1}) didSucceed,message = db.ensure_index(app_id+':queries',{'alg_uid':1}) didSucceed,message = db.ensure_index(app_id+':queries',{'participant_uid':1}) didSucceed,message = db.ensure_index(app_id+':participants',{'participant_uid':1}) didSucceed,message = db.ensure_index(app_id+':participants',{'exp_uid':1}) didSucceed,message = db.ensure_index(app_id+':algorithms',{'alg_uid':1}) didSucceed,message = db.ensure_index(app_id+':algorithms',{'exp_uid':1}) didSucceed,message = ell.ensure_index(app_id+':APP-CALL',{'exp_uid':1}) didSucceed,message = ell.ensure_index(app_id+':APP-CALL',{'timestamp':1}) didSucceed,message = ell.ensure_index(app_id+':APP-CALL',{'exp_uid':1,'timestamp':1}) didSucceed,message = ell.ensure_index(app_id+':APP-CALL',{'exp_uid':1,'task':1}) didSucceed,message = ell.ensure_index(app_id+':APP-RESPONSE',{'exp_uid':1}) didSucceed,message = ell.ensure_index(app_id+':APP-RESPONSE',{'timestamp':1}) didSucceed,message = ell.ensure_index(app_id+':APP-RESPONSE',{'exp_uid':1,'timestamp':1}) didSucceed,message = ell.ensure_index(app_id+':APP-RESPONSE',{'exp_uid':1,'task':1}) didSucceed,message = ell.ensure_index(app_id+':APP-EXCEPTION',{'exp_uid':1}) didSucceed,message = ell.ensure_index(app_id+':APP-EXCEPTION',{'timestamp':1}) didSucceed,message = ell.ensure_index(app_id+':APP-EXCEPTION',{'exp_uid':1,'timestamp':1}) didSucceed,message = ell.ensure_index(app_id+':APP-EXCEPTION',{'exp_uid':1,'task':1}) didSucceed,message = ell.ensure_index(app_id+':ALG-DURATION',{'exp_uid':1}) didSucceed,message = ell.ensure_index(app_id+':ALG-DURATION',{'alg_uid':1}) didSucceed,message = ell.ensure_index(app_id+':ALG-DURATION',{'timestamp':1}) didSucceed,message = ell.ensure_index(app_id+':ALG-DURATION',{'exp_uid':1,'timestamp':1}) didSucceed,message = ell.ensure_index(app_id+':ALG-DURATION',{'alg_uid':1,'task':1}) didSucceed,message = ell.ensure_index(app_id+':ALG-EVALUATION',{'exp_uid':1}) didSucceed,message = ell.ensure_index(app_id+':ALG-EVALUATION',{'alg_uid':1}) didSucceed,message = ell.ensure_index(app_id+':ALG-EVALUATION',{'timestamp':1}) didSucceed,message = ell.ensure_index(app_id+':ALG-EVALUATION',{'exp_uid':1,'timestamp':1}) db.set('experiments_admin',exp_uid,'exp_uid',exp_uid) db.set('experiments_admin',exp_uid,'app_id',app_id) db.set('experiments_admin',exp_uid,'start_date',utils.datetime2str(utils.datetimeNow())) log_entry = { 'exp_uid':exp_uid,'task':'initExp','json':args_json,'timestamp':utils.datetimeNow() } ell.log( app_id+':APP-CALL', log_entry ) # convert args_json to args_dict try: args_dict = json.loads(args_json) except: error = "%s.initExp input args_json is in improper format" % self.app_id return '{}',False,error # check for the fields that must be contained in args or error occurs necessary_fields = ['n','k','failure_probability'] for field in necessary_fields: try: args_dict[field] except KeyError: error = "%s.initExp input arguments missing field: %s" % (self.app_id,str(field)) return '{}',False,error n = args_dict['n'] k = args_dict['k'] delta = args_dict['failure_probability'] if 'alg_list' in args_dict: alg_list = args_dict['alg_list'] supportedAlgs = utils.get_app_supported_algs(self.app_id) for algorithm in alg_list: if algorithm['alg_id'] not in supportedAlgs: error = "%s.initExp unsupported algorithm '%s' in alg_list" % (self.app_id,alg_id) return '{}',False,error else: alg_list = utils.get_app_default_alg_list(self.app_id) if 'instructions' not in args_dict: instructions = utils.get_app_default_instructions(app_id) else: instructions = args_dict['instructions'] if 'debrief' not in args_dict: debrief = utils.get_app_default_instructions(app_id) else: debrief = args_dict['debrief'] if 'num_tries' not in args_dict: num_tries = utils.get_app_default_num_tries(app_id) else: num_tries = args_dict['num_tries'] if 'context_type' not in args_dict: context_type = 'none' else: context_type = args_dict['context_type'] if 'context' not in args_dict: context = '' else: context = args_dict['context'] # ALGORITHM_MANAGEMENT_MODE FORMATTING CHECK if 'algorithm_management_settings' not in args_dict: params = {} params['proportions'] = [] for algorithm in alg_list: params['proportions'].append( { 'alg_label': algorithm['alg_label'] , 'proportion':1./len(alg_list)} ) algorithm_management_settings = {} algorithm_management_settings['mode'] = 'fixed_proportions' algorithm_management_settings['params'] = params else: algorithm_management_settings = args_dict['algorithm_management_settings'] try: mode = algorithm_management_settings['mode'] params = algorithm_management_settings['params'] except: error = "%s.initExp algorithm_management_settings must be a dictionary with fields 'mode' and 'params'" % (self.app_id) return '{}',False,error if mode == 'fixed_proportions': try: algorithm_proportions_list = params['proportions'] except: error = "%s.initExp algorithm_management_settings['params'] must be a dictionary with field 'proportions'" % (self.app_id) return '{}',False,error # check if alg_labels are properly labeled for proportion_item in algorithm_proportions_list: proportion = proportion_item['proportion'] target_alg_label = proportion_item['alg_label'] target_alg_label_in_alg_list = False for algorithm in alg_list: if algorithm['alg_label']==target_alg_label: target_alg_label_in_alg_list = True if not target_alg_label_in_alg_list: error = "%s.initExp algorithm_management_settings['params']['proportions'] must be a list of dictionaries, each dictionary containing the fields 'alg_label' and 'proportion'. The 'alg_label' value must be one of the alg_labels in a provided alg_list and 'proportion' must be nonnegative and sum to 1 : '%s' not in provided alg_list" % (self.app_id,target_alg_label) return '{}',False,error elif mode == 'pure_exploration': error = "%s.initExp Sorry, '%s' is not yet supported." % (self.app_id,mode) return '{}',False,error elif mode == 'explore_exploit': error = "%s.initExp Sorry, '%s' is not yet supported." % (self.app_id,mode) return '{}',False,error else: error = "%s.initExp unsupported algorithm_management_mode: '%s'. Must be in {'pure_exploration','explore_exploit','fixed_proportions'}" % (self.app_id,algorithm_management_mode) return '{}',False,error # ALGORITHM_MANAGEMENT_MODE FORMATTING CHECK if 'participant_to_algorithm_management' not in args_dict: participant_to_algorithm_management = 'one_to_many' else: participant_to_algorithm_management = args_dict['participant_to_algorithm_management'] if participant_to_algorithm_management not in ['one_to_many','one_to_one']: error = "%s.initExp unsupported participant_to_algorithm_management: '%s'. Must be in {'one_to_many','one_to_one'}" % (self.app_id,participant_to_algorithm_management) return '{}',False,error # assign uid to each algorithm and save it for algorithm in alg_list: alg_uid = utils.getNewUID() algorithm['alg_uid'] = alg_uid db.set(app_id+':algorithms',alg_uid,'alg_uid',alg_uid) db.set(app_id+':algorithms',alg_uid,'exp_uid',exp_uid) db.set(app_id+':experiments',exp_uid,'exp_uid',exp_uid) db.set(app_id+':experiments',exp_uid,'app_id',app_id) db.set(app_id+':experiments',exp_uid,'n',n) db.set(app_id+':experiments',exp_uid,'k',k) db.set(app_id+':experiments',exp_uid,'failure_probability',delta) db.set(app_id+':experiments',exp_uid,'alg_list',alg_list) db.set(app_id+':experiments',exp_uid,'algorithm_management_settings',algorithm_management_settings) db.set(app_id+':experiments',exp_uid,'participant_to_algorithm_management',participant_to_algorithm_management) db.set(app_id+':experiments',exp_uid,'instructions',instructions) db.set(app_id+':experiments',exp_uid,'debrief',debrief) db.set(app_id+':experiments',exp_uid,'context_type',context_type) db.set(app_id+':experiments',exp_uid,'context',context) db.set(app_id+':experiments',exp_uid,'num_tries',num_tries) # now create intitialize each algorithm for algorithm in alg_list: alg_id = algorithm['alg_id'] alg_uid = algorithm['alg_uid'] db.set(app_id+':algorithms',alg_uid,'alg_id',alg_id) db.set(app_id+':algorithms',alg_uid,'alg_uid',alg_uid) db.set(app_id+':algorithms',alg_uid,'exp_uid',exp_uid) # get sandboxed database for the specific app_id,alg_id,exp_uid - closing off the rest of the database to the algorithm rc = ResourceClient(app_id,exp_uid,alg_uid,db) # get specific algorithm to make calls to alg = utils.get_app_alg(self.app_id,alg_id) # call initExp didSucceed,dt = utils.timeit(alg.initExp)(resource=rc,n=n,k=k,failure_probability=delta) log_entry = { 'exp_uid':exp_uid,'alg_uid':alg_uid,'task':'initExp','duration':dt,'timestamp':utils.datetimeNow() } ell.log( app_id+':ALG-DURATION', log_entry ) response_json = '{}' log_entry = { 'exp_uid':exp_uid,'task':'initExp','json':response_json,'timestamp':utils.datetimeNow() } ell.log( app_id+':APP-RESPONSE', log_entry ) return response_json,True,'' except Exception, err: error = traceback.format_exc() log_entry = { 'exp_uid':exp_uid,'task':'initExp','error':error,'timestamp':utils.datetimeNow(),'args_json':args_json } ell.log( app_id+':APP-EXCEPTION', log_entry ) return '{}',False,error
def predict(self,exp_uid,args_json,db,ell): """ Have the model learned by some particular algorithm predict a variety of stuff Expected input (in json structure with string keys): (string) predict_id : identifier for the desired prediction (list) params : dictionary of stat_id specific fields. ########## Description: Each algorithm (with an associated alg_label) has a test_alg_label associated with it. Expected input: (string) predict_id : 'evaluate_on_test' (dict) params : dictionary with fields (string) alg_label : describes target algorithm to test (string) test_alg_label : describes the algorithm that whos triplets are evaluated on the target algorithm Expected output (in json structure): (float) num_reported_answers : number of reported answers after which the calculation was made (float) error : 0/1 loss on test set """ try: app_id = self.app_id log_entry = { 'exp_uid':exp_uid,'task':'predict','json':args_json,'timestamp':utils.datetimeNow() } ell.log( app_id+':APP-CALL', log_entry ) # convert args_json to args_dict try: args_dict = json.loads(args_json) except: error = "%s.predict input args_json is in improper format" % self.app_id return '{}',False,error # check for the fields that must be contained in args or error occurs necessary_fields = ['predict_id','params'] for field in necessary_fields: try: args_dict[field] except KeyError: error = "%s.predict input arguments missing field: %s" % (self.app_id,str(field)) return '{}',False,error predict_id = args_dict['predict_id'] params = args_dict['params'] if predict_id == "evaluate_on_test": alg_label = params['alg_label'] test_alg_label = params['test_alg_label'] # get list of algorithms associated with project alg_list,didSucceed,message = db.get(app_id+':experiments',exp_uid,'alg_list') # get alg_id for algorithm in alg_list: if alg_label == algorithm['alg_label']: alg_id = algorithm['alg_id'] alg_uid = algorithm['alg_uid'] num_reported_answers,didSucceed,message = db.get(app_id+':experiments',exp_uid,'num_reported_answers_for_'+alg_uid) if type(num_reported_answers)!=int: num_reported_answers=0 if test_alg_label == algorithm['alg_label']: test_alg_id = algorithm['alg_id'] test_alg_uid = algorithm['alg_uid'] # get list of triplets from test queries,didSucceed,message = db.get_docs_with_filter(app_id+':queries',{'alg_uid':test_alg_uid}) test_S = [] for query in queries: if 'q' in query.keys(): q = query['q'] test_S.append(q) # get sandboxed database for the specific app_id,alg_id,exp_uid - closing off the rest of the database to the algorithm rc = ResourceClient(app_id,exp_uid,alg_uid,db) # get specific algorithm to make calls to alg = utils.get_app_alg(self.app_id,alg_id) ##### Triplet Predict ##### # call predict if len(test_S)>0: labels = [] for idx,q in enumerate(test_S): labels.append(1) R = numpy.random.randn() if R < 0: test_S[idx] = [ q[1], q[0], q[2] ] labels[idx] = -1 test_y,dt = utils.timeit(alg.predict)(rc,test_S) log_entry_durations = { 'exp_uid':exp_uid,'alg_uid':alg_uid,'task':'predict','duration':dt } log_entry_durations.update( rc.getDurations() ) meta = {'log_entry_durations':log_entry_durations} # compute error rate: test_y should match labels number_correct = 0. for i in range(len(test_S)): if test_y[i]*labels[i] > 0: number_correct += 1.0 accuracy = number_correct/len(test_S) err = 1.0-accuracy else: err = 0.5 labels = [] test_y = [] params['num_reported_answers'] = num_reported_answers params['error'] = err params['num_test_triplets'] = len(test_S) params['labels'] = labels params['test_y'] = test_y ##### Get Embedding ##### Xd,X2 = alg.getStats(rc) params['Xd'] = Xd params['X2'] = X2 log_entry = { 'exp_uid':exp_uid,'alg_uid':alg_uid,'timestamp':utils.datetimeNow() } log_entry.update( params ) ell.log( app_id+':ALG-EVALUATION', log_entry ) data = params response_args_dict = data args_out = {'args':response_args_dict,'meta':meta} predict_json = json.dumps(args_out) log_entry = { 'exp_uid':exp_uid,'task':'predict','json':predict_json,'timestamp':utils.datetimeNow() } ell.log( app_id+':APP-RESPONSE', log_entry ) return predict_json,True,'' except Exception, err: error = traceback.format_exc() log_entry = { 'exp_uid':exp_uid,'task':'predict','error':str(error),'timestamp':utils.datetimeNow() } didSucceed,message = ell.log( app_id+':APP-EXCEPTION', log_entry ) return '{}',False,error
where {hostname} and {port} are as they are below """ import sys sys.path.append("/next_backend") import time import traceback import next.utils as utils import subprocess import next.constants as constants import os while(1): timestamp = utils.datetimeNow() print "[ %s ] Calling database daemon..." % str(timestamp) subprocess.call('python ./next/database/database_backup.py',shell=True) time.sleep(3600*6) # once every 6 hours
def getQuery(self, exp_uid, args_json): try: args_dict = self.helper.convert_json(args_json) args_dict = verifier.verify(args_dict, self.reference_dict['getQuery']['args']) experiment_dict = self.butler.experiment.get() alg_list = experiment_dict['args']['alg_list'] participant_to_algorithm_management = experiment_dict['args']['participant_to_algorithm_management'] algorithm_management_settings = experiment_dict['args']['algorithm_management_settings'] # Create the participant dictionary in participants bucket if needed. Also pull out label and id for this algorithm participant_uid = args_dict['args'].get('participant_uid', args_dict['exp_uid']) # Check to see if the first participant has come by and if not, save to db participant_doc = self.butler.participants.get(uid=participant_uid) first_participant_query = participant_doc==None if first_participant_query: participant_doc = {} self.butler.participants.set(uid=participant_uid, value={'exp_uid':exp_uid, 'participant_uid':participant_uid}) if (participant_uid == exp_uid) or (participant_to_algorithm_management == 'one_to_many') or (first_participant_query): if algorithm_management_settings['mode'] == 'fixed_proportions': labels = [alg['alg_label'] for alg in algorithm_management_settings['params']] prop = [prop_item['proportion'] for prop_item in algorithm_management_settings['params']] # reorder prop and alg_list to have same order new_alg_list = [] broken = False for label in labels: broken = False for alg in alg_list: if label == alg['alg_label']: new_alg_list += [alg] broken = True break if not broken: raise Exception('alg_label not present for both porportions and labels') chosen_alg = numpy.random.choice(new_alg_list, p=prop) elif algorithm_management_settings['mode'] == 'custom' : chosen_alg = self.myApp.chooseAlg(self.butler, alg_list, args_dict['args']) else: chosen_alg = numpy.random.choice(alg_list) alg_id = chosen_alg['alg_id'] alg_label = chosen_alg['alg_label'] if (first_participant_query) and (participant_to_algorithm_management=='one_to_one'): self.butler.participants.set(uid=participant_uid, key='alg_id',value=alg_id) self.butler.participants.set(uid=participant_uid, key='alg_label',value=alg_label) elif (participant_to_algorithm_management=='one_to_one'): alg_id = participant_doc['alg_id'] alg_label = participant_doc['alg_label'] query_uid = utils.getNewUID() args_dict['args'].update(query_uid=query_uid) query_doc = self.call_app_fn(alg_label, alg_id, 'getQuery', args_dict) query_doc.update({'participant_uid':participant_uid, 'alg_id':alg_id, 'exp_uid':exp_uid, 'alg_label':alg_label, 'timestamp_query_generated':str(utils.datetimeNow()), 'query_uid':query_uid}) self.butler.queries.set(uid=query_uid, value=query_doc) return json.dumps({'args':query_doc,'meta':{'log_entry_durations':self.log_entry_durations}}), True,'' except Exception, error: exc_type, exc_value, exc_traceback = sys.exc_info() full_error = str(traceback.format_exc())+'\n'+str(error) utils.debug_print("getQuery Exception: " + full_error, color='red') log_entry = { 'exp_uid':exp_uid,'task':'getQuery','error':full_error,'timestamp':utils.datetimeNow(),'args_json':args_json } self.butler.ell.log( self.app_id+':APP-EXCEPTION', log_entry ) traceback.print_tb(exc_traceback) return '{}', False, str(error)
def most_current_embedding(self, app, butler, alg_label): """ Description: Returns embedding in the form of a list of dictionaries, which is conveneint for downstream applications Expected input: (string) alg_label : must be a valid alg_label contained in alg_list list of dicts Expected output (in dict): plot_type : 'scatter2d_noaxis' (float) x_min : minimum x-value to display in viewing box (float) x_max : maximum x-value to display in viewing box (float) y_min : minimum y-value to display in viewing box (float) y_max : maximum y-value to display in viewing box (list of dicts with fields) data : (int) index : index of target (float) x : x-value of target (float) y : y-value of target """ TargetManager = butler.targets item = app.getModel( json.dumps({ 'exp_uid': app.exp_uid, 'args': { 'alg_label': alg_label } })) embedding = item['X'] data = [] x_min = numpy.float('inf') x_max = -numpy.float('inf') y_min = numpy.float('inf') y_max = -numpy.float('inf') for idx, target in enumerate(embedding): target_dict = {} target_dict['target'] = TargetManager.get_target_item( app.exp_uid, idx) target_dict['x'] = target[ 0] # this is what will actually be plotted, try: target_dict['y'] = target[ 1] # takes first two components, (could be replaced by PCA) except: target_dict['y'] = 0. target_dict['darray'] = target x_min = min(x_min, target[0]) x_max = max(x_max, target[0]) y_min = min(y_min, target[1]) y_max = max(y_max, target[1]) data.append(target_dict) return_dict = { 'timestamp': str(utils.datetimeNow()), 'x_min': x_min, 'x_max': x_max, 'y_min': y_min, 'y_max': y_max, 'data': data, 'plot_type': 'scatter2d_noaxis' } return return_dict
def getQuery(self,exp_uid,args_json,db,ell): """ A request to ask the query: "is {center} more similar to {left} or {right}?" Expected input (in jsonstructure with string keys): [optional] (string) participant_uid : unique identifier of session for a participant answering questions (that is, an email address is not good enough as the participant could participate in multiple exp_uids so it would not be unique against all experiments), if key non-existant particpant_uid is assigned as exp_uid. Expected output (in json structure with string keys): (list) target_indices : list that stores dictionary of targets with fields: { (int) index : the index of the target of relevance (str) label : in {'left','right','center'} (int) flag : integer for algorithm's use } (str) query_uid : unique identifier of query (used to look up for reportAnswer) Usage: getQuery_response_json,didSucceed,message = app.getQuery(db_API,exp_uid,getQuery_args_json) Example input: getQuery_args_json = {"participant_uid": "ecaf1d60ab995b3c57afb3a1f3f288f0"} Example output: getQuery_response_json = {"query_uid": "a061ce00742603afc540d23e08ab77b3", "target_indices": [{"index": 19, "flag": 0, "label": "center"}, {"index": 8, "flag": 0, "label": "left"}, {"index": 15, "flag": 0, "label": "right"}]} """ try: app_id = self.app_id log_entry = { 'exp_uid':exp_uid,'task':'getQuery','json':args_json,'timestamp':utils.datetimeNow() } ell.log( app_id+':APP-CALL', log_entry ) # convert args_json to args_dict try: args_dict = json.loads(args_json) except: error = "%s.initExp input args_json is in improper format" % self.app_id return '{}',False,error # get list of algorithms associated with project alg_list,didSucceed,message = db.get(app_id+':experiments',exp_uid,'alg_list') alg_label_to_alg_id = {} alg_label_to_alg_uid = {} for algorithm in alg_list: alg_label_to_alg_id[ algorithm['alg_label'] ] = algorithm['alg_id'] alg_label_to_alg_uid[ algorithm['alg_label'] ] = algorithm['alg_uid'] algorithm_management_settings,didSucceed,message = db.get(app_id+':experiments',exp_uid,'algorithm_management_settings') # ASSIGN ALGORITHM TO PARTICIPANT if 'participant_uid' in args_dict: participant_uid = args_dict['participant_uid'] else: participant_uid = exp_uid participant_doc_exists,didSucceed,message = db.exists(app_id+':participants',participant_uid,'participant_uid') first_participant_query = not participant_doc_exists if first_participant_query: db.set(app_id+':participants',participant_uid,'participant_uid',participant_uid) db.set(app_id+':participants',participant_uid,'exp_uid',exp_uid) participant_to_algorithm_management,didSucceed,message = db.get(app_id+':experiments',exp_uid,'participant_to_algorithm_management') if (participant_uid==exp_uid) or (participant_to_algorithm_management=='one_to_many') or (first_participant_query): if algorithm_management_settings['mode']=='fixed_proportions': proportions_list = algorithm_management_settings['params']['proportions'] prop = [ prop_item['proportion'] for prop_item in proportions_list ] prop_item = numpy.random.choice(alg_list,p=prop) else: raise Exception('algorithm_management_mode : '+algorithm_management_settings['mode']+' not implemented') alg_id = alg_label_to_alg_id[ prop_item['alg_label'] ] alg_uid = alg_label_to_alg_uid[ prop_item['alg_label'] ] alg_label = prop_item['alg_label'] if (first_participant_query) and (participant_to_algorithm_management=='one_to_one'): db.set(app_id+':participants',participant_uid,'alg_id',alg_id) db.set(app_id+':participants',participant_uid,'alg_uid',alg_uid) elif (participant_to_algorithm_management=='one_to_one'): # If here, then alg_uid should already be assigned in participant doc alg_id,didSucceed,message = db.get(app_id+':participants',participant_uid,'alg_id') alg_uid,didSucceed,message = db.get(app_id+':participants',participant_uid,'alg_uid') else: raise Exception('participant_to_algorithm_management : '+participant_to_algorithm_management+' not implemented') # get sandboxed database for the specific app_id,alg_id,exp_uid - closing off the rest of the database to the algorithm rc = ResourceClient(app_id,exp_uid,alg_uid,db) # get specific algorithm to make calls to alg = utils.get_app_alg(self.app_id,alg_id) # call getQuery index_center,index_left,index_right,dt = utils.timeit(alg.getQuery)(resource=rc) log_entry_durations = { 'exp_uid':exp_uid,'alg_uid':alg_uid,'task':'getQuery','duration':dt } log_entry_durations.update( rc.getDurations() ) meta = {'log_entry_durations':log_entry_durations} # create JSON query payload timestamp = str(utils.datetimeNow()) query_uid = utils.getNewUID() query = {} query['query_uid'] = query_uid query['target_indices'] = [ {'index':index_center,'label':'center','flag':0},{'index':index_left,'label':'left','flag':0},{'index':index_right,'label':'right','flag':0} ] # save query data to database query_doc = {} query_doc.update(query) query_doc['participant_uid'] = participant_uid query_doc['alg_uid'] = alg_uid query_doc['exp_uid'] = exp_uid query_doc['alg_label'] = alg_label query_doc['timestamp_query_generated'] = timestamp for field in query_doc: db.set(app_id+':queries',query_uid,field,query_doc[field]) args_out = {'args':query,'meta':meta} response_json = json.dumps(args_out) log_entry = { 'exp_uid':exp_uid,'task':'getQuery','json':response_json,'timestamp':utils.datetimeNow() } ell.log( app_id+':APP-RESPONSE', log_entry ) return response_json,True,'' except Exception, err: error = traceback.format_exc() log_entry = { 'exp_uid':exp_uid,'task':'getQuery','error':error,'timestamp':utils.datetimeNow() } ell.log( app_id+':APP-EXCEPTION', log_entry ) return '{}',False,error