def init_alg(self, exp_uid, algorithm, alg_args):
butler = Butler(self.app_id, exp_uid, self.myApp.TargetManager,
self.butler.db, self.butler.ell,
algorithm['alg_label'], algorithm['alg_id'])
alg = utils.get_app_alg(self.app_id, algorithm['alg_id'])
if 'args' in self.algs_reference_dict['initExp']:
alg_args = verifier.verify(
alg_args, self.algs_reference_dict['initExp']['args'])
# I got rid of a timeit function here; it wasn't handling the
# argument unpacking correctly? --Scott, 2016-3-7
# TODO: put dt back in and change log_entry to relfect that
alg_response = alg.initExp(butler, **alg_args)
alg_response = verifier.verify(
{'rets': alg_response},
{'rets': self.algs_reference_dict['initExp']['rets']})
log_entry = {
'exp_uid': exp_uid,
'alg_label': algorithm['alg_label'],
'task': 'initExp',
'duration': -1,
'timestamp': utils.datetimeNow()
}
self.butler.log('ALG-DURATION', log_entry)
def call_app_fn(self, alg_label, alg_id, func_name, args):
butler = Butler(self.app_id, self.exp_uid, self.myApp.TargetManager,
self.butler.db, self.butler.ell, alg_label, alg_id)
alg = utils.get_app_alg(self.app_id, alg_id)
def alg_wrapper(alg_args={}):
return self.run_alg(butler, alg_label, alg, func_name, alg_args)
return getattr(self.myApp, func_name)(self.butler, alg_wrapper,
args['args'])
def __init__(self, app_id, exp_uid, db, ell):
self.app_id = app_id
self.exp_uid = exp_uid
self.helper = Helper()
self.myApp = __import__('apps.'+self.app_id, fromlist=[''])
self.myApp = getattr(self.myApp, 'MyApp')
self.myApp = self.myApp(db)
self.butler = Butler(self.app_id, self.exp_uid, self.myApp.TargetManager, db, ell)
base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),"../../apps"))
self.reference_dict, app_errs = verifier.load_doc("{}/myApp.yaml".format(app_id), base_dir)
self.algs_reference_dict,alg_errs = verifier.load_doc("{}/algs/Algs.yaml".format(app_id, app_id), base_dir)
if len(app_errs) > 0 or len(alg_errs) > 0:
raise Exception("App YAML formatting errors: \n{}\n\nAlg YAML formatting errors: \n{}".format(
str(app_errs),
str(alg_errs)
))
dashboard_string = 'apps.' + self.app_id + \
'.dashboard.Dashboard'
dashboard_module = __import__(dashboard_string, fromlist=[''])
self.dashboard = getattr(dashboard_module, 'MyAppDashboard')
def apply_sync_by_namespace(app_id, exp_uid, alg_id, alg_label, task_name,
args, namespace, job_uid, enqueue_timestamp,
time_limit):
enqueue_datetime = next.utils.str2datetime(enqueue_timestamp)
dequeue_datetime = next.utils.datetimeNow()
delta_datetime = dequeue_datetime - enqueue_datetime
time_enqueued = delta_datetime.seconds + delta_datetime.microseconds / 1000000.
try:
print '>>>>>>>> Starting namespace:%s, job_uid=%s, time_enqueued=%s <<<<<<<<<' % (
namespace, job_uid, time_enqueued)
# get stateless app
next_app = next.utils.get_app(app_id, exp_uid, db, ell)
target_manager = next_app.myApp.TargetManager
next_alg = next.utils.get_app_alg(app_id, alg_id)
butler = Butler(app_id, exp_uid, target_manager, db, ell, alg_label,
alg_id)
response, dt = next.utils.timeit(getattr(next_alg, task_name))(butler,
args)
log_entry_durations = {
'exp_uid': exp_uid,
'alg_label': alg_label,
'task': 'daemonProcess',
'duration': dt
}
log_entry_durations.update(butler.algorithms.getDurations())
log_entry_durations['app_duration'] = dt
log_entry_durations['duration_enqueued'] = time_enqueued
log_entry_durations['timestamp'] = next.utils.datetimeNow()
ell.log(app_id + ':ALG-DURATION', log_entry_durations)
print '########## Finished namespace:%s, job_uid=%s, time_enqueued=%s, execution_time=%s ##########' % (
namespace, job_uid, time_enqueued, dt)
return
except Exception, error:
exc_type, exc_value, exc_traceback = sys.exc_info()
print "tasks Exception: {} {}".format(error, traceback.format_exc())
traceback.print_tb(exc_traceback)
# error = traceback.format_exc()
# log_entry = { 'exp_uid':exp_uid,'task':'daemonProcess','error':error,'timestamp':next.utils.datetimeNow() }
# ell.log( app_id+':APP-EXCEPTION', log_entry )
return None
def __init__(self, app_id, exp_uid, db, ell):
self.app_id = app_id
self.exp_uid = exp_uid
self.helper = Helper()
self.myApp = __import__('apps.'+self.app_id, fromlist=[''])
self.myApp = getattr(self.myApp, 'MyApp')
self.myApp = self.myApp(db)
self.butler = Butler(self.app_id, self.exp_uid, self.myApp.TargetManager, db, ell)
base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),"../../apps"))
self.reference_dict, app_errs = verifier.load_doc("{}/myApp.yaml".format(app_id), base_dir)
self.algs_reference_dict,alg_errs = verifier.load_doc("{}/algs/Algs.yaml".format(app_id, app_id), base_dir)
if len(app_errs) > 0 or len(alg_errs) > 0:
raise Exception("App YAML formatting errors: \n{}\n\nAlg YAML formatting errors: \n{}".format(
str(app_errs),
str(alg_errs)
))
dashboard_string = 'apps.' + self.app_id + \
'.dashboard.Dashboard'
dashboard_module = __import__(dashboard_string, fromlist=[''])
self.dashboard = getattr(dashboard_module, 'MyAppDashboard')
class App(object):
def __init__(self, app_id, exp_uid, db, ell):
self.app_id = app_id
self.exp_uid = exp_uid
self.helper = Helper()
self.myApp = __import__('apps.'+self.app_id, fromlist=[''])
self.myApp = getattr(self.myApp, 'MyApp')
self.myApp = self.myApp(db)
self.butler = Butler(self.app_id, self.exp_uid, self.myApp.TargetManager, db, ell)
base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),"../../apps"))
self.reference_dict, app_errs = verifier.load_doc("{}/myApp.yaml".format(app_id), base_dir)
self.algs_reference_dict,alg_errs = verifier.load_doc("{}/algs/Algs.yaml".format(app_id, app_id), base_dir)
if len(app_errs) > 0 or len(alg_errs) > 0:
raise Exception("App YAML formatting errors: \n{}\n\nAlg YAML formatting errors: \n{}".format(
str(app_errs),
str(alg_errs)
))
dashboard_string = 'apps.' + self.app_id + \
'.dashboard.Dashboard'
dashboard_module = __import__(dashboard_string, fromlist=[''])
self.dashboard = getattr(dashboard_module, 'MyAppDashboard')
def run_alg(self, butler, alg_label, alg, func_name, alg_args):
if 'args' in self.algs_reference_dict[func_name]:
alg_args = verifier.verify(alg_args, self.algs_reference_dict[func_name]['args'])
alg_response, dt = utils.timeit(getattr(alg, func_name))(butler, **alg_args)
alg_response = verifier.verify({'rets':alg_response},
{'rets':self.algs_reference_dict[func_name]['rets']})
log_entry_durations = {'exp_uid':self.exp_uid,
'alg_label':alg_label,
'task':func_name,
'duration':dt}
log_entry_durations.update(butler.algorithms.getDurations())
self.log_entry_durations = log_entry_durations
return alg_response['rets']
def call_app_fn(self, alg_label, alg_id, func_name, args):
butler = Butler(self.app_id, self.exp_uid, self.myApp.TargetManager, self.butler.db, self.butler.ell, alg_label, alg_id)
alg = utils.get_app_alg(self.app_id, alg_id)
def alg_wrapper(alg_args={}):
return self.run_alg(butler, alg_label, alg, func_name, alg_args)
return getattr(self.myApp, func_name)(self.butler, alg_wrapper, args['args'])
def init_alg(self, exp_uid, algorithm, alg_args):
butler = Butler(self.app_id, exp_uid, self.myApp.TargetManager, self.butler.db, self.butler.ell, algorithm['alg_label'], algorithm['alg_id'])
alg = utils.get_app_alg(self.app_id, algorithm['alg_id'])
if 'args' in self.algs_reference_dict['initExp']:
alg_args = verifier.verify(alg_args, self.algs_reference_dict['initExp']['args'])
# I got rid of a timeit function here; it wasn't handling the
# argument unpacking correctly? --Scott, 2016-3-7
# TODO: put dt back in and change log_entry to relfect that
alg_response = alg.initExp(butler, **alg_args)
alg_response = verifier.verify({'rets':alg_response}, {'rets':self.algs_reference_dict['initExp']['rets']})
log_entry = {'exp_uid':exp_uid, 'alg_label':algorithm['alg_label'], 'task':'initExp', 'duration':-1, 'timestamp':utils.datetimeNow()}
self.butler.log('ALG-DURATION', log_entry)
def init_app(self, exp_uid, alg_list, args):
utils.debug_print(str(args))
def init_algs_wrapper(alg_args={}):
for algorithm in alg_list:
# Set doc in algorithms bucket. These objects are used by the algorithms to store data.
algorithm['exp_uid'] = exp_uid
self.butler.algorithms.set(uid=algorithm['alg_label'], value=algorithm)
self.init_alg(exp_uid, algorithm, alg_args)
# params = algorithm.get('params',None)
return self.myApp.initExp(self.butler, init_algs_wrapper, args)
def initExp(self, exp_uid, args_json):
try:
self.helper.ensure_indices(self.app_id,self.butler.db, self.butler.ell)
args_dict = self.helper.convert_json(args_json)
args_dict = verifier.verify(args_dict, self.reference_dict['initExp']['args'])
args_dict['exp_uid'] = exp_uid # to get doc from db
args_dict['start_date'] = utils.datetime2str(utils.datetimeNow())
self.butler.admin.set(uid=exp_uid,value={'exp_uid': exp_uid, 'app_id':self.app_id, 'start_date':str(utils.datetimeNow())})
utils.debug_print("ASD "+str(args_dict))
args_dict['args'] = self.init_app(exp_uid, args_dict['args']['alg_list'], args_dict['args'])
args_dict['git_hash'] = git_hash
self.butler.experiment.set(value=args_dict)
return '{}', True, ''
except Exception, error:
exc_type, exc_value, exc_traceback = sys.exc_info()
full_error = str(traceback.format_exc())+'\n'+str(error)
utils.debug_print("initExp Exception: " + full_error, color='red')
log_entry = { 'exp_uid':exp_uid,'task':'initExp','error':full_error,'timestamp':utils.datetimeNow(),'args_json':args_json }
self.butler.ell.log( self.app_id+':APP-EXCEPTION', log_entry )
traceback.print_tb(exc_traceback)
return '{}', False, str(error)
class App(object):
def __init__(self, app_id, exp_uid, db, ell):
self.app_id = app_id
self.exp_uid = exp_uid
self.helper = Helper()
self.myApp = __import__('apps.'+self.app_id, fromlist=[''])
self.myApp = getattr(self.myApp, 'MyApp')
self.myApp = self.myApp(db)
self.butler = Butler(self.app_id, self.exp_uid, self.myApp.TargetManager, db, ell)
base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),"../../apps"))
self.reference_dict, app_errs = verifier.load_doc("{}/myApp.yaml".format(app_id), base_dir)
self.algs_reference_dict,alg_errs = verifier.load_doc("{}/algs/Algs.yaml".format(app_id, app_id), base_dir)
if len(app_errs) > 0 or len(alg_errs) > 0:
raise Exception("App YAML formatting errors: \n{}\n\nAlg YAML formatting errors: \n{}".format(
str(app_errs),
str(alg_errs)
))
dashboard_string = 'apps.' + self.app_id + \
'.dashboard.Dashboard'
dashboard_module = __import__(dashboard_string, fromlist=[''])
self.dashboard = getattr(dashboard_module, 'MyAppDashboard')
def run_alg(self, butler, alg_label, alg, func_name, alg_args):
if 'args' in self.algs_reference_dict[func_name]:
alg_args = verifier.verify(alg_args, self.algs_reference_dict[func_name]['args'])
alg_response, dt = utils.timeit(getattr(alg, func_name))(butler, **alg_args)
alg_response = verifier.verify({'rets':alg_response},
{'rets':self.algs_reference_dict[func_name]['rets']})
log_entry_durations = {'exp_uid':self.exp_uid,
'alg_label':alg_label,
'task':func_name,
'duration':dt}
log_entry_durations.update(butler.algorithms.getDurations())
self.log_entry_durations = log_entry_durations
return alg_response['rets']
def call_app_fn(self, alg_label, alg_id, func_name, args):
butler = Butler(self.app_id, self.exp_uid, self.myApp.TargetManager, self.butler.db, self.butler.ell, alg_label, alg_id)
alg = utils.get_app_alg(self.app_id, alg_id)
def alg_wrapper(alg_args={}):
return self.run_alg(butler, alg_label, alg, func_name, alg_args)
return getattr(self.myApp, func_name)(self.butler, alg_wrapper, args['args'])
def init_alg(self, exp_uid, algorithm, alg_args):
butler = Butler(self.app_id, exp_uid, self.myApp.TargetManager, self.butler.db, self.butler.ell, algorithm['alg_label'], algorithm['alg_id'])
alg = utils.get_app_alg(self.app_id, algorithm['alg_id'])
if 'args' in self.algs_reference_dict['initExp']:
alg_args = verifier.verify(alg_args, self.algs_reference_dict['initExp']['args'])
# I got rid of a timeit function here; it wasn't handling the
# argument unpacking correctly? --Scott, 2016-3-7
# TODO: put dt back in and change log_entry to relfect that
alg_response = alg.initExp(butler, **alg_args)
alg_response = verifier.verify({'rets':alg_response}, {'rets':self.algs_reference_dict['initExp']['rets']})
log_entry = {'exp_uid':exp_uid, 'alg_label':algorithm['alg_label'], 'task':'initExp', 'duration':-1, 'timestamp':utils.datetimeNow()}
self.butler.log('ALG-DURATION', log_entry)
def init_app(self, exp_uid, alg_list, args):
def init_algs_wrapper(alg_args={}):
for algorithm in alg_list:
# Set doc in algorithms bucket. These objects are used by the algorithms to store data.
algorithm['exp_uid'] = exp_uid
self.butler.algorithms.set(uid=algorithm['alg_label'], value=algorithm)
self.init_alg(exp_uid, algorithm, alg_args)
# params = algorithm.get('params',None)
return self.myApp.initExp(self.butler, init_algs_wrapper, args)
def initExp(self, exp_uid, args_json):
try:
self.helper.ensure_indices(self.app_id,self.butler.db, self.butler.ell)
args_dict = self.helper.convert_json(args_json)
args_dict = verifier.verify(args_dict, self.reference_dict['initExp']['args'])
args_dict['exp_uid'] = exp_uid # to get doc from db
args_dict['start_date'] = utils.datetime2str(utils.datetimeNow())
self.butler.admin.set(uid=exp_uid,value={'exp_uid': exp_uid, 'app_id':self.app_id, 'start_date':str(utils.datetimeNow())})
self.butler.experiment.set(value={'exp_uid': exp_uid})
args_dict['args'] = self.init_app(exp_uid, args_dict['args']['alg_list'], args_dict['args'])
args_dict['git_hash'] = git_hash
self.butler.experiment.set_many(key_value_dict=args_dict)
return '{}', True, ''
except Exception, error:
exc_type, exc_value, exc_traceback = sys.exc_info()
full_error = str(traceback.format_exc())+'\n'+str(error)
utils.debug_print("initExp Exception: " + full_error, color='red')
log_entry = { 'exp_uid':exp_uid,'task':'initExp','error':full_error,'timestamp':utils.datetimeNow(),'args_json':args_json }
self.butler.ell.log( self.app_id+':APP-EXCEPTION', log_entry )
traceback.print_tb(exc_traceback)
return '{}', False, str(error)
def __init__(self, app_id, exp_uid, db, ell):
self.app_id = app_id
self.exp_uid = exp_uid
self.next_app = next.utils.get_app(app_id, exp_uid, db, ell)
self.butler = Butler(app_id, exp_uid,
self.next_app.myApp.TargetManager, db, ell)