def init_alg(self, exp_uid, algorithm, alg_args):
butler = Butler(self.app_id, exp_uid, self.myApp.TargetManager,
self.butler.db, self.butler.ell,
algorithm['alg_label'], algorithm['alg_id'])
alg = utils.get_app_alg(self.app_id, algorithm['alg_id'])
if 'args' in self.algs_reference_dict['initExp']:
alg_args = verifier.verify(
alg_args, self.algs_reference_dict['initExp']['args'])
# I got rid of a timeit function here; it wasn't handling the
# argument unpacking correctly? --Scott, 2016-3-7
# TODO: put dt back in and change log_entry to relfect that
alg_response = alg.initExp(butler, **alg_args)
alg_response = verifier.verify(
{'rets': alg_response},
{'rets': self.algs_reference_dict['initExp']['rets']})
log_entry = {
'exp_uid': exp_uid,
'alg_label': algorithm['alg_label'],
'task': 'initExp',
'duration': -1,
'timestamp': utils.datetimeNow()
}
self.butler.log('ALG-DURATION', log_entry)
def call_app_fn(self, alg_label, alg_id, func_name, args):
butler = Butler(self.app_id, self.exp_uid, self.myApp.TargetManager,
self.butler.db, self.butler.ell, alg_label, alg_id)
alg = utils.get_app_alg(self.app_id, alg_id)
def alg_wrapper(alg_args={}):
return self.run_alg(butler, alg_label, alg, func_name, alg_args)
return getattr(self.myApp, func_name)(self.butler, alg_wrapper,
args['args'])
def apply_sync_by_namespace(app_id, exp_uid, alg_id, alg_label, task_name,
args, namespace, job_uid, enqueue_timestamp,
time_limit):
enqueue_datetime = next.utils.str2datetime(enqueue_timestamp)
dequeue_datetime = next.utils.datetimeNow()
delta_datetime = dequeue_datetime - enqueue_datetime
time_enqueued = delta_datetime.seconds + delta_datetime.microseconds / 1000000.
try:
print '>>>>>>>> Starting namespace:%s, job_uid=%s, time_enqueued=%s <<<<<<<<<' % (
namespace, job_uid, time_enqueued)
# get stateless app
next_app = next.utils.get_app(app_id, exp_uid, db, ell)
target_manager = next_app.myApp.TargetManager
next_alg = next.utils.get_app_alg(app_id, alg_id)
butler = Butler(app_id, exp_uid, target_manager, db, ell, alg_label,
alg_id)
response, dt = next.utils.timeit(getattr(next_alg, task_name))(butler,
args)
log_entry_durations = {
'exp_uid': exp_uid,
'alg_label': alg_label,
'task': 'daemonProcess',
'duration': dt
}
log_entry_durations.update(butler.algorithms.getDurations())
log_entry_durations['app_duration'] = dt
log_entry_durations['duration_enqueued'] = time_enqueued
log_entry_durations['timestamp'] = next.utils.datetimeNow()
ell.log(app_id + ':ALG-DURATION', log_entry_durations)
print '########## Finished namespace:%s, job_uid=%s, time_enqueued=%s, execution_time=%s ##########' % (
namespace, job_uid, time_enqueued, dt)
return
except Exception, error:
exc_type, exc_value, exc_traceback = sys.exc_info()
print "tasks Exception: {} {}".format(error, traceback.format_exc())
traceback.print_tb(exc_traceback)
# error = traceback.format_exc()
# log_entry = { 'exp_uid':exp_uid,'task':'daemonProcess','error':error,'timestamp':next.utils.datetimeNow() }
# ell.log( app_id+':APP-EXCEPTION', log_entry )
return None
def __init__(self, app_id, exp_uid, db, ell):
self.app_id = app_id
self.exp_uid = exp_uid
self.helper = Helper()
self.myApp = __import__('apps.'+self.app_id, fromlist=[''])
self.myApp = getattr(self.myApp, 'MyApp')
self.myApp = self.myApp(db)
self.butler = Butler(self.app_id, self.exp_uid, self.myApp.TargetManager, db, ell)
base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),"../../apps"))
self.reference_dict, app_errs = verifier.load_doc("{}/myApp.yaml".format(app_id), base_dir)
self.algs_reference_dict,alg_errs = verifier.load_doc("{}/algs/Algs.yaml".format(app_id, app_id), base_dir)
if len(app_errs) > 0 or len(alg_errs) > 0:
raise Exception("App YAML formatting errors: \n{}\n\nAlg YAML formatting errors: \n{}".format(
str(app_errs),
str(alg_errs)
))
dashboard_string = 'apps.' + self.app_id + \
'.dashboard.Dashboard'
dashboard_module = __import__(dashboard_string, fromlist=[''])
self.dashboard = getattr(dashboard_module, 'MyAppDashboard')
def __init__(self, app_id, exp_uid, db, ell):
self.app_id = app_id
self.exp_uid = exp_uid
self.next_app = next.utils.get_app(app_id, exp_uid, db, ell)
self.butler = Butler(app_id, exp_uid,
self.next_app.myApp.TargetManager, db, ell)