def init_alg(self, exp_uid, algorithm, alg_args): butler = Butler(self.app_id, exp_uid, self.myApp.TargetManager, self.butler.db, self.butler.ell, algorithm['alg_label'], algorithm['alg_id']) alg = utils.get_app_alg(self.app_id, algorithm['alg_id']) if 'args' in self.algs_reference_dict['initExp']: alg_args = verifier.verify( alg_args, self.algs_reference_dict['initExp']['args']) # I got rid of a timeit function here; it wasn't handling the # argument unpacking correctly? --Scott, 2016-3-7 # TODO: put dt back in and change log_entry to relfect that alg_response = alg.initExp(butler, **alg_args) alg_response = verifier.verify( {'rets': alg_response}, {'rets': self.algs_reference_dict['initExp']['rets']}) log_entry = { 'exp_uid': exp_uid, 'alg_label': algorithm['alg_label'], 'task': 'initExp', 'duration': -1, 'timestamp': utils.datetimeNow() } self.butler.log('ALG-DURATION', log_entry)
def call_app_fn(self, alg_label, alg_id, func_name, args): butler = Butler(self.app_id, self.exp_uid, self.myApp.TargetManager, self.butler.db, self.butler.ell, alg_label, alg_id) alg = utils.get_app_alg(self.app_id, alg_id) def alg_wrapper(alg_args={}): return self.run_alg(butler, alg_label, alg, func_name, alg_args) return getattr(self.myApp, func_name)(self.butler, alg_wrapper, args['args'])
def apply_sync_by_namespace(app_id, exp_uid, alg_id, alg_label, task_name, args, namespace, job_uid, enqueue_timestamp, time_limit): enqueue_datetime = next.utils.str2datetime(enqueue_timestamp) dequeue_datetime = next.utils.datetimeNow() delta_datetime = dequeue_datetime - enqueue_datetime time_enqueued = delta_datetime.seconds + delta_datetime.microseconds / 1000000. try: print '>>>>>>>> Starting namespace:%s, job_uid=%s, time_enqueued=%s <<<<<<<<<' % ( namespace, job_uid, time_enqueued) # get stateless app next_app = next.utils.get_app(app_id, exp_uid, db, ell) target_manager = next_app.myApp.TargetManager next_alg = next.utils.get_app_alg(app_id, alg_id) butler = Butler(app_id, exp_uid, target_manager, db, ell, alg_label, alg_id) response, dt = next.utils.timeit(getattr(next_alg, task_name))(butler, args) log_entry_durations = { 'exp_uid': exp_uid, 'alg_label': alg_label, 'task': 'daemonProcess', 'duration': dt } log_entry_durations.update(butler.algorithms.getDurations()) log_entry_durations['app_duration'] = dt log_entry_durations['duration_enqueued'] = time_enqueued log_entry_durations['timestamp'] = next.utils.datetimeNow() ell.log(app_id + ':ALG-DURATION', log_entry_durations) print '########## Finished namespace:%s, job_uid=%s, time_enqueued=%s, execution_time=%s ##########' % ( namespace, job_uid, time_enqueued, dt) return except Exception, error: exc_type, exc_value, exc_traceback = sys.exc_info() print "tasks Exception: {} {}".format(error, traceback.format_exc()) traceback.print_tb(exc_traceback) # error = traceback.format_exc() # log_entry = { 'exp_uid':exp_uid,'task':'daemonProcess','error':error,'timestamp':next.utils.datetimeNow() } # ell.log( app_id+':APP-EXCEPTION', log_entry ) return None
def __init__(self, app_id, exp_uid, db, ell): self.app_id = app_id self.exp_uid = exp_uid self.helper = Helper() self.myApp = __import__('apps.'+self.app_id, fromlist=['']) self.myApp = getattr(self.myApp, 'MyApp') self.myApp = self.myApp(db) self.butler = Butler(self.app_id, self.exp_uid, self.myApp.TargetManager, db, ell) base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),"../../apps")) self.reference_dict, app_errs = verifier.load_doc("{}/myApp.yaml".format(app_id), base_dir) self.algs_reference_dict,alg_errs = verifier.load_doc("{}/algs/Algs.yaml".format(app_id, app_id), base_dir) if len(app_errs) > 0 or len(alg_errs) > 0: raise Exception("App YAML formatting errors: \n{}\n\nAlg YAML formatting errors: \n{}".format( str(app_errs), str(alg_errs) )) dashboard_string = 'apps.' + self.app_id + \ '.dashboard.Dashboard' dashboard_module = __import__(dashboard_string, fromlist=['']) self.dashboard = getattr(dashboard_module, 'MyAppDashboard')
def __init__(self, app_id, exp_uid, db, ell): self.app_id = app_id self.exp_uid = exp_uid self.next_app = next.utils.get_app(app_id, exp_uid, db, ell) self.butler = Butler(app_id, exp_uid, self.next_app.myApp.TargetManager, db, ell)