コード例 #1
0
ファイル: tasks.py プロジェクト: dconathan/NEXT
def apply_dashboard(app_id, exp_uid, args_in_json, enqueue_timestamp):
	enqueue_datetime = next.utils.str2datetime(enqueue_timestamp)
	dequeue_datetime = next.utils.datetimeNow()
	delta_datetime = dequeue_datetime - enqueue_datetime
	time_enqueued = delta_datetime.seconds + delta_datetime.microseconds/1000000.
        dir, _ = os.path.split(__file__)
        reference_dict,errs = verifier.load_doc('{}/myApp.yaml'.format(app_id, app_id),"apps/")
        if len(errs) > 0:
                raise Exception("App YAML format errors: \n{}".format(str(errs)))
        args_dict = verifier.verify(args_in_json, reference_dict['getStats']['args'])
        stat_id = args_dict['args'].get('stat_id','none')

        stat_args = args_dict['args']

        hash_object = hashlib.md5(stat_id+'_'+json.dumps(stat_args['params']))
        stat_uid = hash_object.hexdigest()
        stat_uid += '_' + exp_uid

        app = App_Wrapper(app_id, exp_uid, db, ell)
        cached_doc = app.butler.dashboard.get(uid=stat_uid)
        cached_response = None
        if (int(stat_args.get('force_recompute',0))==0) and (cached_doc is not None):    
          delta_datetime = (next.utils.datetimeNow() - next.utils.str2datetime(cached_doc['timestamp']))
          if delta_datetime.seconds < next.constants.DASHBOARD_STALENESS_IN_SECONDS:
            cached_response = json.loads(cached_doc['data_dict'])
            if 'meta' not in cached_response:
              cached_response['meta']={}
            cached_response['meta']['cached'] = 1
            if delta_datetime.seconds/60<1:
                cached_response['meta']['last_dashboard_update'] = '<1 minute ago'
            else:
                cached_response['meta']['last_dashboard_update'] = str(delta_datetime.seconds/60)+' minutes ago'

        if cached_response==None:
            dashboard_string = 'apps.' + app_id + '.dashboard.Dashboard'
            dashboard_module = __import__(dashboard_string, fromlist=[''])
            dashboard = getattr(dashboard_module, 'MyAppDashboard')
            dashboard = dashboard(db, ell)
            stats_method = getattr(dashboard, stat_id)
            response,dt = next.utils.timeit(stats_method)(app,app.butler,**args_dict['args']['params'])
            
            save_dict = {'exp_uid':app.exp_uid,
                    'stat_uid':stat_uid,
                    'timestamp':next.utils.datetime2str(next.utils.datetimeNow()),
                    'data_dict':json.dumps(response)}
            app.butler.dashboard.set_many(uid=stat_uid,key_value_dict=save_dict)

            # update the admin timing with the timing of a getModel
            if hasattr(app, 'log_entry_durations'):
                app.log_entry_durations['app_duration'] = dt
                app.log_entry_durations['duration_enqueued'] = time_enqueued
                app.butler.ell.log(app.app_id+':ALG-DURATION', app.log_entry_durations)
        else:
            response = cached_response

        if DEBUG_ON:
            next.utils.debug_print('#### Finished Dashboard %s, time_enqueued=%s,  execution_time=%s ####' % (stat_id, time_enqueued, dt), color='white')
	return json.dumps(response), True, ''
コード例 #2
0
ファイル: App.py プロジェクト: widoptimization-willett/NEXT
 def __init__(self, app_id, exp_uid, db, ell):
     self.app_id = app_id
     self.exp_uid = exp_uid
     self.helper = Helper()
     self.myApp = __import__('apps.'+self.app_id, fromlist=[''])
     self.myApp = getattr(self.myApp, 'MyApp')
     self.myApp = self.myApp(db)
     self.butler = Butler(self.app_id, self.exp_uid, self.myApp.TargetManager, db, ell)
     base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),"../../apps"))
     self.reference_dict, app_errs = verifier.load_doc("{}/myApp.yaml".format(app_id), base_dir)
     self.algs_reference_dict,alg_errs = verifier.load_doc("{}/algs/Algs.yaml".format(app_id, app_id), base_dir)
     if len(app_errs) > 0 or len(alg_errs) > 0:
         raise Exception("App YAML formatting errors: \n{}\n\nAlg YAML formatting errors: \n{}".format(
             str(app_errs),
             str(alg_errs)
         ))
     dashboard_string = 'apps.' + self.app_id + \
                        '.dashboard.Dashboard'
     dashboard_module = __import__(dashboard_string, fromlist=[''])
     self.dashboard = getattr(dashboard_module, 'MyAppDashboard')
コード例 #3
0
def init_form(app_id=None):
    if app_id:
        filename = '{0}/{0}.yaml'.format(app_id)

        api,_ = verifier.load_doc(filename, 'apps/')
        return render_template('form.html',api_doc=api, submit="/api/experiment", function_name="initExp", base_dir="/assistant/static")
    
    message = ('Welcome to the next.discovery system.\n '
               'Available apps {}'.format(', '.join(utils.get_supported_apps())))

    return render_template('raw.html',doc=message)
コード例 #4
0
ファイル: App.py プロジェクト: nextml/NEXT
 def __init__(self, app_id, exp_uid, db, ell):
     self.app_id = app_id
     self.exp_uid = exp_uid
     self.helper = Helper()
     self.myApp = __import__('apps.'+self.app_id, fromlist=[''])
     self.myApp = getattr(self.myApp, 'MyApp')
     self.myApp = self.myApp(db)
     self.butler = Butler(self.app_id, self.exp_uid, self.myApp.TargetManager, db, ell)
     base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),"../../apps"))
     self.reference_dict, app_errs = verifier.load_doc("{}/myApp.yaml".format(app_id), base_dir)
     self.algs_reference_dict,alg_errs = verifier.load_doc("{}/algs/Algs.yaml".format(app_id, app_id), base_dir)
     if len(app_errs) > 0 or len(alg_errs) > 0:
         raise Exception("App YAML formatting errors: \n{}\n\nAlg YAML formatting errors: \n{}".format(
             str(app_errs),
             str(alg_errs)
         ))
     dashboard_string = 'apps.' + self.app_id + \
                        '.dashboard.Dashboard'
     dashboard_module = __import__(dashboard_string, fromlist=[''])
     self.dashboard = getattr(dashboard_module, 'MyAppDashboard')
コード例 #5
0
ファイル: tasks.py プロジェクト: abiswas3/NEXT
def apply_dashboard(app_id, exp_uid, args_in_json, enqueue_timestamp):
    enqueue_datetime = next.utils.str2datetime(enqueue_timestamp)
    dequeue_datetime = next.utils.datetimeNow()
    delta_datetime = dequeue_datetime - enqueue_datetime
    time_enqueued = delta_datetime.seconds + delta_datetime.microseconds / 1000000.
    dir, _ = os.path.split(__file__)
    reference_dict, errs = verifier.load_doc(
        '{}/{}.yaml'.format(app_id, app_id), "apps/")
    if len(errs) > 0:
        raise Exception("App YAML format errors: \n{}".format(str(errs)))
    args_dict = verifier.verify(args_in_json,
                                reference_dict['getStats']['args'])
    stat_id = args_dict['args'].get('stat_id', 'none')

    stat_args = args_dict['args']
    hash_object = hashlib.md5(stat_id + '_' + json.dumps(stat_args['params']))
    stat_uid = hash_object.hexdigest()

    app = App_Wrapper(app_id, exp_uid, db, ell)
    cached_doc = app.butler.dashboard.get(uid=stat_uid)
    cached_response = None
    if (int(stat_args.get('force_recompute', 0)) == 0) and (cached_doc
                                                            is not None):
        delta_datetime = (next.utils.datetimeNow() -
                          next.utils.str2datetime(cached_doc['timestamp']))
        if delta_datetime.seconds < next.constants.DASHBOARD_STALENESS_IN_SECONDS:
            cached_response = json.loads(cached_doc['data_dict'])
            if 'meta' not in cached_response:
                cached_response['meta'] = {}
            cached_response['meta']['cached'] = 1
            if delta_datetime.seconds / 60 < 1:
                cached_response['meta'][
                    'last_dashboard_update'] = '<1 minute ago'
            else:
                cached_response['meta']['last_dashboard_update'] = str(
                    delta_datetime.seconds / 60) + ' minutes ago'

    if cached_response == None:
        dashboard_string = 'apps.' + app_id + '.dashboard.Dashboard'
        dashboard_module = __import__(dashboard_string, fromlist=[''])
        dashboard = getattr(dashboard_module, app_id + 'Dashboard')
        dashboard = dashboard(db, ell)
        stats_method = getattr(dashboard, stat_id)
        response, dt = next.utils.timeit(stats_method)(
            app, app.butler, **args_dict['args']['params'])

        save_dict = {
            'exp_uid': app.exp_uid,
            'stat_uid': stat_uid,
            'timestamp': next.utils.datetime2str(next.utils.datetimeNow()),
            'data_dict': json.dumps(response)
        }
        app.butler.dashboard.set_many(uid=stat_uid, key_value_dict=save_dict)

        # update the admin timing with the timing of a getModel
        if hasattr(app, 'log_entry_durations'):
            app.log_entry_durations['app_duration'] = dt
            app.log_entry_durations['duration_enqueued'] = time_enqueued
            app.butler.ell.log(app.app_id + ':ALG-DURATION',
                               app.log_entry_durations)
    else:
        response = cached_response

    if DEBUG_ON:
        next.utils.debug_print(
            '#### Finished Dashboard %s, time_enqueued=%s,  execution_time=%s ####'
            % (stat_id, time_enqueued, dt),
            color='white')
    return json.dumps(response), True, ''