def update_graph(selected_algo): df_pipe, df_ref = dataLoader('pipeline.csv', 'ref.csv') algo_df = df_pipe.copy() algo = selected_algo d1, d2 = hyperProfiler(algo, algo_df) plnc, plnn = hyperprofilercharts(d1, d2, algo) num_temp_divs = [] num_final_divs = [] cat_temp_divs = [] cat_final_divs = [] for i in plnn: # create subsets of 2 graphs num_temp_divs.append(i) if len(num_temp_divs) == 2: num_final_divs.append( html.Div([html.Br(), html.Div(num_temp_divs, className='row')])) num_temp_divs = [] for i in plnc: # create subsets of 2 graphs cat_temp_divs.append(i) if len(cat_temp_divs) == 2: cat_final_divs.append( html.Div([html.Br(), html.Div(cat_temp_divs, className='row')])) cat_temp_divs = [] return cat_final_divs, num_final_divs
def long_process(gen, X, y, metric, algo): global stat score = [] pipeline_name = [] if semaphore.is_locked(): raise Exception('Resource is locked') total_pipes = 0 for i in range(gen): semaphore.lock() stat['status'] = 'Started' po, po_score, pipes = runTPOT(X, y, metric, algo) createsklearnPipeline(po, total_pipes) total_pipes = total_pipes + pipes df_pipe, df_ref = dataLoader('pipeline.csv', 'ref.csv') pct_algo, pct_hyper = percentStatus(df_pipe, df_ref) highest_score, evalgo, evalpipes, evalhyper = kpi(df_pipe, algo) df_pipeline_score = df_pipe[['PIPELINE', 'SCORE']].copy().drop_duplicates() #df_pipeline_score = df_pipeline_score[df_pipeline_score['SCORE']>0.1] pipeline_name = pipeline_name + list(df_pipeline_score['PIPELINE']) score = score + list(df_pipeline_score['SCORE']) print(len(score)) # top 10 pipeline df_top_10 = df_pipeline_score[['PIPELINE', 'SCORE']].copy().drop_duplicates() if algo == 'Classifier': df_top_10 = df_top_10.sort_values(['SCORE'], ascending=False).nlargest( n, ['SCORE']) else: df_top_10 = df_top_10.sort_values(['SCORE'], ascending=False).nsmallest( n, ['SCORE']) # Update real-time dict stat['top_n_pipe'] = [ "P-" + str(e) for e in list(df_top_10['PIPELINE']) ] stat['top_n_score'] = list(df_top_10['SCORE']) stat['pipeline'] = pipeline_name stat['score'] = score stat['highest_score'] = highest_score stat['evalgo'] = evalgo stat['evalpipes'] = evalpipes stat['evalhyper'] = evalhyper stat['pct_algo'] = pct_algo stat['pct_hyper'] = pct_hyper stat['gen'] = i + 1 stat['time_start'] = datetime.datetime.now() semaphore.unlock() stat['time_end'] = datetime.datetime.now() #time.sleep(2) stat['status'] = 'Completed' return (stat['time_end'] - stat['time_start']).total_seconds() * 1e6, stat
def algo_hist_detail_display(status): num_temp_divs = [] num_final_divs = [] if status == 'Completed': df_pipe, df_ref = dataLoader('pipeline.csv', 'ref.csv') df_algo = df_pipe[df_pipe['PP_FLAG'] == 'N'] algo_hist_detail_div = algoHistDetails(df_algo) return html.Div(algo_hist_detail_div, className='row') else: return dcc.Loading( type='graph', children=html.Div( [html.H6("Generating Individual Algorithm Histograms....")]))
def update_pipeline(selectedData): df_pipe, df_ref = dataLoader('pipeline.csv', 'ref.csv') algo_df = df_pipe.copy() df_pipeline_table = df_pipe[[ 'PIPELINE', 'ALGO_NAME', 'HYPER_NAME', 'HYPER_VALUE' ]] if selectedData is None: raise PreventUpdate # get the values from df and return it if selectedData is not None: selectedPipeline = selectedData['points'][0]['y'] selectedPipeline = selectedPipeline[2:] pipe = int(float(selectedPipeline)) radio_o, radio_v = algoOptions( df_pipeline_table[df_pipeline_table['PIPELINE'] == pipe]) return radio_o, radio_v, hyperTable( df_pipeline_table[df_pipeline_table['PIPELINE'] == pipe])
import model import tensorflow as tf import config import util import time import numpy as np import os C = config.Config() model = model.Model() labels = util.labelGenerator() dataLoader = util.dataLoader(labels.label2id) compute_graph = tf.get_default_graph() X_test = np.load(os.path.join(C.numpy_serialized_path, C.x_test_serialized)) y_test = np.load(os.path.join(C.numpy_serialized_path, C.y_test_serialized)) batchLoader = util.batchGenerator(X_test,y_test,barchsize=1) batchIter = iter(batchLoader) #batchLoader is an endless iterator providing X,Y with compute_graph.as_default(): model.lstm_crf_predict() #Define graph model.load_word2vec() #Initialize all variables model.restore() trainHelper = util.trainHelper(model.word2vector) #Train helper do the padding for x,y in batchLoader: x_raw = x.copy() x,y,sequence_length,seq_max_len = trainHelper.generateBatch(x,y) viterbi_sequence = model.predict(x,sequence_length,seq_max_len) print(x_raw[0]) print(labels.ID2label(viterbi_sequence[0][0]))