def test_corpus(self): with open("../data/pt_BR/nnp") as f: nnp = [line.rstrip() for line in f.readlines()] with open("../data/pt_BR/terms") as f: terms = [line.rstrip() for line in f.readlines()] with open("../data/pt_BR/patterns") as f: patterns = [line.rstrip() for line in f.readlines()] data = LoadData(['../corpus/sel1.csv', '../corpus/sel2.csv']).load() p = PreProcessing(nnp, terms, patterns) tokens = [] for d in data.values(): tokens += p.clean_and_stem(d) bow, bow_features_names = p.build_bow(tokens) dist = np.sum(bow.toarray(), axis=0) tbow = {} for term, count in zip(bow_features_names, dist): tbow[term] = count import operator with open("bow", "w") as f: f.write(str(len(tbow))) f.write( str( sorted(tbow.items(), key=operator.itemgetter(1), reverse=True))) terms = p.compute_tfidf(data.values(), eliminate_zeros=True) with open("terms", "w") as f: f.write(str(terms))
def testall(directory, pred_file=None, label_file=None, out_path=None): folders = os.listdir(directory) networks = [] for folder in folders: if os.path.isfile(directory+folder+"/network.cfg") and os.path.exists(directory+folder+"/results"): networks.append(folder) config_file = directory+networks[0]+"/network.cfg" config = ConfigParser.ConfigParser() config.read(config_file) test_data = LoadData(directory = config.get('Testing Data', 'folders').split(','), data_file_name = config.get('Testing Data', 'data_file'), label_file_name = config.get('Testing Data', 'label_file'), seg_file_name = config.get('Testing Data', 'seg_file')) res = Analyzer(raw = test_data.get_data()[0], target = test_data.get_labels()[0]) for net in networks: config_file = directory+net+"/network.cfg" config = ConfigParser.ConfigParser() config.read(config_file) res.add_results(results_folder = config.get('General','directory'), name = net, prediction_file = config.get('Testing', 'prediction_file')+'_0', learning_curve_file = 'learning_curve') res.analyze(-1, pred_file=pred_file, label_file=label_file, out_path=out_path) return res
def index(): endpoints_reader = LoadData() data = Processor() df_csv, df_json = endpoints_reader.read_data() summary_csv_info, summary_json_info = data.process_endpoints(df_csv, df_json) return 'Service A result: ' + summary_csv_info + \ '\n' + 'Service B result: ' + summary_json_info
def init_with_data(self, is_load=True, max_outgoing=10, max_airports=500, silent=False): ld = LoadData(self, is_load, max_outgoing, max_airports, silent) ld.load() silent or self.print_info()
def start_rnn(): load_data = LoadData() process_data = ProcessDataRNN(sales=load_data.make_dataset( categorize=CATEGORIZE, start=START, upper=UPPER)) # xt, yt, xv, yv, xe, ye = process_data.run() xv, yv, xe, ye = process_data.run() if not ONLY_LOAD_MODEL: ckpt = ModelCheckpoint(OUTPUT_MODEL, monitor='val_loss', verbose=1, save_best_only=True, mode='min') reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=5, min_lr=MIN_LR, verbose=1) es = EarlyStopping(monitor='val_loss', patience=3) # net = make_model(len(process_data.FEATS)) net = make_model_rnn(process_data.Z.shape[2]) print(net.summary()) n_slices = LEN // 28 brks = np.array( [LEN - (n_slices - i) * 28 for i in range(n_slices + 1)]) brks = brks[brks >= max(LAGS) + 28] print("#" * 30) print(LEN, process_data.C.shape, process_data.Z.shape) print(brks) print(process_data.C.min(), process_data.ys.min(), process_data.Z[:, 66:].min()) print("#" * 30) net.fit_generator(DataGenerator( (process_data.C, process_data.Z, process_data.ys), brks[:-1], batch_size=BATCH_SIZE), epochs=EPOCH, validation_data=(xv, yv), callbacks=[ckpt, reduce_lr, es]) # net.fit(xt, yt, batch_size=BATCH_SIZE, epochs=EPOCH, validation_data=(xv, yv), callbacks=[ckpt, reduce_lr, es]) # nett = make_model(len(process_data.FEATS)) nett = make_model_rnn(process_data.Z.shape[2]) nett.load_weights(OUTPUT_MODEL) pv = nett.predict(xv, batch_size=BATCH_SIZE, verbose=1) pe = nett.predict(xe, batch_size=BATCH_SIZE, verbose=1) print("Eva result:", nett.evaluate(xv, yv, batch_size=BATCH_SIZE)) # pv = pv.reshape((-1, 28, 9)) # pe = pe.reshape((-1, 28, 9)) sv = process_data.sv.reshape((-1, 28)) se = process_data.se.reshape((-1, 28)) # Yv = yv.reshape((-1, 28)) return process_data, yv, pv, pe, sv, se
def test_load_data_full_csv(self): endpoints_reader = LoadData() json_link = '[{"date":"22-01-2021", "impressions":1376}, \ {"date":"21-01-2021","impressions":1906}, \ {"date":"20-01-2021","impressions":2818}, \ {"date":"19-01-2021","impressions":1024}]' csv_link = 'mock_unittest.csv' csv_file, _ = endpoints_reader.read_data(csv_link, json_link) self.assertEqual(csv_file.shape[0], 2)
def main(): # ------------------ config ------------------ # n_classes = 130 batch_size = 10 epochs = 25 save_model_path = './' # -------------------------------------------- # data_transforms = { 'train': transforms.Compose([ transforms.RandomResizedCrop(224), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) ]), 'valid': transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) ]), } dataset = LoadData(data_dir='./dataset/JsonData/', mode='train', transforms=data_transforms['train']) train_loaders = torch.utils.data.DataLoader(dataset, batch_size=batch_size, shuffle=True) dataset = LoadData(data_dir='./dataset/JsonData/', mode='valid', transforms=data_transforms['valid']) valid_loaders = torch.utils.data.DataLoader(dataset, batch_size=batch_size, shuffle=True) dataloaders = {'train': train_loaders, 'valid': valid_loaders} print('GPUs Available:', torch.cuda.is_available()) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") print("training on ", device) net = Net(n_classes=n_classes) criterion = nn.CrossEntropyLoss() optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9) loss = torch.nn.CrossEntropyLoss() train(net, criterion, optimizer, loss, device, dataloaders, epochs, save_model_path)
def main(): # ------------------ config ------------------ # n_classes = 130 batch_size = 2 epochs = 30 save_model_path = './' img_size = 448 # -------------------------------------------- # data_transforms = { 'train': transforms.Compose([ transforms.RandomResizedCrop(img_size), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) ]), 'valid': transforms.Compose([ transforms.Resize(img_size + 50), transforms.CenterCrop(img_size), transforms.ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) ]), } dataset = LoadData(data_dir='../dataset/JsonData/', mode='train', transforms=data_transforms['train']) train_loaders = torch.utils.data.DataLoader(dataset, batch_size=batch_size, shuffle=True) dataset = LoadData(data_dir='../dataset/JsonData/', mode='valid', transforms=data_transforms['valid']) valid_loaders = torch.utils.data.DataLoader(dataset, batch_size=batch_size, shuffle=True) dataloaders = {'train': train_loaders, 'valid': valid_loaders} print('GPUs Available:', torch.cuda.is_available()) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") print("training on ", device) net = Net(n_classes=n_classes, pretrained=False).to(device) optimizer = optim.SGD(net.parameters(), lr=0.08, momentum=0.9) scheduler = lr_scheduler.StepLR(optimizer, step_size=1000, gamma=0.1) loss = torch.nn.CrossEntropyLoss(size_average=False) train_model(net, optimizer, loss, scheduler, device, dataloaders, epochs, save_model_path)
def test_processor(self): endpoints_reader = LoadData() data = Processor() json_link = '[{"date":"22-01-2021", "impressions":1376}, \ {"date":"21-01-2021","impressions":1906}, \ {"date":"20-01-2021","impressions":2818}, \ {"date":"19-01-2021","impressions":1024}]' csv_link = 'mock_unittest.csv' df_csv, df_json = endpoints_reader.read_data(csv_link, json_link) summary_csv_info, summary_json_info = data.process_endpoints(df_csv, df_json) self.assertEqual(str(summary_csv_info), '{"mean": 1732.0, "sum": 3464}') self.assertEqual(str(summary_json_info), '{"mean": 1781.0, "sum": 7124}')
def test_load(self): """test load_data.py. Args: no """ load = LoadData('test_load_data') load.main() active = redis.StrictRedis(host="192.168.0.2", port=6379, db=6, password='******') # check up self.assertEqual(len(active.keys("*:*:2018-12-09 *")), 4, "total number check failed") self.assertEqual(len(active.keys("visit:*:2018-12-09 *")), 1, "visit incre data check failed") self.assertEqual(len(active.keys("search:*:2018-12-09 *")), 3, "search incre data check failed") self.assertEqual(active.zcard("visit:2018-12-09 00"), 1, "visit set num check failed") self.assertEqual(active.zcard("search:2018-12-09 01"), 3, "search set num check failed") self.assertEqual(active.get("stock:600796:jianpin"), "qjsh", "jianpin check failed") self.assertEqual(int(active.get("visit:000001:2018-12-09 00")), 33, " visit value check failed") index_visit_set = active.zrange("visit:2018-12-09 00", 0, -1).index("000001") self.assertEqual(active.zrange("visit:2018-12-09 00", index_visit_set, index_visit_set, withscores=True)[0][1], 33.0, "visit set value check failed") self.assertEqual(int(active.get("search:000007:2018-12-09 01")), 13, "search value check failed") self.assertEqual(int(active.get("search:000721:2018-12-09 01")), 13, "search value check failed") self.assertEqual(int(active.get("search:600796:2018-12-09 01")), 13, "search value check failed") search_set_keys = active.zrange("search:2018-12-09 01", 0, -1) index_search_set_1 = search_set_keys.index("000007") index_search_set_2 = search_set_keys.index("000721") index_search_set_3 = search_set_keys.index("600796") self.assertEqual(active.zrange("search:2018-12-09 01", index_search_set_1, index_search_set_1, withscores=True)[0][1], 13.0, "search set value check failed") self.assertEqual(active.zrange("search:2018-12-09 01", index_search_set_2, index_search_set_2, withscores=True)[0][1], 13.0, "search set value check failed") self.assertEqual(active.zrange("search:2018-12-09 01", index_search_set_3, index_search_set_3, withscores=True)[0][1], 13.0, "search set value check failed")
def input_drug_gene_condense(): dir_opt = '/datainfo' deletion_list = [] final_dl_input_df = pd.read_table( './datainfo/mid_data/final_GDSC2_dl_input.txt', delimiter=',') drug_map_dict, drug_dict, gene_target_num_dict = LoadData( dir_opt).pre_load_dict() target_index_list = gene_target_num_dict.values() drug_target_matrix = np.load( './datainfo/filtered_data/drug_target_matrix.npy') for row in final_dl_input_df.itertuples(): drug_a = drug_map_dict[row[2]] cellline_name = row[1] # DRUG_A AND 929 TARGET GENES drug_a_target_list = [] drug_index = drug_dict[drug_a] for target_index in target_index_list: if target_index == -1: effect = 0 else: effect = drug_target_matrix[drug_index, target_index] drug_a_target_list.append(effect) if all([a == 0 for a in drug_a_target_list]): deletion_list.append(row[0]) print('=====================' + str(len(deletion_list))) zero_final_dl_input_df = final_dl_input_df.drop( final_dl_input_df.index[deletion_list]).reset_index(drop=True) zero_final_dl_input_df.to_csv( '.' + dir_opt + '/filtered_data/zerofinal_GDSC2_dl_input.txt', index=False, header=True) print(zero_final_dl_input_df)
def create_datasets(load_d: LoadData): """ loads already created important files :param load_d: query processor in case of a missing file :return: loaded objects """ try: wf_matching = pd.read_csv('matching.csv') except FileNotFoundError: create_matching('admissions.csv', "RECORDS-waveforms.csv") wf_matching = pd.read_csv('matching.csv') try: numeric_matching = pd.read_csv('matching_numerics.csv') except FileNotFoundError: create_matching('admissions.csv', "RECORDS-numerics.csv", numerics=True) numeric_matching = pd.read_csv('matching.csv') try: mortality = pd.read_csv("in_hospital_mortality.csv") except FileNotFoundError: mortality = load_d.query_db(queries.in_hospital_mortality()) mortality.to_csv("in_hospital_mortality.csv") try: clinical_features = pd.read_csv("features_clinical.csv") except FileNotFoundError as e: print(str(e)) print("please create clinical features file for all patients") return wf_matching, numeric_matching, mortality, clinical_features
def main(mode): print('Loading data...') data = LoadData(dataset_path='dataset', images_path='dataset/images/') print('Creating Batch Generator...') batch_creator = BatchGenerator(data_dict=data.data_dict, captions_int=data.captions_int, image_addr=data.image_addr, **batch_params) if mode == 'train': print('Creating Models...') caption_model = CaptionLSTM(model_params=model_params, int2word=data.int2word) print('Starting training...') class_weights = calc_class_weights(data.captions_int.values) train(caption_model, batch_creator, class_weights, **train_params) elif mode == 'sample': print('Loading model...') model_file = open('vgg_lstm.pkl', 'rb') model = pickle.load(model_file) print('Creating sample..') sample(model, batch_creator, top_k=10, seq_len=16, show_image=True) elif mode == 'test': print('Loading model') model_file = open('vgg_lstm.pkl', 'rb') model = pickle.load(model_file) print('Testing model...') test(model, batch_creator, top_k=10, seq_len=16)
def continue_run_rand_nn(matrixA, matrixB, num_gene, num_pathway, layer1, layer2, layer3, path, dir_opt, RNA_seq_filename, input_num, epoch, batch_size, verbose): # RECONSTRCUT TO BE TRAINED MODEL model = RandNN().keras_rand_nn(matrixA, matrixB, num_gene, layer0, layer1, layer2, layer3) with open(path + '/layer_bias_list.txt', 'rb') as filebias: layer_bias_list = pickle.load(filebias) with open(path + '/layer_weight_list.txt', 'rb') as fileweight: layer_weight_list = pickle.load(fileweight) model.compile(loss='mean_squared_error', optimizer='adam', metrics=['mse', 'accuracy']) xTmp, yTmp = LoadData(dir_opt, RNA_seq_filename).load_train(0, 1) model.fit(xTmp, yTmp, epochs=1, validation_split=1, verbose=0) model_layer_list = [] num_layer = len(model.layers) for i in range(num_layer): each_layer_list = [layer_weight_list[i], layer_bias_list[i]] model_layer_list.append(each_layer_list) model.layers[i].set_weights(each_layer_list) # AUTO UPDATE WEIGHT model, history, num_layer, path = RunRandNN( model, dir_opt, RNA_seq_filename).train(input_num, epoch, batch_size, verbose) return model, history, path
def drug_gene_pathway_reform(self): dir_opt = self.dir_opt rna_df = pd.read_csv('./datainfo/filtered_data/tailed_rnaseq_fpkm_20191101.csv') gene_list = list(rna_df['symbol']) drug_map_dict, drug_dict, gene_target_num_dict = LoadData(dir_opt).pre_load_dict() # 929 MAPPED GENES INDEX IN [drug_target_matrix] target_index_list = gene_target_num_dict.values() # 24 DRUGS IN DEEP LEARNING TASK zero_final_dl_input_df = pd.read_table('./datainfo/filtered_data/zerofinal_GDSC2_dl_input.txt', delimiter = ',') zero_final_drug_list = [] for drug in zero_final_dl_input_df['DRUG_NAME']: if drug not in zero_final_drug_list: zero_final_drug_list.append(drug) zero_final_drug_list = sorted(zero_final_drug_list) # 24 MAPPED DRUGS IN DEEP LEARNING TASK mapped_drug_list = [] for zero_drug in zero_final_drug_list: mapped_drug_list.append(drug_map_dict[zero_drug]) drug_target_matrix = np.load('.' + dir_opt + '/filtered_data/drug_target_matrix.npy') # FIND DRUGS CAN TARGET ON GENES multi_drug_list = [] for target_index in target_index_list: temp_drug_list = [] if target_index == -1: temp_drug_list = ['NaN'] multi_drug_list.append(temp_drug_list) else: for mapped_drug in mapped_drug_list: drug_index = drug_dict[mapped_drug] effect = drug_target_matrix[drug_index, target_index] if effect == 1: temp_drug_list.append(mapped_drug) if len(temp_drug_list) == 0: temp_drug_list = ['NaN'] multi_drug_list.append(temp_drug_list) # CONVERT EACH GENES TARGETED DRUGS TO DATAFRAME drug_gene = {'Drugs': multi_drug_list, 'Genes': gene_list} drug_gene_df = pd.DataFrame(drug_gene, columns=['Drugs','Genes']) drug_gene_df.to_csv('./datainfo/filtered_data/drug_gene.csv', index = False, header = True) # ADD PATHWAYS TO CORRESPONDING GENES gene_pathway_df = pd.read_csv('./datainfo/filtered_data/Tailed_Selected_Kegg_Pathways2.csv') pathway_name_list = list(gene_pathway_df.columns)[1:] multi_pathway_list = [] # import pdb; pdb.set_trace() for row in gene_pathway_df.itertuples(): temp_pathway_list = [] for index in np.arange(2, 48): if row[index] == 1: temp_pathway_list.append(pathway_name_list[index - 2]) if len(temp_pathway_list) == 0: temp_pathway_list = ['NaN'] print(row[1]) multi_pathway_list.append(temp_pathway_list) # print(multi_pathway_list) # print(len(multi_pathway_list)) # CONVERT EACH GENES TARGETED DRUGS/ CONNECTION TO PATHWAYS TO DATAFRAME drug_gene_pathway = {'Drugs': multi_drug_list, 'Genes': gene_list, 'Pathways': multi_pathway_list} drug_gene_pathway_df = pd.DataFrame(drug_gene_pathway, columns=['Drugs', 'Genes', 'Pathways']) drug_gene_pathway_df.to_csv('./datainfo/filtered_data/drug_gene_pathway.csv', index = False, header = True) print(drug_gene_pathway_df)
def verify_input(): # get the data from input folder val1 = v1.get() chem_type = get_chemType(val1) chem_file = get_chemFile(chem_type) region_file = CUR_PATH + './Input/Region.xlsx' release_file = CUR_PATH + './Input/ChemRelease.xlsx' start_date_temp = '2005 1 1' end_date_temp = '2005 12 31' if chem_type != 'Nanomaterial': load_data_nonNano = LoadData(chem_type, chem_file, region_file, release_file, start_date_temp, end_date_temp) chem_params, presence, env, climate, bgConc, release, release_scenario = load_data_nonNano.run_loadData( ) else: time, presence, env, climate, bgConc, chem_params, release, release_scenario = load_data( region_file, release_file, chem_file, start_date_temp, end_date_temp) chem_txt4 = Label(text='Chemical:', bg=bg_col).place(x=140, y=170) chem_name = StringVar(window, value=chem_params['name']) entry1 = Entry(window, textvariable=chem_name, bg='lightgrey', state='disabled') entry1.place(x=220, y=170, width=250) chem_txt5 = Label(text='Region:', bg=bg_col).place(x=140, y=200) region_name = StringVar(window, value=env['name']) entry2 = Entry(window, textvariable=region_name, bg='lightgrey', state='disabled') entry2.place(x=220, y=200, width=250) chem_txt6 = Label(text='Release:', bg=bg_col).place(x=140, y=230) release_scenario = StringVar(window, value=release_scenario) entry3 = Entry(window, textvariable=release_scenario, bg='lightgrey', state='disabled') entry3.place(x=220, y=230, width=250) return chem_params, env, release_scenario
def testprediction(config_file, pred_file=None, label_file=None, out_path=None): config = ConfigParser.ConfigParser() config.read(config_file) test_data = LoadData(directory = config.get('Testing Data', 'folders').split(','), data_file_name = config.get('Testing Data', 'data_file'), label_file_name = config.get('Testing Data', 'label_file'), seg_file_name = config.get('Testing Data', 'seg_file')) res = Analyzer(raw = test_data.get_data()[0], target = test_data.get_labels()[0]) res.add_results(results_folder = config.get('General','directory'), name = config_file.split('/')[-3], prediction_file = config.get('Testing', 'prediction_file')+'_0', learning_curve_file = 'learning_curve') res.analyze(-1, pred_file=pred_file, label_file=label_file, out_path=out_path) return res
def evaluate_accuracy(): torch.backends.cudnn.deterministic = True device = torch.device("cuda") print("CUDA visible devices: " + str(torch.cuda.device_count())) print("CUDA Device Name: " + str(torch.cuda.get_device_name(device))) # Create test dataset loader test_dataset = LoadData(dataset_dir, TEST_SIZE, 2.0, test=True) test_loader = DataLoader(dataset=test_dataset, batch_size=1, shuffle=False, num_workers=1, pin_memory=True, drop_last=False) # Define the model architecture and restore it from the .pth file, e.g.: model = PyNET(level=0, instance_norm=True, instance_norm_level_1=True).to(device) model = torch.nn.DataParallel(model) model.load_state_dict(torch.load("models/original/pynet_level_0.pth"), strict=True) # Define the losses MSE_loss = torch.nn.MSELoss() MS_SSIM = MSSSIM() loss_psnr = 0.0 loss_msssim = 0.0 model.eval() with torch.no_grad(): test_iter = iter(test_loader) for j in range(len(test_loader)): x, y = next(test_iter) x = x.to(device, non_blocking=True) y = y.to(device, non_blocking=True) # Process raw images with your model: enhanced = model(x) # Compute losses loss_mse_temp = MSE_loss(enhanced, y).item() loss_psnr += 20 * math.log10(1.0 / math.sqrt(loss_mse_temp)) loss_msssim += MS_SSIM(y, enhanced).detach().cpu().numpy() loss_psnr = loss_psnr / TEST_SIZE loss_msssim = loss_msssim / TEST_SIZE output_logs = "PSNR: %.4g, MS-SSIM: %.4g\n" % (loss_psnr, loss_msssim) print(output_logs)
def start_cnn(): load_data = LoadData() process_data = ProcessDataCNN(sales=load_data.make_dataset( categorize=CATEGORIZE, start=START, upper=UPPER)) xt, yt, xv, yv, xe, ye = process_data.run() if not ONLY_LOAD_MODEL: ckpt = ModelCheckpoint(OUTPUT_MODEL, monitor='val_loss', verbose=1, save_best_only=True, mode='min') reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=5, min_lr=MIN_LR, verbose=1) es = EarlyStopping(monitor='val_loss', patience=3) net = make_model_cnn(len(process_data.FEATS)) plot_model(net, to_file='model.png') print(net.summary()) # exit() net.fit(xt, yt, batch_size=BATCH_SIZE, epochs=EPOCH, validation_data=(xv, yv), callbacks=[reduce_lr, ckpt, es]) nett = make_model_cnn(len(process_data.feats_list)) nett.load_weights(OUTPUT_MODEL) pv = nett.predict(xv, batch_size=BATCH_SIZE, verbose=1) pe = nett.predict(xe, batch_size=BATCH_SIZE, verbose=1) print("Eva result:", nett.evaluate(xv, yv, batch_size=BATCH_SIZE)) pv = pv.reshape((-1, 28, 9)) pe = pe.reshape((-1, 28, 9)) sv = process_data.sv.reshape((-1, 28)) se = process_data.se.reshape((-1, 28)) Yv = yv.reshape((-1, 28)) return process_data, Yv, pv, pe, sv, se
def test_load_data_expect_summean_when_links_are_valid(self): endpoints_reader = LoadData() data = Processor() json_link = '[{"date":"22-01-2021","impressions": 1376}, \ {"date":"21-01-2021","impressions": 1906},{"date":"20-01-2021","impressions": 2818},\ {"date":"19-01-2021","impressions": 1024}, \ {"date":"18-01-2021","impressions": 646},{"date":"17-01-2021","impressions": 2885}, \ {"date":"16-01-2021","impressions": 1889},{"date":"15-01-2021","impressions": 1534}, \ {"date":"14-01-2021","impressions": 995},{"date":"13-01-2021","impressions": 1251}, \ {"date":"12-01-2021","impressions": 2062},{"date":"11-01-2021","impressions": 1204}, \ {"date":"10-01-2021","impressions": 2030},{"date":"09-01-2021","impressions": 1166}, \ {"date":"08-01-2021","impressions": 2025},{"date":"07-01-2021","impressions": 1221}, \ {"date":"06-01-2021","impressions": 2018},{"date":"05-01-2021","impressions": 2484}, \ {"date":"04-01-2021","impressions": 1145},{"date":"03-01-2021","impressions": 2686}, \ {"date":"02-01-2021","impressions": 2186},{"date":"01-01-2021","impressions": 1527}, \ {"date":"31-12-2020","impressions": 1710},{"date":"30-12-2020","impressions": 1343}, \ {"date":"29-12-2020","impressions": 2466},{"date":"28-12-2020","impressions": 952}, \ {"date":"27-12-2020","impressions": 532},{"date":"26-12-2020","impressions": 2690}, \ {"date":"25-12-2020","impressions": 2428},{"date":"24-12-2020","impressions": 602}, \ {"date":"23-12-2020","impressions": 995},{"date":"22-12-2020","impressions": 615}, \ {"date":"21-12-2020","impressions": 2055},{"date":"20-12-2020","impressions": 1337}, \ {"date":"19-12-2020","impressions": 1824},{"date":"18-12-2020","impressions": 1645}, \ {"date":"17-12-2020","impressions": 2655},{"date":"16-12-2020","impressions": 2619}, \ {"date":"15-12-2020","impressions": 1189},{"date":"14-12-2020","impressions": 2391}, \ {"date":"13-12-2020","impressions": 1612},{"date":"12-12-2020","impressions": 510}, \ {"date":"11-12-2020","impressions": 2655},{"date":"10-12-2020","impressions": 2029}, \ {"date":"09-12-2020","impressions": 2899},{"date":"08-12-2020","impressions": 1170}, \ {"date":"07-12-2020","impressions": 526},{"date":"06-12-2020","impressions": 2092}, \ {"date":"05-12-2020","impressions": 1453},{"date":"04-12-2020","impressions": 738}, \ {"date":"03-12-2020","impressions": 700},{"date":"02-12-2020","impressions": 1485}, \ {"date":"01-12-2020","impressions": 1571},{"date":"30-11-2020","impressions": 2106}, \ {"date":"29-11-2020","impressions": 2646},{"date":"28-11-2020","impressions": 1092}, \ {"date":"27-11-2020","impressions": 1495},{"date":"26-11-2020","impressions": 2356}, \ {"date":"25-11-2020","impressions": 1474},{"date":"24-11-2020","impressions": 1431}, \ {"date":"23-11-2020","impressions": 1359},{"date":"22-11-2020","impressions": 1420}]' csv_link = 'ba026992-281a-42a6-8447-ae1c8a04106e.csv' csv_file, json_file = endpoints_reader.read_data(csv_link, json_link) summary_csv_info, summary_json_info = data.process_endpoints(csv_file, json_file) print(summary_csv_info, summary_json_info) self.assertEqual(str(summary_csv_info), '{"mean": 1781.85, "sum": 110475}') self.assertEqual(str(summary_json_info), '{"mean": 1660.4, "sum": 102945}')
def zero_final_drug_count(): zero_final_dl_input_df = pd.read_table('./datainfo/filtered_data/zerofinal_GDSC2_dl_input.txt', delimiter = ',') zero_final_drug_list = [] for drug in zero_final_dl_input_df['DRUG_NAME']: if drug not in zero_final_drug_list: zero_final_drug_list.append(drug) zero_final_drug_list = sorted(zero_final_drug_list) print(zero_final_drug_list) print(len(zero_final_drug_list)) # Count the Number of Drugs Intersection Between [dl_input, drugBank] dir_opt = '/datainfo' drug_map_dict, drug_dict, gene_target_num_dict = LoadData(dir_opt).pre_load_dict() count = 0 anti_count = 0 for key, value in drug_map_dict.items(): if type(value) is str: count = count + 1 elif math.isnan(value) == True: anti_count = anti_count + 1 print(count) print(anti_count)
def cardiac_patients(query_proccesor: LoadData, admissions: str, gender: str, path: str, cardiac=True): """ Extract patients using a filtering scheme :param query_proccesor: sql handler :param admissions: relevant admissions :param gender: desired gender for the patients :param path: path to already created file :param cardiac: cardiac or non-cardiac patients :return: patient dataframe with demographics """ try: patients = pd.read_csv(path) except FileNotFoundError: patients = query_proccesor.query_db(queries.patient_selection(admissions, gender, cardiac)) patients.to_csv(path) return patients
def manual_test_rand_nn(matrixA, matrixB, num_gene, num_pathway, layer1, layer2, layer3, path, dir_opt, learning_rate): # MANUAL REBUILD THE MODEL input_model, gene_model, pathway_model, model = build_rand_nn(matrixA, matrixB, num_gene, num_pathway, layer1, layer2, layer3) with open(path + '/layer_list.txt', 'rb') as filelayer: layer_list = pickle.load(filelayer) model.compile(loss='mean_squared_error', optimizer=Adam(lr=learning_rate), metrics=['mse', 'accuracy']) xTmp, yTmp = LoadData(dir_opt).load_train(0, 1) model.fit(xTmp, yTmp, epochs = 1, validation_split = 1, verbose = 0) num_layer = len(model.layers) for i in range(num_layer): model.get_layer(index = i).set_weights(layer_list[i]) # PREDICT MODEL USING [xTe, yTe] verbose = 1 y_pred, score = RunRandNN(model, dir_opt).test(verbose, path)
def rna_proteomic_mismatch_probabilities(train=True): # Train siamese network data = LoadData() pro_data = data.proteomic rna_data = data.rna prot_x = pd.concat([pro_data] * 3 + [data.test_proteomic] * 3) shuffled_rna = rna_data.sample(frac=1) rna_x = pd.concat([ rna_data, shuffled_rna, rna_data.sample(frac=1), data.test_rna, data.test_rna.sample(frac=1), data.test_rna.sample(frac=1) ]) labels = [1.0] * 80 + [0.0] * 160 + [1.0] * 80 + [0.0] * 160 network = SiameseNet([(pro_data.shape[-1], ), (rna_data.shape[-1], )]) network.fit([prot_x, rna_x], labels, epochs=100, batch_size=5, verbose=False) # Calculate pairwise probabilities if not train: pro_data = data.test_proteomic rna_data = data.test_rna vals = { "Proteomic": [], "RNA": [], "Probability": [], } for i, x in pro_data.iterrows(): for j, y in rna_data.iterrows(): vals['Proteomic'].append(x.name) vals['RNA'].append(y.name) vals['Probability'].append(network.predict([[x], [y]])[0][0]) probs = pd.DataFrame(vals) if train: order = data.clinical.index.tolist() else: order = data.test_clinical.index.tolist() probs = probs.pivot(index='RNA', columns='Proteomic', values='Probability')[order].reindex(order) return probs
def continue_run_rand_nn(matrixA, matrixB, num_gene, num_pathway, layer1, layer2, layer3, path, dir_opt, input_num, epoch, batch_size, verbose, learning_rate, end_epoch): # REBUILD DECOMPOSED MODEL FROM SAVED MODEL input_model, gene_model, pathway_model, model = build_rand_nn(matrixA, matrixB, num_gene, num_pathway, layer1, layer2, layer3) with open(path + '/layer_list.txt', 'rb') as filelayer: layer_list = pickle.load(filelayer) model.compile(loss='mean_squared_error', optimizer=Adam(lr=learning_rate), metrics=['mse', 'accuracy']) xTmp, yTmp = LoadData(dir_opt).load_train(0, 1) model.fit(xTmp, yTmp, epochs = 1, validation_split = 1, verbose = 0) num_layer = len(model.layers) for i in range(num_layer): model.get_layer(index = i).set_weights(layer_list[i]) # RUN MODEL (AUTO UPDATE WEIGHT) model, history, path = run_rand_nn(model, dir_opt, matrixA, matrixB, input_num, epoch, batch_size, verbose, learning_rate, end_epoch) return model, history, path
def test(self, verbose, path): model = self.model dir_opt = self.dir_opt RNA_seq_filename = self.RNA_seq_filename xTe, yTe = LoadData(dir_opt, RNA_seq_filename).load_test() # TEST OUTPUT PRED y_pred = model.predict(xTe) y_pred_list = [item for elem in y_pred for item in elem] score = model.evaluate(xTe, yTe, verbose = verbose) final_test_input_df = pd.read_csv('.' + dir_opt + '/filtered_data/TestInput.txt', delimiter = ',') final_row, final_col = final_test_input_df.shape final_test_input_df.insert(final_col, 'Pred Score', y_pred_list, True) final_test_input_df.to_csv(path + '/PredTestInput.txt', index = False, header = True) # ANALYSE PEARSON CORR test_pearson = final_test_input_df.corr(method = 'pearson') print(score) print(test_pearson) return y_pred, score
def clinical_labels_dict(train=True): data = LoadData() def clinical_to_int(row): output = 0 if row.msi == "MSI-High": output += 2 if row.gender == "Female": output += 1 return output if train: clin_data = data.clinical else: clin_data = data.test_clinical labels = clin_data.apply(clinical_to_int, axis="columns") return {sample: label for sample, label in zip(clin_data.index, labels)}
def clinical_probabilities(train=True, learner=learner_functions.train_rf): """Get the probabilities of each clinical class for each sample for proteomic and rna data. Returns (proteomic_probabilities_df, rna_probabilities_df) """ data = LoadData() def clinical_to_int(row): output = 0 if row.msi == "MSI-High": output += 2 if row.gender == "Female": output += 1 return output train_labels = pd.concat([data.clinical, data.test_clinical]).apply(clinical_to_int, axis="columns") train_proteomic = pd.concat([data.proteomic, data.test_proteomic]) train_rna = pd.concat([data.rna, data.test_rna]) model = learner(train_proteomic, train_labels) if train: proteomic_probabilities_df = pd.DataFrame(model[0].predict_proba( data.proteomic)) proteomic_probabilities_df['sample'] = data.clinical.index else: proteomic_probabilities_df = pd.DataFrame(model[0].predict_proba( data.test_proteomic)) proteomic_probabilities_df['sample'] = data.test_clinical.index proteomic_probabilities_df = proteomic_probabilities_df.set_index('sample') model = learner(train_rna, train_labels) if train: rna_probabilities_df = pd.DataFrame(model[0].predict_proba(data.rna)) rna_probabilities_df['sample'] = data.clinical.index else: rna_probabilities_df = pd.DataFrame(model[0].predict_proba( data.test_rna)) rna_probabilities_df['sample'] = data.test_clinical.index rna_probabilities_df = rna_probabilities_df.set_index('sample') return proteomic_probabilities_df, rna_probabilities_df
def input_drug_gene_condense(self, RNA_seq_filename): dir_opt = self.dir_opt deletion_list = [] final_dl_input_df = pd.read_table( '.' + dir_opt + '/mid_data/FinalDeepLearningInput.txt', delimiter=',') drug_map_dict, cellline_map_dict, drug_dict, gene_target_num_dict = LoadData( dir_opt, RNA_seq_filename).pre_load_dict() target_index_list = gene_target_num_dict.values() drug_target_matrix = np.load('.' + dir_opt + '/filtered_data/drug_target_matrix.npy') for row in final_dl_input_df.itertuples(): drug_a = drug_map_dict[row[1]] drug_b = drug_map_dict[row[2]] cellline_name = cellline_map_dict[row[3]] # DRUG_A AND 1130 TARGET GENES drug_a_target_list = [] drug_index = drug_dict[drug_a] for target_index in target_index_list: if target_index == -1: effect = 0 else: effect = drug_target_matrix[drug_index, target_index] drug_a_target_list.append(effect) # DRUG_B AND 1130 TARGET GENES drug_b_target_list = [] drug_index = drug_dict[drug_b] for target_index in target_index_list: if target_index == -1: effect = 0 else: effect = drug_target_matrix[drug_index, target_index] drug_b_target_list.append(effect) if all([a == 0 for a in drug_a_target_list]) or all( [b == 0 for b in drug_b_target_list]): deletion_list.append(row[0]) zero_final_dl_input_df = final_dl_input_df.drop( final_dl_input_df.index[deletion_list]).reset_index(drop=True) zero_final_dl_input_df.to_csv( '.' + dir_opt + '/filtered_data/ZeroFinalDeepLearningInput.txt', index=False, header=True) print(zero_final_dl_input_df)
def main(mode): dataset_path = 'data' data = LoadData(dataset_path=dataset_path, **data_params) print('Creating Batch Generator...') batch_gen = BatchGenerator(data_dict=data.data_dict, label_dict=data.label_dict, **batch_params) if mode == 'train': train(vocabs=[data.word2int, data.int2word], batch_gen=batch_gen, train_params=train_params, model_params=model_params) elif mode == 'test': print('Loading model') model_file = open('results/seq2seq.pkl', 'rb') model = pkl.load(model_file) print('Testing model...') test(model, data.int2word, batch_gen)
def ViewResults(**kwargs): directory = kwargs.get("directory", "") network = kwargs.get("network", None) prediction_file = kwargs.get("predictions_file", None) if network: # Assume that all networks are tested on the same set of data config = ConfigParser.ConfigParser() config.read("networks/" + network + "/network.cfg") data = LoadData( directory=config.get("Testing Data", "folders").split(",")[0], data_file_name=config.get("Testing Data", "data_file"), label_file_name=config.get("Testing Data", "label_file"), ) if not prediction_file: prediction_file = "test_prediction_0" results = Analyzer(target=data.get_labels()[0], raw=data.get_data()[0]) results.add_results(results_folder="networks/" + network + "/", name=network, prediction_file=prediction_file) else: folders = os.listdir(directory) networks = [] for folder in folders: if os.path.isfile(directory + folder + "/network.cfg"): networks.append(folder) # Assume that all networks are tested on the same set of data config = ConfigParser.ConfigParser() config.read(directory + networks[0] + "/network.cfg") data = LoadData( directory=config.get("Testing Data", "folders").split(",")[0], data_file_name=config.get("Testing Data", "data_file"), label_file_name=config.get("Testing Data", "label_file"), ) if not prediction_file: prediction_file = "test_prediction_0" results = Analyzer(target=data.get_labels()[0], raw=data.get_data()[0]) for net in networks: results.add_results(results_folder=directory + net + "/", name=net, prediction_file=prediction_file) return results
def test_cleanup(self): with open("../data/pt_BR/nnp") as f: nnp = [line.rstrip() for line in f.readlines()] with open("../data/pt_BR/terms") as f: terms = [line.rstrip() for line in f.readlines()] with open("../data/pt_BR/patterns") as f: patterns = [line.rstrip() for line in f.readlines()] data = LoadData(['../corpus/sel1.csv', '../corpus/sel2.csv'], ['D', 'C']).load() p = CleanupData(nnp, terms, patterns) new_data = p.clean(data) with open("results/clean.txt", "w") as f: f.write("\n".join("%s;%s;%s" % (d.identifier, d.text, d.status) for d in new_data)) for i, d in enumerate(new_data): dirname = "conectado" if d.status == "C" else "desconectado" filename = "results/%s/clt_%s.txt" % (dirname, i) with open(filename, "w") as f: f.write(d.text)
def manual_test_rand_nn(matrixA, matrixB, num_gene, num_pathway, layer1, layer2, layer3, path, dir_opt, RNA_seq_filename): # RECONSTRCUT TEST MODEL model = RandNN().keras_rand_nn(matrixA, matrixB, num_gene, layer0, layer1, layer2, layer3) with open(path + '/layer_bias_list.txt', 'rb') as filebias: layer_bias_list = pickle.load(filebias) with open(path + '/layer_weight_list.txt', 'rb') as fileweight: layer_weight_list = pickle.load(fileweight) model.compile(loss='mean_squared_error', optimizer='adam', metrics=['mse', 'accuracy']) xTmp, yTmp = LoadData(dir_opt, RNA_seq_filename).load_train(0, 1) model.fit(xTmp, yTmp, epochs=1, validation_split=1, verbose=0) model_layer_list = [] num_layer = len(model.layers) for i in range(num_layer): each_layer_list = [layer_weight_list[i], layer_bias_list[i]] model_layer_list.append(each_layer_list) model.layers[i].set_weights(each_layer_list) # PREDICT MODEL USING [xTe, yTe] verbose = 1 y_pred, score = RunRandNN(model, dir_opt, RNA_seq_filename).test(verbose, path)
def main(tablename, use_json=0, database='tnt', start_date=0, end_date=0, batch_size=30): try: last_version = get_last_version(tablename) file_path = "{}/data/{}_v{}.json".format( Settings.BASE_DIR, tablename, last_version) if use_json == 0 else use_json source_conn = Connection( 'mo4jo').connect_mssql() if use_json is None else None target_conn = Connection(database).connect_mysql() start_date = start_date if start_date != None else '2012-01-01' batch_size = Settings.DATABASE_CONFIG['tnt']['batch_size'] now = datetime.utcnow().date() mo4jo_pac_clients_mapping = json.loads( open( "{}/data/mo4jo_pac_clients_mapping.json".format( Settings.BASE_DIR), "r").read()) end_date = end_date if end_date != None else now # entity_json = PullData(source_conn, tablename, file_path).get_data(kwargs={'start_date':start_date,'end_date':end_date}) entity_json = PullData(source_conn, tablename, file_path).get_data() if type(end_date) == 'datetime.date': end_date = datetime.strptime(end_date, '%Y-%m-%d').date() LoadData(target_conn, tablename, entity_json, start_date, end_date, batch_size).load_data(mo4jo_pac_clients_mapping) # LoadData(target_conn, tablename, entity_json).load_data(mo4jo_pac_clients_mapping) except Exception as e: import traceback print(traceback.format_exc())
def bias_variable(shape): initial = tf.constant(0.1, shape=shape) return tf.Variable(initial) def conv2d(x, W): return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME') def max_pool_2x2(x): return tf.nn.max_pool(x, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME') # Load the Digit DataSet load_data = LoadData() train_set_x, train_set_y = load_data.load_train_data("/home/darshan/Documents/DigitRecognizer/MNIST_data/", "train.csv") #mnist = input_data.read_data_sets("MNIST_data/", one_hot=True) x = tf.placeholder(tf.float32, [None, 784]) x_image = tf.reshape(x, [-1, 28, 28, 1]) y_ = tf.placeholder(tf.float32, [None, 10]) # First Layer of Convnet W_conv1 = weight_variable([5, 5, 1, 32]) b_conv1 = bias_variable([32]) # 28 x 28 -> 24 x 24 h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1)
theano.sandbox.cuda.use(device) if device != "cpu": theano.config.nvcc.flags = "-use=fast=math" theano.config.allow_gc = False print "\nLoading Data..." # ------------------------------------------------------------------------------ # Load data (even though most of it won't be used) train_data_folder = "/nobackup/turaga/data/fibsem_medulla_7col/trvol-250-1-h5/" data_file = "img_normalized.h5" label_file = "groundtruth_aff.h5" seg_file = "groundtruth_seg.h5" data = LoadData( directory=train_data_folder, data_file_name=data_file, label_file_name=label_file, seg_file_name=seg_file ) # ------------------------------------------------------------------------------ # Search log_interval = 100 results = [] print "\nBeginning Search:" fsize = 7 while fsize > 2: nlayers = 4 while nlayers < max_layers: nfilters = 50 while nfilters < max_filters: name = "conv-" + ` nlayers ` + "." + ` nfilters ` + "." + ` fsize `
def makeprediction(config_file, data_file=None, out_path=None, out_file=None, gpu=None): #Open configuration file for this network config = ConfigParser.ConfigParser() config.read(config_file) #Set the device on which to perform these computations if gpu: theano.sandbox.cuda.use(gpu) theano.config.nvcc.flags='-use=fast=math' theano.config.allow_gc=False else: device = config.get('General', 'device') theano.sandbox.cuda.use(device) if (device != 'cpu'): theano.config.nvcc.flags='-use=fast=math' theano.config.allow_gc=False #------------------------------------------------------------------------------ starttime=time.clock() print '\nInitializing Network' if os.path.exists(config.get('General', 'directory')+config.get('Network', 'weights_folder')): network = CNN(weights_folder = config.get('General', 'directory')+config.get('Network', 'weights_folder'), activation = config.get('Network', 'activation')) else: print 'Error: Weights folder does not exist. Could not initialize network' return; #------------------------------------------------------------------------------ print 'Opening Data Files' if data_file: test_data = LoadData(directory = '', data_file_name = data_file) else: test_data = LoadData(directory = config.get('Testing Data', 'folders').split(','), data_file_name = config.get('Testing Data', 'data_file')) #------------------------------------------------------------------------------ init_time = time.clock() - starttime print "Initialization = " + `init_time` + " seconds" starttime = time.clock() print 'Making Predictions' if out_path and out_file: network.predict(test_data.get_data(), results_folder = out_path, name = out_file) elif out_path: network.predict(test_data.get_data(), results_folder = out_path, name = config.get('Testing', 'prediction_file')) elif out_file: network.predict(test_data.get_data(), results_folder = config.get('General', 'directory')+config.get('Testing', 'prediction_folder'), name = out_file) else: network.predict(test_data.get_data(), results_folder = config.get('General', 'directory')+config.get('Testing', 'prediction_folder'), name = config.get('Testing', 'prediction_file')) pred_time = time.clock() - starttime #------------------------------------------------------------------------------ print "Prediction Time = " + `pred_time` + " seconds" test_data.close()
def trainnetwork(config_file): #Open configuration file for this network config = ConfigParser.ConfigParser() config.read(config_file) #Set the device on which to perform these computations device = config.get('General', 'device') theano.sandbox.cuda.use(device) if (device != 'cpu'): theano.config.nvcc.flags='-use=fast=math' theano.config.allow_gc=False #------------------------------------------------------------------------------ print '\nOpening Data Files' #Load the data for training training_data = LoadData(directory = config.get('Training Data', 'folders').split(','), data_file_name = config.get('Training Data', 'data_file'), label_file_name = config.get('Training Data', 'label_file'), seg_file_name = config.get('Training Data', 'seg_file')) #------------------------------------------------------------------------------ starttime=time.clock() #Create the network and trainer if os.path.exists(config.get('General', 'directory')+config.get('Network', 'weights_folder')): print 'Loading Network' network = CNN(weights_folder = config.get('General', 'directory')+config.get('Network', 'weights_folder'), activation = config.get('Network', 'activation')) print 'Loading Trainer' network_trainer = Trainer(network, training_data.get_data(), training_data.get_labels(), training_data.get_segments(), chunk_size = config.getint('Training', 'chunk_size'), batch_size = config.getint('Training', 'batch_size'), cost_func = config.get('Training', 'cost_func'), learning_method = config.get('Training', 'learning_method'), learning_rate = config.getfloat('Training', 'learning_rate'), beta1 = config.getfloat('Training', 'beta1'), beta2 = config.getfloat('Training', 'beta2'), damping = config.getfloat('Training', 'damping'), trainer_folder = config.get('General', 'directory')+config.get('Training', 'trainer_folder'), log_interval = config.getint('Training', 'log_interval'), log_folder = config.get('General', 'directory')+config.get('Training', 'log_folder')) else: print 'Initializing Network' network = CNN(num_layers = config.getint('Network', 'num_layers'), num_filters = config.getint('Network', 'num_filters'), filter_size = config.getint('Network', 'filter_size'), activation = config.get('Network', 'activation')) print 'Initializing Trainer' network_trainer = Trainer(network, training_data.get_data(), training_data.get_labels(), training_data.get_segments(), chunk_size = config.getint('Training', 'chunk_size'), batch_size = config.getint('Training', 'batch_size'), cost_func = config.get('Training', 'cost_func'), learning_method = config.get('Training', 'learning_method'), learning_rate = config.getfloat('Training', 'learning_rate'), beta1 = config.getfloat('Training', 'beta1'), beta2 = config.getfloat('Training', 'beta2'), damping = config.getfloat('Training', 'damping'), log_interval = config.getint('Training', 'log_interval'), log_folder = config.get('General', 'directory')+config.get('Training', 'log_folder')) init_time = time.clock() - starttime #------------------------------------------------------------------------------ print "Initialization = " + `init_time` + " seconds" starttime = time.clock() #Train the network print 'Training...\n' train_error = network_trainer.train(config.getint('Training', 'num_epochs'), config.getboolean('Training', 'early_stop'), config.getboolean('Training', 'print_updates')) total_time = time.clock() - starttime #------------------------------------------------------------------------------ print "Total Time =",total_time,"seconds" training_data.close() return train_error
def _do(self, data): load_data = LoadData(data) load_data.main()