def main(): args = read_args() logging.info("Reading the file. This may take a few mintues") f_in, file_size = read_file(args.file_in, args.size) f = DNAFountain(file_in = f_in, file_size = file_size, chunk_size = args.size , rs = args.rs, max_homopolymer = args.max_homopolymer, gc = args.gc, delta = args.delta, c_dist = args.c_dist, np = args.rand_numpy, alpha = args.alpha, stop = args.stop) logging.info("Upper bounds on packets for decoding is %d (x%f) with %f probability\n", int(json.loads(f.PRNG.debug())['K_prime']), json.loads(f.PRNG.debug())['Z'], json.loads(f.PRNG.debug())['delta']) if (args.out == '-'): out = sys.stdout else: out = open (args.out, 'w') pbar = tqdm(total= f.final, desc = "Valid oligos") prepare(args.max_homopolymer) used_bc = dict() while f.good < f.final: print('f.good',f.good,'f.final',f.final) d = f.droplet() #print('droplet is',d) print('f.screen(d) is', f.screen(d)) if f.screen(d): print('in f.screen, d is',d) if not args.no_fasta: out.write(">packet {}_{}\n".format(f.good, d.degree)) out.write("{}\n".format(d.to_human_readable_DNA())) if d.seed in used_bc: logging.error("Seed %d has been seen before\nDone", d.seed) sys.exit(1) used_bc[d.seed] = 1 if (args.out != '-'): pbar.update() if (args.out != '-'): pbar.close() logging.info("Finished. Generated %d packets out of %d tries (%.3f)", f.good, f.tries, (f.good+0.0)/f.tries) out.close()
def read_pat(filename, elemsep, linesep, pat): filenames = glob.glob(filename + pat) print(filenames) data = [ np.array(pp.read_file(filename, elemsep, linesep, "all")) for filename in filenames ] return data
def main(args): filename = args.filename datatype = args.datatype if datatype == "SEQUENTIAL": #pattern = 'PL1331LAGLX6PH.csv' pattern = args.pattern filenames = glob.glob(filename + pattern) #print(filename+pattern) #print(filenames) data = [ pp.read_file(filename, elemsep=args.elemsep, linesep=args.linesep, readlines=args.readlines, mapping=lambda x: x) for filename in filenames ] for dat in data: dat.remove([]) name_location = 1 stream_count_location = 3 for dat, filename in zip(data, filenames): try: stream_stats = np.array( [int(row[stream_count_location]) for row in dat]) except ValueError: print(filename) stream_stats = [0] name = dat[0][name_location] if np.max(stream_stats) > 3e+6: plt.figure() plt.title(name) plt.axis([0, 40, 0, 1e+7]) plt.plot(stream_stats) plt.show() #numeric_per_row = [pp.count_numeric(row) for row in data[0]] #print("Max numeric: " + str(max(numeric_per_row))) #print("Argmax numeric: " + str(np.argmax(numeric_per_row))) elif datatype == "INSTANCE": pattern = '[a-z0-9-]*.csv' #pattern = args.pattern #serial_number = "MJ0351YNG9Z0XA" location = 4 melt_instance(args, filename, pattern, location)
for ngramRange in [(1, 1), (1, 2), (1, 3)]: for tfidfFlags in [(False, False, False), (True, False, False), (False, False, True)]: parametersList.append( Parameters(lowerCaseFlag, removeStopWordsFlag, stemFlag, testSize, maxFeatures, ngramRange, tfidfFlags)) original_stdout = sys.stdout # Save a reference to the original standard output print("Using test param[0]") # Find optimal params. fileData = preprocessing.read_file("../input-functional.txt") Corpus, X, names = getInfoFromParameters(fileData, parametersList[0]) # Encoder = LabelEncoder() # Y = Encoder.fit_transform(Corpus["Class"]) Y = Corpus["Class"] TrainX, TestX, TrainY, TestY = model_selection.train_test_split( X, Y, test_size=parametersList[0].testSize) # Some prints. # print("Corpus[Comment][1]: ",Corpus["Comment"][1]) # print("Corpus[Class][1]: ",Corpus["Class"][1]) # print("X[1]: ",X[1]) # print("Y[1]: ",Y[1]) # print("names: ",names) # print("Corpus[Class]: ",Corpus["Class"])
def main(): args = read_args() logging.info("Reading the file. This may take a few mintues") f_in, file_size = read_file(args.file_in, args.size) comp = None if args.composite_DNA is not None: # alphabet is a dict of int->letter including the std 0->A,1->C,2->G,3->T # the composite alphabet file only contains an ordered list of the *additional* letters alphabet = read_composite_alphabet(args.composite_DNA[0]) BC_bases = int(args.composite_DNA[1]) # TODO - set max binary block limit somehow, get oligo length from somewhere if args.composite_encoder is not None: composite_encoder = create_composite_encoder(alphabet, int(args.composite_encoder[0]), int(args.composite_encoder[1])) else: composite_encoder = create_optimal_composite_encoder(alphabet, 10, 136) comp = {'alphabet' : alphabet, 'BC_bases': BC_bases,'encoder':composite_encoder} f = DNAFountain(file_in = f_in, file_size = file_size, chunk_size = args.size , rs = args.rs, max_homopolymer = args.max_homopolymer, gc = args.gc, delta = args.delta, c_dist = args.c_dist, np = args.rand_numpy, alpha = args.alpha, stop = args.stop, comp = comp, maxseed = args.maxseed) logging.info("Upper bounds on packets for decoding is %d (x%f) with %f probability\n", int(json.loads(f.PRNG.debug())['K_prime']), json.loads(f.PRNG.debug())['Z'], json.loads(f.PRNG.debug())['delta']) if (args.out == '-'): out = sys.stdout else: out = open (args.out, 'w') pbar = tqdm(total= f.final, desc = "Valid oligos") prepare(args.max_homopolymer) used_bc = dict() while f.good < f.final: d = f.droplet() if f.screen(d): if not args.no_fasta: out.write(">packet {}_{}\n".format(f.good, d.degree)) out.write("{}\n".format(d.to_human_readable_DNA(comp = f.comp))) if d.seed in used_bc: logging.error("Seed %d has been seen before\nDone", d.seed) sys.exit(1) used_bc[d.seed] = 1 if (args.out != '-'): pbar.update() if (args.out != '-'): pbar.close() logging.info("Finished. Generated %d packets out of %d tries (%.3f)", f.good, f.tries, (f.good+0.0)/f.tries) out.close()
def main(): args = read_args() if args.debug_barcodes: valid_barcodes = load_barcodes(args) comp = None if args.composite_DNA is not None: # alphabet is a dict of int->letter including the std 0->A,1->C,2->G,3->T # the composite alphabet file only contains an ordered list of the *additional* letters alphabet = read_composite_alphabet(args.composite_DNA[0]) BC_bases = int(args.composite_DNA[1]) # TODO - set max binary block limit somehow, get oligo length from somewhere if args.composite_encoder is not None: composite_encoder = create_composite_encoder( alphabet, int(args.composite_encoder[0]), int(args.composite_encoder[1])) else: composite_encoder = create_optimal_composite_encoder( alphabet, 10, 136) comp = { 'alphabet': alphabet, 'BC_bases': BC_bases, 'encoder': composite_encoder } truth = None if args.truth is not None: truth, file_size = read_file(args.truth, args.size) g = Glass(args.chunk_num, header_size=args.header_size, rs=args.rs, c_dist=args.c_dist, delta=args.delta, flag_correct=not (args.no_correction), gc=args.gc, max_homopolymer=args.max_homopolymer, max_hamming=args.max_hamming, decode=not (args.mock), chunk_size=args.size, np=args.rand_numpy, truth=truth, out=args.out, comp=comp) line = 0 errors = 0 seen_seeds = defaultdict(int) # pbar = tqdm(total= args.chunk_num, desc = "Valid oligos") if args.file_in == '-': f = sys.stdin else: try: f = open(args.file_in, 'r') except: logging.error("%s file not found", args.text_file) sys.exit(0) aggressive = None if args.aggressive: aggressive = Aggressive(g=g, file_in=f, times=args.aggressive) ######## Main loop while True: try: dna = f.readline().rstrip('\n') except: logging.info("Finished reading input file!") break if len(dna) == 0: logging.info("Finished reading input file!") break if (args.fasta and re.search(r"^>", dna)): continue coverage = 0 # when the file is in the format of coverage \t DNA if (len(dna.split()) == 2): coverage, dna = dna.split() ####Aggresive mode if aggressive is not None and aggressive.turn_on( int(coverage), seen_seeds): best_file, value = aggressive.start() if best_file is not None: copyfile(best_file, args.out) logging.info("Done!") else: logging.error("Could not decode all file...") sys.exit(1) ### End of aggressive mode if 'N' in dna and comp is None: continue line += 1 seed, data = g.add_dna(dna) if seed == -1: # reed-solomon error! errors += 1 else: # pbar.update() if args.debug_barcodes: if not dna in valid_barcodes: logging.error( "Seed or data %d in line %d are not valid:%s", seed, line, dna) else: seen_seeds[dna] += 1 else: seen_seeds[seed] += 1 if line % 10000 == 0: logging.info( "After reading %d lines, %d chunks are done. So far: %d rejections (%f) %d barcodes", line, g.chunksDone(), errors, errors / (line + 0.0), g.len_seen_seed()) pass if line == args.max_line: logging.info("Finished reading maximal number of lines") break if g.isDone(): logging.info( "After reading %d lines, %d chunks are done. So far: %d rejections (%f) %d barcodes", line, g.chunksDone(), errors, errors / (line + 0.0), g.len_seen_seed()) logging.info("Done!") break if not g.isDone(): logging.error("Could not decode all file...") sys.exit(1) outstring = g.getString() f = open(args.out, 'wb') f.write(outstring) f.close() logging.info("MD5 is %s", md5.new(outstring).hexdigest()) json.dump(seen_seeds, open("seen_barocdes.json", 'w'), sort_keys=True, indent=4)
files = [ open("datasets/en_ewt-ud-{}.conllu".format(glob.train_mode), 'r'), open("datasets/fr_gsd-ud-{}.conllu".format(glob.train_mode), 'r'), open("datasets/es_gsd-ud-{}.conllu".format(glob.train_mode), 'r') ] files_test = [ open("datasets/en_ewt-ud-{}.conllu".format(glob.test_mode), 'r'), open("datasets/fr_gsd-ud-{}.conllu".format(glob.test_mode), 'r'), open("datasets/es_gsd-ud-{}.conllu".format(glob.test_mode), 'r') ] sentences = [] sentences_test = [] for f, f_test in zip(files, files_test): sentences += read_file(f) sentences_test += read_file(f_test) else: raw_file = open("datasets/fr_gsd-ud-{}.conllu".format(glob.train_mode), 'r') raw_file_test = open("datasets/fr_gsd-ud-{}.conllu".format(glob.test_mode), 'r') sentences = read_file(raw_file) sentences_test = read_file(raw_file_test) dico = build_dictionary(sentences) """___________________________________ INITIALIZING MODEL _______________________________________""" Net = XLMEmbMultiLayerBiLSTMPosTagger(dico)
def run(): config = Config() save_path = "trained_model/saved_model" x_train_path = 'data/xtrain.txt' y_train_path = 'data/ytrain.txt' x_idx = prep.Indexer() X = prep.read_file(x_train_path, raw=True) y = prep.read_file(y_train_path, label=True) t = CountVectorizer(analyzer='char', ngram_range=(config.ngram_min, config.ngram_max)) X = np.array( pad_sequences( x_idx.transform(t.inverse_transform(t.fit_transform(X)), matrix=True), config.maxlen)) x_train, x_test, y_train, y_test = train_test_split( X, y, test_size=config.test_size, shuffle=config.shuffle) del X, y # Generate batches train_batches = prep.generate_instances(data=x_train, labels_data=y_train, n_word=x_idx.max_number() + 1, n_label=config.label_size, max_timesteps=config.max_timesteps, batch_size=config.batch_size) validation_batches = prep.generate_instances( data=x_test, labels_data=y_test, n_word=x_idx.max_number() + 1, n_label=config.label_size, max_timesteps=config.max_timesteps, batch_size=config.batch_size) # Train the model train.train(config, train_batches, validation_batches, x_idx.max_number() + 1, save_path, from_saved=True) # Final Validation prediction_batches = prep.generate_instances( data=x_test, labels_data=None, n_word=x_idx.max_number() + 1, n_label=config.label_size, max_timesteps=config.max_timesteps, batch_size=config.batch_size) # Predict the model predicted_labels = predict.predict(config, prediction_batches, x_idx.max_number() + 1, save_path) report = classification_report(y_test[:len(predicted_labels)], predicted_labels) print(report) # Final output x_test_path = 'data/xtest.txt' X = prep.read_file(x_test_path, raw=True) t = CountVectorizer(analyzer='char', ngram_range=(config.ngram_min, config.ngram_max)) X = np.array( pad_sequences( x_idx.transform(t.inverse_transform(t.fit_transform(X)), matrix=True, add_if_new=False), config.maxlen)) prediction_batches = prep.generate_instances( data=X, labels_data=None, n_word=x_idx.max_number() + 1, n_label=config.label_size, max_timesteps=config.max_timesteps, batch_size=config.batch_size) predicted_labels = predict.predict(config, prediction_batches, x_idx.max_number() + 1, save_path, write_to_file=True)
#Main should just call the other files import support_vector_machine as svm import knn as knn import preprocessing as preprocessing import CSV_creator as csv_maker from pylab import * print('starting...') file_path = csv_maker.read() data_frame = preprocessing.read_file(file_path) data_frame = preprocessing.clean(data_frame) score_averages = svm.get_plot_feature_scores(data_frame) plt.plot(score_averages) plt.ylabel("score averages") plt.xlabel("number of features") plt.show() score_averages = knn.get_plot_feature_scores(data_frame) plt.plot(score_averages) plt.ylabel("score averages") plt.xlabel("number of features") plt.show()
import preprocessing import n_gram import another_method import OWLWriter if __name__ == "__main__": wrdlst = preprocessing.get_wordlist() preprocessing.read_file(wrdlst) raw_data = preprocessing.removestopwords() unigram = n_gram.uni_gram(raw_data) bigram = n_gram.bi_gram(raw_data) OWLWriter.generate_owl_file(bigram) another_method.manual(bigram)
def test_incre(data_file, link_file, concept_file, file_type, incre_course_num, incre_concept_num, undirect=False, save=True, update_A1=False): from preprocessing import generate_triple, generate_trn, row_normlize, read_file X, links, concept = read_file(data_file, link_file, concept_file=concept_file, file_type=file_type) X = X.todense() n_course, n_concept = X.shape[0], X.shape[1] trn = generate_trn(links, n_course, undirect=undirect) tripple = generate_triple(trn) split_tripple_list = split_tripple( tripple, range(X.shape[0] - incre_course_num, X.shape[0])) # 找到一个只有最后一门课才有的词 X = row_normlize(X) # 先训不带增量最后一行的 # 用全量数据训练模型 import model A0, F0, st = model.cgl_rank(X, tripple, lamb=0.01, eta=1, tolerence=1, silence=False) print('finish training whole A\n\n') # 用增量数据训练模型 T = split_tripple_list[0] A, F, st = model.cgl_rank(X[:-incre_course_num, :-incre_concept_num], T, lamb=0.01, eta=1, tolerence=1, silence=False) A = gene_incre_matrix(A, incre_concept_num) print('\n\n\n') A1, F1, st = incre_cgl_rank_new( X, (n_course - incre_course_num, n_concept - incre_concept_num), tripple, split_tripple_list, A, eta=5, lamb=0.01, tolerrence=1, update_A1=update_A1) file_prefix = 'undirect' if undirect else 'direct' file_prefix += '_update_A1' if update_A1 else 'noupdate_A1' if save: np.savetxt('result/ruc_A_whole_with_essay_{}.txt'.format(file_prefix), A0) np.savetxt('result/ruc_A_incre_with_essay_{}.txt'.format(file_prefix), A1) np.savetxt('result/ruc_F_whole_with_essay_{}.txt'.format(file_prefix), F0) np.savetxt('result/ruc_F_incre_with_essay_{}.txt'.format(file_prefix), F1)
@application.route('/') def index(): return render_template('index.html') @application.route('/search/', methods=['POST']) def search(): query = request.form['query'] docs = get_top_k_docs(model, query, corpus, k=100) return jsonify(docs) @application.route('/save_relevance/', methods=['POST']) def save_relevance(): query = request.form['query'] doc_id = request.form['doc_id'] ip = request.form['ip'] is_rel = request.form['is_rel'] store_relevance_judgements(query, doc_id, ip, is_rel) return ('', 204) if __name__ == '__main__': print('Loading Corpus...') corpus, tokenized_corpus = read_file() model = BM25Plus(tokenized_corpus) print('Corpus Loaded!') application.run()
def main(args): filename = args.filename names = pp.just_the_names([filename]) print(names) if args.model == "ESN": lag = 1 elif args.model == "SVM": lag = 4 elif args.model == "MLP": lag = 5 global name name = names[0] data_filename = filename + ".data" event_filename = filename + ".events" data = pp.read_file(data_filename, elemsep=",", linesep="\n", readlines="all", mapping=lambda x: x) events = pp.read_file(event_filename, elemsep=",", linesep="\n", readlines="all", mapping=lambda x: x) #data.remove([]) data = list(filter(([]).__ne__, data)) if name == "CalIt2": data = [row[1:] for row in data] event_times = [ Time_interval(event[0], event[1], event[2]) for event in events ] gt = generate_gt(data, event_times) gt = gt[lag:] if name == "Dodgers": intensity = np.array([int(row[2]) for row in data]) #data = [row for row,ints in zip(data,intensity) if ints != -1] T = 288 num_subspace = 2 #intensity = intensity[np.where(intensity!=-1)[0]] raw_intensity = intensity.reshape([len(intensity), 1]) intensity, gt = preprocess(intensity, gt, T, num_subspace, lag) elif name == "CalIt2": for row in data: if len(row) < 3: print(row) intensity = np.array([int(row[2]) for row in data]) gt = gt[::2] #data = [row[1:] for row,ints in zip(data,intensity) if 1 ] #ints != -1] T = 48 num_subspace = 3 raw_intensity = intensity.reshape([int(len(intensity) / 2), 2]) intensity_out, gt = preprocess(raw_intensity[:, 0], gt, T, num_subspace, lag) intensity_in, __ = preprocess(raw_intensity[:, 1], gt, T, num_subspace, lag) intensity = np.concatenate([intensity_in, intensity_out], axis=1) ''' print(gt.shape) plt.plot(intensity[:,-1]) plt.plot(gt) plt.show() ''' explanations = ["intensity"] if 0: #__name__ == '__main__': plt.figure() plt.plot(raw_intensity[:, 0] / 10, "b") plt.plot(intensity[:, 0], "g") plt.plot(gt, "r") plt.title("{0:s}".format(names[0])) plt.xlabel("Sample no. (time)") if name == "CalIt2": plt.legend([ "Raw intensity in flow", "Processed intensity", "Event in building" ]) plt.ylabel("No. people/ 3 mins") else: plt.legend([ "Raw intensity flow", "Processed intensity", "Event at stadium" ]) plt.ylabel("No. cars/ 5 secs") if name == "CalIt2": plt.figure() plt.plot(raw_intensity[:, 1] / 10, "k") plt.plot(intensity[:, 1], "g") plt.plot(gt, "r") plt.legend([ "Raw intensity out flow", "Processed intensity", "Event at stadium" ]) plt.xlabel("Sample no. (time)") plt.ylabel("No. people/ 3 mins") plt.show() return [intensity], [gt], explanations, names
def main(args): filename = args.filename datatype = args.datatype if datatype == "SEQUENTIAL": #pattern = 'PL1331LAGLX6PH.csv' pattern = args.pattern filenames = glob.glob(filename + pattern) #print(filename+pattern) #print(filenames) data = [ pp.read_file(filename, elemsep=args.elemsep, linesep=args.linesep, readlines=args.readlines) for filename in filenames ] for dat in data: dat.remove([]) data = [np.array(dat) for dat in data] #numeric_per_row = [pp.count_numeric(row) for row in data[0]] #print("Max numeric: " + str(max(numeric_per_row))) #print("Argmax numeric: " + str(np.argmax(numeric_per_row))) if __name__ == '__main__': names = just_the_names(filenames) # Data selection data = [dat[:, normalized_idx] for dat in data] #dead_rows(data,filenames) qualified = BB_SMART_order #data,__ = remove_caught_failures(data,names,qualified) #print(max_length) #return #data = filter_wrt(data,0,2) #while True: # x = input('Which feature do you want to look at? ') # x = int(x) data = pp.normalize_all(data, leave_zero=True) #plot_featurewise(data,names) plot_unitwise(data, names) else: names = just_the_names(filenames) # Data selection data = [dat[:, normalized_idx] for dat in data] dead_rows(data, names) idxs = set(all_smart_except([194, 5, 187, 188, 197, 198])) print(idxs) #print(set(pp.numeric_idxs(data))) idxs = set.intersection(idxs, set(pp.numeric_idxs(data))) print(idxs) #print(pp.changing_idxs(data)) idxs = set.intersection(idxs, set(pp.changing_idxs(data))) print(idxs) print("Qualified indexes: " + str(sorted(idxs))) print("Qualified indexes: " + str(sorted([BB_SMART_order[i] for i in idxs]))) qualified = sorted([BB_SMART_order[i] for i in idxs]) data = [dat[:, sorted(idxs)] for dat in data] keys = [smart_expl(i)[0] for i in sorted(idxs)] explanations = [smart_expl(i)[1] for i in sorted(idxs)] print("before removing missing, small, and predicted failures: " + str(len(data))) __, no_missing = pp.remove_instances_with_missing(data) min_sample_time = args.settings["failure_horizon"] + 70 __, no_small = pp.remove_small_samples(data, min_sample_time) #print(no_small) if 1: __, no_predicted_failures = remove_caught_failures( data, names, qualified) else: no_predicted_failures = list(range(len(data))) cleared_idxs = set.intersection(set(no_missing), set(no_small), set(no_predicted_failures)) cleared_idxs = list(cleared_idxs) #print(cleared_idxs) data = [data[idx] for idx in sorted(cleared_idxs)] #print([len(dat) for dat in data]) names = [names[idx] for idx in sorted(cleared_idxs)] print("after removing missing, small, and predicted failures: " + str(len(data))) # Mathematical preprocessing # add 0 to beginning #num_features = len(idxs) #data = [np.concatenate([np.zeros([1,num_features]),dat],axis=0) for dat in data] extended_features = False if extended_features: exta = pp.differentiate(data) #data = pp.smooth(data,5) exta = pp.filter(exta, np.array([1]), np.array([1, -0.8])) data = [ dat[1:, :] for dat in data ] # have to take a away first so that lengths are correct #print(exta[0].shape) #print(data[0].shape) data = [ np.concatenate([dat, ext], axis=1) for dat, ext in zip(data, exta) ] expl_ext = [expl + " (modified)" for expl in explanations] explanations = explanations + expl_ext else: pass #data = pp.differentiate(data) #data = pp.smooth(data,10) #data = pp.filter(data,np.array([1]),np.array([1,-0.8])) data = pp.normalize_all(data, leave_zero=True, mean_def=0) #print(explanations) #print(keys) print("Explanations " + " ".join([ "{0:s}: {1:s}".format(str(key), str(explanation)) for key, explanation in zip(keys, explanations) ])) if args.test_type == "PREDICTION": gt = [] elif args.test_type in ["CLASSIFICATION", "REGRESSION"]: X = [] Y = [] failed = ["_fail" in name for name in names] for x, y in pp.impending_failure( data, failed, args.settings["failure_horizon"], "CLASSIFICATION"): X.append(x) Y.append(y) #order = np.random.choice(len(X),len(X),replace=False) #data = [X[ord_i] for ord_i in order] #gt = [Y[ord_i] for ord_i in order] #data = X gt = Y return data, gt, explanations, names elif datatype == "INSTANCE": pattern = '[0-9-]*.csv' #pattern = args.pattern #serial_number = "MJ0351YNG9Z0XA" serial_location = 1 model_location = 2 melt_instance(args, filename, pattern, serial_location, model_location) elif datatype == "POPULATION_STATISTICS": product_failures(args, filename)
def main(): args = read_args() if args.debug_barcodes: valid_barcodes = load_barcodes(args) truth = None if args.truth is not None: truth, file_size = read_file(args.truth, args.size) g = Glass(args.chunk_num, header_size=args.header_size, rs=args.rs, c_dist=args.c_dist, delta=args.delta, flag_correct=not (args.no_correction), gc=args.gc, max_homopolymer=args.max_homopolymer, max_hamming=args.max_hamming, decode=not (args.mock), exDNA=args.expand_nt, chunk_size=args.size, np=args.rand_numpy, truth=truth, out=args.out) line = 0 errors = 0 seen_seeds = defaultdict(int) #pbar = tqdm(total= args.chunk_num, desc = "Valid oligos") if args.file_in == '-': f = sys.stdin else: try: f = open(args.file_in, 'r') except: logging.error("%s file not found", args.text_file) sys.exit(0) aggressive = None if args.aggressive: aggressive = Aggressive(g=g, file_in=f, times=args.aggressive) ######## Main loop while True: try: dna = f.readline().rstrip('\n') except: logging.info("Finished reading input file!") break if len(dna) == 0: logging.info("Finished reading input file!") break if (args.fasta and re.search(r"^>", dna)): continue coverage = 0 #when the file is in the format of coverage \t DNA if (len(dna.split()) == 2): coverage, dna = dna.split() ####Aggresive mode if aggressive is not None and aggressive.turn_on( int(coverage), seen_seeds): best_file, value = aggressive.start() if best_file is not None: copyfile(best_file, args.out) logging.info("Done!") else: logging.error("Could not decode all file...") sys.exit(1) ### End of aggressive mode if 'N' in dna: continue line += 1 seed, data = g.add_dna(dna) if seed == -1: #reed-solomon error! errors += 1 else: #pbar.update() if args.debug_barcodes: if not dna in valid_barcodes: logging.error( "Seed or data %d in line %d are not valid:%s", seed, line, dna) else: seen_seeds[dna] += 1 else: seen_seeds[seed] += 1 if line % 1000 == 0: logging.info( "After reading %d lines, %d chunks are done. So far: %d rejections (%f) %d barcodes", line, g.chunksDone(), errors, errors / (line + 0.0), g.len_seen_seed()) pass if line == args.max_line: logging.info("Finished reading maximal number of lines") break if g.isDone(): logging.info( "After reading %d lines, %d chunks are done. So far: %d rejections (%f) %d barcodes", line, g.chunksDone(), errors, errors / (line + 0.0), g.len_seen_seed()) logging.info("Done!") break if not g.isDone(): logging.error("Could not decode all file...") sys.exit(1) outstring = g.getString() f = open(args.out, 'wb') f.write(outstring) f.close() logging.info("MD5 is %s", md5.new(outstring).hexdigest()) json.dump(seen_seeds, open("seen_barocdes.json", 'w'), sort_keys=True, indent=4)
stemFlag, maxFeatures, ngramRange, tfidfFlags, alpha_value)) cnt = 0 # Go through all of the input files and configurations and export the results to a .csv file. for input_file, output_file_path, singleFunctionalClass in [ ("../input.txt", "output/outputNBdirectAlphaAll.csv", False), ("../input-functional.txt", "output/outputNBdirectAlphaFunctionalAll.csv", True) ]: with open(output_file_path, 'w') as output: print(utilities.getHeader(singleFunctionalClass), file=output) output.flush() fileData = preprocessing.read_file(input_file) for parameters in parametersList: print(cnt, ' / ', len(parametersList)) # datetime object containing current date and time print(">>>>>>>>>>>>>>>>>>>>> get info start. now =", datetime.now()) classifier = MultinomialNB(alpha=parameters.alphaNaiveBayes) Corpus, pipeline = utilities.getInfoFromParameters( fileData, parameters, classifier) outer_cv = StratifiedKFold(n_splits=10, shuffle=True, random_state=42)
def main(args): filename = args.filename datatype = args.datatype if datatype == "SEQUENTIAL": #pattern = 'PL1331LAGLX6PH.csv' pattern = args.pattern filenames = glob.glob(filename + pattern) filenames = sorted(filenames, reverse=True) names = just_the_names(filenames) #print(filename+pattern) #print(sorted(filenames,reverse=False)) print(names) feature_idxs = [2, 3, 4, 5, 6] gt_idx = [7] data = [ pp.read_file(filename, elemsep=args.elemsep, linesep=args.linesep, readlines=args.readlines) for filename in sorted(filenames, reverse=True) ] for dat in data: dat.remove([]) data = [np.array(dat) for dat in data] data = [dat[1:, :] for dat in data] ''' for feat in feature_idxs: plt.figure() for dat in data: plt.plot(dat[:,feat]) plt.show() ''' ''' data = pp.normalize_all(data,leave_zero=True) for dat in data: plt.figure() for feat in feature_idxs: plt.plot(dat[:,feat]) gt_array = dat[:,gt_idx] gt_array = gt_array - min(gt_array) gt_array = gt_array/max(gt_array) plt.plot(gt_array) plt.title("Occupancy and predictors") plt.xlabel("Sample no. (time)") plt.ylabel("Value (normalized)") plt.legend(explanations+["occupancy"]) plt.show() ''' gt = [dat[:, gt_idx] for dat in data] data = [dat[:, feature_idxs] for dat in data] data = pp.normalize_all(data, leave_zero=True) return data, gt, explanations, names
def run(): config = Config() # Load configs save_path = 'keras_models/keras_model' # Model save path x_train_path = 'data/xtrain.txt' x_test_path = 'data/xtest.txt' y_train_path = 'data/ytrain.txt' x_idx = prep.Indexer() X = prep.read_file(x_train_path, raw=True) y = prep.read_file(y_train_path, label=True) t = CountVectorizer(analyzer='char', ngram_range=(config.ngram, config.ngram)) t.fit(X) X = prep.transform(X, t, x_idx) X = np.array(pad_sequences(X, config.maxlen)) x_train, x_test, y_train, y_test = train_test_split( X, y, test_size=config.test_size, shuffle=config.shuffle) ############################################# # Train model print("BEGINNING TRAINING") tsv_logger = CSVLogger('training-data.tsv', append=True, separator='\t') m = model_fn(config=config, input_length=x_idx.max_number() + 1) # m = load_model(save_path) m.fit(x_train, y_train, epochs=config.n_epochs, batch_size=config.batch_size, verbose=1, shuffle=True, callbacks=[tsv_logger], validation_data=(x_test, y_test)) m.save(save_path) print("MODEL REPORT") score, acc = m.evaluate(x_test, y_test) print("\nSCORE: ", score) print("ACCURACY: ", acc) pred = [np.argmax(label) for label in m.predict(x_test)] report = classification_report(y_test, pred) print(report) ############################################### # Predict and write labels for xtest.txt print("PREDICTION") X = prep.read_file(x_test_path, raw=True) X = prep.transform(X, t, x_idx, add_if_new=False) X = np.array(pad_sequences(X, config.maxlen)) pred = [np.argmax(label) for label in m.predict(X)] with open("".join(["keras_prediction/ytest.txt"]), "w+", encoding="utf-8") as rec: for label in pred: rec.write("%s\n" % label) rec.close()
def main(args): filename = args.filename #datatype = args.datatype #if datatype == "SEQUENTIAL": #pattern = 'PL1331LAGLX6PH.csv' pattern = args.pattern filenames = glob.glob(filename + pattern) filenames = sorted(filenames, reverse=True) names = just_the_names(filenames) #print(filename+pattern) #print(sorted(filenames,reverse=False)) print(names) feature_idxs = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13] #feature_idxs = [0,2,11,13] gt_idx = [14] data = [ pp.read_file(filename, elemsep=args.elemsep, linesep=args.linesep, readlines=args.readlines) for filename in sorted(filenames, reverse=True) ] for dat in data: dat.remove([]) data = [np.array(dat) for dat in data] #data = [dat[1:,:] for dat in data] data = [neutralize_outliers(dat) for dat in data] ''' gt = [dat[:,gt_idx] for dat in data] data = [dat[:,feature_idxs] for dat in data] data = pp.normalize_all(data,leave_zero=True) x = np.linspace(0,117,14980) print(x.shape) for dat in data: print(dat.shape) plt.figure() for feat in feature_idxs: plt.plot(x,dat[:,feat]) plt.plot(x,gt[0]*5,'b') plt.xlabel('time / s') plt.ylabel('Normalized EEG value') plt.title('EEG Eye Features') plt.legend(explanations+["Ground Truth"]) plt.show() ''' ''' data = pp.normalize_all(data,leave_zero=True) for dat in data: plt.figure() for feat in feature_idxs: plt.plot(dat[:,feat]) gt_array = dat[:,gt_idx] gt_array = gt_array - min(gt_array) gt_array = gt_array/max(gt_array) plt.plot(gt_array) plt.title("Occupancy and predictors") plt.xlabel("Sample no. (time)") plt.ylabel("Value (normalized)") plt.legend(explanations+["occupancy"]) plt.show() ''' gt = [dat[:, gt_idx] for dat in data] data = [dat[:, feature_idxs] for dat in data] data = pp.normalize_all(data, leave_zero=True) return data, gt, explanations, names