def run_pipeline(filename): """filename: file path for the otu table you want to run the pipeline on run the pipeline using the parameters from the pipeline.py file. See documentation on pipeline.py if confused for parameters""" t_start = time.perf_counter() pipeline.main(False, filename, '', 'graph_centrality', 'add_one', 3, 'all', 'all', None, None, None, 'degree', 0.2, 'MIC', True, 'both', 'kl_divergence', False, False, False, '', None) t_end = time.perf_counter() print('runtime = ', t_end - t_start) return
def upload(): print("hi") if request.method == 'POST': f = request.files['file'] f.save('filename.jpg') x = pipeline.main(True) print(x) var = '/static/' + x['id'] + '.jpg' return render_template("other.html", variable=var, name=x['name'])
def create_confusion_matrix(): ''' creates a confusion matrix ''' X_text, X_num, y = main() X_text_train, X_text_test, X_num_train, X_num_test, y_train, y_test = train_test_split( X_text, X_num, y) model = MyModel() model.fit(X_text_train, X_num_train, y_train) predictions = model.predict(X_text_test, X_num_test) return confusion_matrix(y_test, predictions)
def call_pipeline(met, input_file, cent, corr, conditioning, total, incr, weight, c_prop, thresh, min, plot_metric, create_graph, plot_pca, name_file, run_number): """ def main(ab_comp, infile1, infile2, metric, c_type, min_count, total_select, iteration_select, pca_components, smooth_type, window_size, centrality_type, keep_threshold, correlation, weighted, corr_prop, evaluation_type, plot_metric, create_graph, plot_pca, naming_file):""" #try: proc_id = os.getpid() t_start = time.perf_counter() pipeline.main(False, input_file, '', met, conditioning, min, total, incr, None, None, None, cent, thresh, corr, weight, c_prop, 'kl_divergence', plot_metric, create_graph, plot_pca, name_file, run_number) t_end = time.perf_counter() print('runtime = ', t_end - t_start) #except: # pass return
def main(): prList = [] #haven't yet decided on the starting and stopping points and increments #probably gonna read a few papers before doing this super long process #n_segments = range(start, stop, seg) #compactness = range(start,stop,seg) #threshold = range(start, stop, seg) #nri = range(value) for a in n_segments: for b in compactness: for c in threshold: for d in nri: #returns tuples (precision, recall) and (n_segments, compactness, threshold, nri) pr, parameters = pipeline.main(700, .29, .02, 1) prList.append(pr, parameters) npArray = np.array(prList) #finding the parameters for the max precision+recall #npArray structure is like this [ [(Precision, Recall),(nseg, compact, thresh, nri)] # [(Precision, Recall),(nseg, compact, thresh, nri)] # .................................................. # ....[(Precision, Recall),(nseg, compact, thresh, nri)] ] allPRValues = npArray[:,0,:] #gets all the (P,R) pairs in npArray sumPRValues = np.add.reduce(allPRValues, 1) #sums the precision and recall pairs index = np.argmax(sumPRValues) #finds the index of the max precision+recall value maxPR = npArray[index][0] maxParameters = npArray[index][1] #parameters that correspond with the max P+R value print(maxParameters) #these will be your ideal parameters #Plot Precision-Recall curve plt.clf() plt.plot(npArray[:,0,0], npArray[:,0,1], label="Precision-Recall curve") plt.xlabel("Precision") plt.ylabel("Recall") plt.xlim([0.0, 1.05]) plt.ylim([0.0, 1.05]) plt.title("Precision-Recall Curve") plt.legend(loc="lower right") plt.show()
def run_job(job_id): job_dir = output_dir + "job_" + job_id + "/" graph_json = "graph_elements_" + job_id + ".json" table_file = "interaction_table_" + job_id + ".json" stats_file = "req_parameters_" + job_id + ".json" if not (os.path.isfile(job_dir + graph_json) and os.path.isfile(job_dir + table_file) and os.path.isfile(job_dir + stats_file)): with open(job_dir + "input_" + job_id + ".json", "r") as f: d = json.load(f) # try: error = pipeline.main( INPUT_1=d["prots_input"], INPUT_2=d["muts_input"], SP=d["sps"], ADDITIONAL_INTERACTORS=d["add_interactors"], MAIN_OUTPUT_DIR=output_dir, CUSTOM_ID=job_id, BLASTDB_DIR="/net/home.isilon/ds-russell/blastdb/", CLIENT=client, MAKE_NETWORK=d["make_graph"], HIDE_NO_INT=d["hide"], TABLE_FORMAT="json") # except: # print "except error" # return render_template(error_template) if error: return render_template(error_template) # Read stats file (table_columns, not_found, no_int_prots) = get_stats_for_charts(job_dir + stats_file) return render_template( results_template, graph_json="jobs/" + "job_" + job_id + "/" + graph_json, ints_json="jobs/" + "job_" + job_id + "/" + table_file, stats_json="jobs/" + "job_" + job_id + "/" + stats_file, not_found=not_found, no_int_prots=no_int_prots, table_columns=table_columns)
def test_pipeline_args(): pipeline.main() pipeline.main([""]) pipeline.main(["-i"]) pipeline.main(["-i", "src/tests/pipeline/test_data"])
ngood[species] += 1 chimericity = float(max(ngood.values())) / sum(ngood.values()) outf.write("%s\t%s\n" % contig, (str(chimericity))) outf.close() @follows(taxonomy, plotCoverageOverGenomes, buildChimerasBasedOnReads) def full(): pass #################### # report building #################### @follows(mkdir("report")) def build_report(): '''build report from scratch.''' E.info("starting documentation build process from scratch") P.run_report(clean=True) @follows(mkdir("report")) def update_report(): '''update report.''' E.info("updating documentation") P.run_report(clean=False) if __name__ == "__main__": sys.exit(P.main(sys.argv))
import pipeline import time input_file = 'bromeA_all_Oct2017.csv' centrality = ['betweenness'] corr_type = ['spearman'] smoothing_type = ['hellinger'] total = '100' increment = ['10', '20', 'all'] weighted = [True] corr_prop = ['both'] threshold = ['0.3', '0.5'] selected = increment for weight in weighted: for corr in corr_type: for c_prop in corr_prop: for cent in centrality: for incr in increment: try: t_start = time.perf_counter() pipeline.main(False, input_file, '', 'graph_centrality', 'add_one', 3, total, incr, None, None, None, cent, threshold, corr, weight, c_prop, 'kl_divergence') t_end = time.perf_counter() print('runtime = ', t_end - t_start) except: pass
#!/usr/bin/env python3 """ Module description """ from pipeline import main if __name__ == '__main__': dataset_ids = [1, 2, 3, 4] c_method = 'agglomerative' n_clusters = 2 distance_threshold = 2 encoding = 'use' # for dataset 1, agglomerative + w dist_threshold 2 # for dataset 2, agglomerative + w dist_threshold 2 # for dataset 3, cosine # for dataset 4, birch with threshold 0.2 main(dataset_ids, clustering_method=c_method, encoding=encoding, cluster_num=n_clusters, distance_threshold=distance_threshold, verbose=True, evaluate=True, optimize_method=True, store=True)
if __name__ == '__main__': import pipeline import fmri import protein # DAX files pipeline.main("-D", "pipeline.xml") fmri.main("-D", "fmri.xml") protein.main("-D", "protein.xml") # DOT files pipeline.main("-d", "pipeline.dot") fmri.main("-d", "fmri.dot") protein.main("-d", "protein.dot")
distance_thresholds = [0.5, 0.8, 1, 1.25, 1.5, 1.75, 2, 2.5] clustering_step = ['cosine', 'kmeans', 'jaccard', 'agglomerative', 'birch'] encodings = ['use'] scores_to_store = list() for c_method in clustering_step: if c_method == 'kmeans': for cluster_num in clusters_nums: for text_encoding in encodings: print('-' * 50) print("\nRunning {} for the clustering step.".format( c_method)) print("Running for cluster number: {}".format(cluster_num)) eval_scores = main(dataset_ids, evaluate=True, clustering_method=c_method, cluster_num=cluster_num, encoding=text_encoding) scores_to_store += eval_scores elif c_method == 'jaccard': print('-' * 50) print("\nRunning {} for the clustering step.".format(c_method)) eval_scores = main(dataset_ids, evaluate=True, clustering_method=c_method) scores_to_store += eval_scores elif c_method == 'agglomerative': for distance_threshold in distance_thresholds: for text_encoding in encodings:
lb.init_svc_lb() utils.task_record("init_service_lb") if utils.no_task_record("init_applications"): applications.init_app_list() utils.task_record("init_applications") if utils.no_task_record("init_application_detail"): applications.init_app_svc_detail() utils.task_record("init_application_detail") if utils.no_task_record("init_applications_service_lb"): lb.init_app_svc_lb() utils.task_record("init_applications_service_lb") if utils.no_task_record("init_cm"): cm.init_cm() utils.task_record("init_cm") if utils.no_task_record("create_cm"): cm.create_cm() utils.task_record("create_cm") # service trans services.main() applications.main() pipeline.main() if utils.no_task_record("update_jakiro_resource"): services.build_volume_sql() utils.task_record("update_jakiro_resource")
import find_deal_num import datetime while True: start_time = datetime.datetime.now() sources = ["dida", "dianping", "ftuan", "manzuo", "meituan", "nuomi", "wowo", "wuba"] for source in sources: print "start crawling deals from", source async_pipeline.main(source, None, None) find_deal_num.main(source, True) miner.main(source, True) print "start crawling deals from lashou" pipeline.main("lashou", None, None) find_deal_num.main("lashou", True) miner.main("lashou", True) end_time = datetime.datetime.now() duration = end_time - start_time print "=====================" print "The task takes", duration.seconds, "seconds to finish" print "=====================" fhandler = open("finish_log.txt", "a") fhandler.write(str(duration.seconds) + "\n") fhandler.close() if os.path.exists("main_term_sig"): break
self.rf_text.fit(X_text, y) self.rf_num.fit(X_num, y) def predict_proba(self, X_text, X_num): ''' calculate probability that item is fraud ''' X_t = self.vect.transform(X_text) pred_txt = self.rf_text.predict_proba(X_t)[:, 1] pred_num = self.rf_num.predict_proba(X_num)[:, 1] return np.mean([pred_txt, pred_num], axis=0) def predict(self, X_text, X_num): ''' predict true/false for item ''' proba = self.predict_proba(X_text, X_num) return (proba > THRESHOLD).astype(int) def predict(model, X_text, X_num): ''' return predictions for a new item''' prediction = model.predict(X_text, X_num) probability = model.predict_proba(X_text, X_num) return prediction, probability if __name__ == '__main__': ''' run model.py to generate a model to base predictions on''' X_text, X_num, y = main() model = MyModel() model.fit(X_text, X_num, y) with open('model.pkl', 'wb') as f: pickle.dump(model, f)
def create_dataset(source, target): files = combine_sources.main(source, target) new_files = {'all': files} pipeline.main(new_files, target)
def take(): x = pipeline.main(False) print(x) var = 'static/' + x['id'] + '.jpg' return render_template("other.html", variable=var, name=x['name'])
from pipeline import main main()