def run_pipeline(filename):
"""filename: file path for the otu table you want to run the pipeline on
run the pipeline using the parameters from the pipeline.py file. See documentation on pipeline.py if confused for parameters"""
t_start = time.perf_counter()
pipeline.main(False, filename, '', 'graph_centrality', 'add_one', 3, 'all',
'all', None, None, None, 'degree', 0.2, 'MIC', True, 'both',
'kl_divergence', False, False, False, '', None)
t_end = time.perf_counter()
print('runtime = ', t_end - t_start)
return
def upload():
print("hi")
if request.method == 'POST':
f = request.files['file']
f.save('filename.jpg')
x = pipeline.main(True)
print(x)
var = '/static/' + x['id'] + '.jpg'
return render_template("other.html", variable=var, name=x['name'])
def create_confusion_matrix():
''' creates a confusion matrix '''
X_text, X_num, y = main()
X_text_train, X_text_test, X_num_train, X_num_test, y_train, y_test = train_test_split(
X_text, X_num, y)
model = MyModel()
model.fit(X_text_train, X_num_train, y_train)
predictions = model.predict(X_text_test, X_num_test)
return confusion_matrix(y_test, predictions)
def call_pipeline(met, input_file, cent, corr, conditioning, total, incr,
weight, c_prop, thresh, min, plot_metric, create_graph,
plot_pca, name_file, run_number):
"""
def main(ab_comp, infile1, infile2, metric, c_type, min_count,
total_select, iteration_select, pca_components, smooth_type,
window_size, centrality_type, keep_threshold, correlation,
weighted, corr_prop, evaluation_type, plot_metric,
create_graph, plot_pca, naming_file):"""
#try:
proc_id = os.getpid()
t_start = time.perf_counter()
pipeline.main(False, input_file, '', met, conditioning, min, total, incr,
None, None, None, cent, thresh, corr, weight, c_prop,
'kl_divergence', plot_metric, create_graph, plot_pca,
name_file, run_number)
t_end = time.perf_counter()
print('runtime = ', t_end - t_start)
#except:
# pass
return
def main():
prList = []
#haven't yet decided on the starting and stopping points and increments
#probably gonna read a few papers before doing this super long process
#n_segments = range(start, stop, seg)
#compactness = range(start,stop,seg)
#threshold = range(start, stop, seg)
#nri = range(value)
for a in n_segments:
for b in compactness:
for c in threshold:
for d in nri:
#returns tuples (precision, recall) and (n_segments, compactness, threshold, nri)
pr, parameters = pipeline.main(700, .29, .02, 1)
prList.append(pr, parameters)
npArray = np.array(prList)
#finding the parameters for the max precision+recall
#npArray structure is like this [ [(Precision, Recall),(nseg, compact, thresh, nri)]
# [(Precision, Recall),(nseg, compact, thresh, nri)]
# ..................................................
# ....[(Precision, Recall),(nseg, compact, thresh, nri)] ]
allPRValues = npArray[:,0,:] #gets all the (P,R) pairs in npArray
sumPRValues = np.add.reduce(allPRValues, 1) #sums the precision and recall pairs
index = np.argmax(sumPRValues) #finds the index of the max precision+recall value
maxPR = npArray[index][0]
maxParameters = npArray[index][1] #parameters that correspond with the max P+R value
print(maxParameters) #these will be your ideal parameters
#Plot Precision-Recall curve
plt.clf()
plt.plot(npArray[:,0,0], npArray[:,0,1], label="Precision-Recall curve")
plt.xlabel("Precision")
plt.ylabel("Recall")
plt.xlim([0.0, 1.05])
plt.ylim([0.0, 1.05])
plt.title("Precision-Recall Curve")
plt.legend(loc="lower right")
plt.show()
def run_job(job_id):
job_dir = output_dir + "job_" + job_id + "/"
graph_json = "graph_elements_" + job_id + ".json"
table_file = "interaction_table_" + job_id + ".json"
stats_file = "req_parameters_" + job_id + ".json"
if not (os.path.isfile(job_dir + graph_json)
and os.path.isfile(job_dir + table_file)
and os.path.isfile(job_dir + stats_file)):
with open(job_dir + "input_" + job_id + ".json", "r") as f:
d = json.load(f)
# try:
error = pipeline.main(
INPUT_1=d["prots_input"],
INPUT_2=d["muts_input"],
SP=d["sps"],
ADDITIONAL_INTERACTORS=d["add_interactors"],
MAIN_OUTPUT_DIR=output_dir,
CUSTOM_ID=job_id,
BLASTDB_DIR="/net/home.isilon/ds-russell/blastdb/",
CLIENT=client,
MAKE_NETWORK=d["make_graph"],
HIDE_NO_INT=d["hide"],
TABLE_FORMAT="json")
# except:
# print "except error"
# return render_template(error_template)
if error:
return render_template(error_template)
# Read stats file
(table_columns, not_found,
no_int_prots) = get_stats_for_charts(job_dir + stats_file)
return render_template(
results_template,
graph_json="jobs/" + "job_" + job_id + "/" + graph_json,
ints_json="jobs/" + "job_" + job_id + "/" + table_file,
stats_json="jobs/" + "job_" + job_id + "/" + stats_file,
not_found=not_found,
no_int_prots=no_int_prots,
table_columns=table_columns)
def test_pipeline_args():
pipeline.main()
pipeline.main([""])
pipeline.main(["-i"])
pipeline.main(["-i", "src/tests/pipeline/test_data"])
ngood[species] += 1
chimericity = float(max(ngood.values())) / sum(ngood.values())
outf.write("%s\t%s\n" % contig, (str(chimericity)))
outf.close()
@follows(taxonomy, plotCoverageOverGenomes, buildChimerasBasedOnReads)
def full():
pass
####################
# report building
####################
@follows(mkdir("report"))
def build_report():
'''build report from scratch.'''
E.info("starting documentation build process from scratch")
P.run_report(clean=True)
@follows(mkdir("report"))
def update_report():
'''update report.'''
E.info("updating documentation")
P.run_report(clean=False)
if __name__ == "__main__":
sys.exit(P.main(sys.argv))
import pipeline
import time
input_file = 'bromeA_all_Oct2017.csv'
centrality = ['betweenness']
corr_type = ['spearman']
smoothing_type = ['hellinger']
total = '100'
increment = ['10', '20', 'all']
weighted = [True]
corr_prop = ['both']
threshold = ['0.3', '0.5']
selected = increment
for weight in weighted:
for corr in corr_type:
for c_prop in corr_prop:
for cent in centrality:
for incr in increment:
try:
t_start = time.perf_counter()
pipeline.main(False, input_file, '',
'graph_centrality', 'add_one', 3, total,
incr, None, None, None, cent, threshold,
corr, weight, c_prop, 'kl_divergence')
t_end = time.perf_counter()
print('runtime = ', t_end - t_start)
except:
pass
#!/usr/bin/env python3
"""
Module description
"""
from pipeline import main
if __name__ == '__main__':
dataset_ids = [1, 2, 3, 4]
c_method = 'agglomerative'
n_clusters = 2
distance_threshold = 2
encoding = 'use'
# for dataset 1, agglomerative + w dist_threshold 2
# for dataset 2, agglomerative + w dist_threshold 2
# for dataset 3, cosine
# for dataset 4, birch with threshold 0.2
main(dataset_ids,
clustering_method=c_method,
encoding=encoding,
cluster_num=n_clusters,
distance_threshold=distance_threshold,
verbose=True,
evaluate=True,
optimize_method=True,
store=True)
if __name__ == '__main__':
import pipeline
import fmri
import protein
# DAX files
pipeline.main("-D", "pipeline.xml")
fmri.main("-D", "fmri.xml")
protein.main("-D", "protein.xml")
# DOT files
pipeline.main("-d", "pipeline.dot")
fmri.main("-d", "fmri.dot")
protein.main("-d", "protein.dot")
distance_thresholds = [0.5, 0.8, 1, 1.25, 1.5, 1.75, 2, 2.5]
clustering_step = ['cosine', 'kmeans', 'jaccard', 'agglomerative', 'birch']
encodings = ['use']
scores_to_store = list()
for c_method in clustering_step:
if c_method == 'kmeans':
for cluster_num in clusters_nums:
for text_encoding in encodings:
print('-' * 50)
print("\nRunning {} for the clustering step.".format(
c_method))
print("Running for cluster number: {}".format(cluster_num))
eval_scores = main(dataset_ids,
evaluate=True,
clustering_method=c_method,
cluster_num=cluster_num,
encoding=text_encoding)
scores_to_store += eval_scores
elif c_method == 'jaccard':
print('-' * 50)
print("\nRunning {} for the clustering step.".format(c_method))
eval_scores = main(dataset_ids,
evaluate=True,
clustering_method=c_method)
scores_to_store += eval_scores
elif c_method == 'agglomerative':
for distance_threshold in distance_thresholds:
for text_encoding in encodings:
lb.init_svc_lb()
utils.task_record("init_service_lb")
if utils.no_task_record("init_applications"):
applications.init_app_list()
utils.task_record("init_applications")
if utils.no_task_record("init_application_detail"):
applications.init_app_svc_detail()
utils.task_record("init_application_detail")
if utils.no_task_record("init_applications_service_lb"):
lb.init_app_svc_lb()
utils.task_record("init_applications_service_lb")
if utils.no_task_record("init_cm"):
cm.init_cm()
utils.task_record("init_cm")
if utils.no_task_record("create_cm"):
cm.create_cm()
utils.task_record("create_cm")
# service trans
services.main()
applications.main()
pipeline.main()
if utils.no_task_record("update_jakiro_resource"):
services.build_volume_sql()
utils.task_record("update_jakiro_resource")
import find_deal_num
import datetime
while True:
start_time = datetime.datetime.now()
sources = ["dida", "dianping", "ftuan", "manzuo", "meituan", "nuomi", "wowo", "wuba"]
for source in sources:
print "start crawling deals from", source
async_pipeline.main(source, None, None)
find_deal_num.main(source, True)
miner.main(source, True)
print "start crawling deals from lashou"
pipeline.main("lashou", None, None)
find_deal_num.main("lashou", True)
miner.main("lashou", True)
end_time = datetime.datetime.now()
duration = end_time - start_time
print "====================="
print "The task takes", duration.seconds, "seconds to finish"
print "====================="
fhandler = open("finish_log.txt", "a")
fhandler.write(str(duration.seconds) + "\n")
fhandler.close()
if os.path.exists("main_term_sig"):
break
self.rf_text.fit(X_text, y)
self.rf_num.fit(X_num, y)
def predict_proba(self, X_text, X_num):
''' calculate probability that item is fraud '''
X_t = self.vect.transform(X_text)
pred_txt = self.rf_text.predict_proba(X_t)[:, 1]
pred_num = self.rf_num.predict_proba(X_num)[:, 1]
return np.mean([pred_txt, pred_num], axis=0)
def predict(self, X_text, X_num):
''' predict true/false for item '''
proba = self.predict_proba(X_text, X_num)
return (proba > THRESHOLD).astype(int)
def predict(model, X_text, X_num):
''' return predictions for a new item'''
prediction = model.predict(X_text, X_num)
probability = model.predict_proba(X_text, X_num)
return prediction, probability
if __name__ == '__main__':
''' run model.py to generate a model to base predictions on'''
X_text, X_num, y = main()
model = MyModel()
model.fit(X_text, X_num, y)
with open('model.pkl', 'wb') as f:
pickle.dump(model, f)
def create_dataset(source, target):
files = combine_sources.main(source, target)
new_files = {'all': files}
pipeline.main(new_files, target)
def take():
x = pipeline.main(False)
print(x)
var = 'static/' + x['id'] + '.jpg'
return render_template("other.html", variable=var, name=x['name'])
if __name__ == '__main__':
import pipeline
import fmri
import protein
# DAX files
pipeline.main("-D", "pipeline.xml")
fmri.main("-D", "fmri.xml")
protein.main("-D", "protein.xml")
# DOT files
pipeline.main("-d", "pipeline.dot")
fmri.main("-d", "fmri.dot")
protein.main("-d", "protein.dot")
from pipeline import main main()
