def nutshell(): print("Nutshell") fig, ax = newfig(1.) names = list(filter(lambda x: 'l1_' not in x, df.columns.values)) errors = np.mean(kf_scores, axis=0) hyb_errors = np.mean(kf_scores_hybrid, axis=0) limit = [] pl.barplot(names, [('GMRF', errors), ('HRF', hyb_errors)], [1], limit=limit, ax=ax) savefig('nutshell')
def collectNodeUsagePlot(trial, core, timing=None, medium=None, verification=None): res = database.cursor().execute( """ SELECT nodes.name, COUNT(DISTINCT answers.solution) AS c1 FROM nodes LEFT JOIN answers ON (nodes.id = answers.src OR nodes.id = answers.dest) LEFT JOIN solutions ON (answers.solution = solutions.id) LEFT JOIN students ON (solutions.student = students.id) WHERE solutions.trial=? AND (timing=? OR %d) AND (medium=? OR %d) AND (verification = ? OR %d) GROUP BY nodes.id ORDER BY c1 desc """ % (timing == None, medium == None, verification == None), (trial, timing, medium, verification)).fetchall() res = list(map(lambda r: [r[0], [r[1]]], res)) return [ "image", plot.barplot("nodeusage-%s-%s-%s.png" % (timing, medium, verification), res) ]
def collectNodeUsagePlot(trial, core, timing = None, medium = None, verification = None): res = database.cursor().execute(""" SELECT nodes.name, COUNT(DISTINCT answers.solution) AS c1 FROM nodes LEFT JOIN answers ON (nodes.id = answers.src OR nodes.id = answers.dest) LEFT JOIN solutions ON (answers.solution = solutions.id) LEFT JOIN students ON (solutions.student = students.id) WHERE solutions.trial=? AND (timing=? OR %d) AND (medium=? OR %d) AND (verification = ? OR %d) GROUP BY nodes.id ORDER BY c1 desc """ % (timing == None, medium == None, verification == None), (trial,timing,medium,verification)).fetchall() res = list(map(lambda r: [r[0], [r[1]]], res)) return ["image", plot.barplot("nodeusage-%s-%s-%s.png" % (timing,medium,verification), res)]
def detect(seq_dir, rootdir, database, barcode, pathogen): max_threads = cpu_count() free = True abs_rootdir = os.path.abspath(rootdir) seq_rootdir = os.path.abspath(seq_dir) batch = 0 files_old = [] while free: batch += 1 fastq = [ y for x in os.walk(seq_rootdir) for y in glob(os.path.join(x[0], '*.fastq')) if "analysed" not in y ] fq = [ y for x in os.walk(seq_rootdir) for y in glob(os.path.join(x[0], '*.fq')) if "analysed" not in y ] files = fastq + fq file_to_do = [] for file in files: if file not in files_old: file_to_do.append(file) files_old = files_old + file_to_do output_barcode = os.path.join(abs_rootdir, "analysed_barcode") analysed_fasta = os.path.join(abs_rootdir, "analysed_fasta") analysed_fastq = os.path.join(abs_rootdir, "analysed_fastq") analysed_blastn = os.path.join(abs_rootdir, "analysed_blastn") analysing_fasta = os.path.join(abs_rootdir, "analysing_fasta") if not os.path.exists(output_barcode): os.mkdir(output_barcode) if not os.path.exists(analysed_fasta): os.mkdir(analysed_fasta) if not os.path.exists(analysed_fastq): os.mkdir(analysed_fastq) if not os.path.exists(analysed_blastn): os.mkdir(analysed_blastn) if not os.path.exists(analysing_fasta): os.mkdir(analysing_fasta) if len(file_to_do) == 0: sys.exit("NO NEW FASTQ FILES") fastq_combined = os.path.join(analysed_fastq, 'combined.' + str(batch) + '.fq') with open(fastq_combined, 'wb') as wfd: for fileIn in file_to_do: if fileIn.endswith("fastq") or fileIn.endswith("fq"): if not "combined" in fileIn: with open(fileIn, 'rb') as fd: shutil.copyfileobj(fd, wfd, 1024 * 1024 * 10) sys.stdout.write("ANALYSING BATCH N. " + str(batch) + "\n") #for file in file_to_do: # shutil.move(file, os.path.join(analysed_fastq, file.split("/")[-1])) if barcode: with open(os.path.join(abs_rootdir, "porechop.log"), "w") as fhlog, open( os.path.join(abs_rootdir, "porechop.err"), "w") as fherr: cmd_porechop = "porechop -i %s -b %s" % (fastq_combined, output_barcode) p = subprocess.Popen(cmd_porechop, stdout=fhlog, stderr=fherr, universal_newlines=True, shell=True) p.communicate() reads_number = [] for root, dirs, files in os.walk(output_barcode): for file in files: if file.startswith("BC") and file.endswith("fastq"): count = 0 file_split = os.path.join(root, file) fasta_name = ".".join( [file.split(".")[0], str(batch), "fasta"]) fasta_seq = [] for record in SeqIO.parse(file_split, "fastq"): count += 1 record.id = file.split(".")[0] + "_" + str(count) fasta_seq.append(record) reads_number.append([file.split(".")[0], count]) SeqIO.write(fasta_seq, os.path.join(analysing_fasta, fasta_name), "fasta") fasta = [ y for x in os.walk(analysing_fasta) for y in glob(os.path.join(x[0], '*.fasta')) ] fasta_ready = [] for fasta_file in fasta: fasta_ready.append([fasta_file, database, analysed_blastn]) if len(fasta) < max_threads: max_threads = len(fasta) with Pool(max_threads) as p: results = p.map(blast, fasta_ready) for file_fasta in fasta: shutil.move( file_fasta, os.path.join(analysed_fasta, file_fasta.split("/")[-1])) plot.barplot(results) #, reads_number) plt.show() print("DONE")