def nutshell():
print("Nutshell")
fig, ax = newfig(1.)
names = list(filter(lambda x: 'l1_' not in x, df.columns.values))
errors = np.mean(kf_scores, axis=0)
hyb_errors = np.mean(kf_scores_hybrid, axis=0)
limit = []
pl.barplot(names, [('GMRF', errors), ('HRF', hyb_errors)], [1], limit=limit, ax=ax)
savefig('nutshell')
def collectNodeUsagePlot(trial,
core,
timing=None,
medium=None,
verification=None):
res = database.cursor().execute(
"""
SELECT
nodes.name,
COUNT(DISTINCT answers.solution) AS c1
FROM nodes
LEFT JOIN answers ON (nodes.id = answers.src OR nodes.id = answers.dest)
LEFT JOIN solutions ON (answers.solution = solutions.id)
LEFT JOIN students ON (solutions.student = students.id)
WHERE solutions.trial=? AND (timing=? OR %d) AND (medium=? OR %d) AND (verification = ? OR %d)
GROUP BY nodes.id
ORDER BY c1 desc
""" % (timing == None, medium == None, verification == None),
(trial, timing, medium, verification)).fetchall()
res = list(map(lambda r: [r[0], [r[1]]], res))
return [
"image",
plot.barplot("nodeusage-%s-%s-%s.png" % (timing, medium, verification),
res)
]
def collectNodeUsagePlot(trial, core, timing = None, medium = None, verification = None):
res = database.cursor().execute("""
SELECT
nodes.name,
COUNT(DISTINCT answers.solution) AS c1
FROM nodes
LEFT JOIN answers ON (nodes.id = answers.src OR nodes.id = answers.dest)
LEFT JOIN solutions ON (answers.solution = solutions.id)
LEFT JOIN students ON (solutions.student = students.id)
WHERE solutions.trial=? AND (timing=? OR %d) AND (medium=? OR %d) AND (verification = ? OR %d)
GROUP BY nodes.id
ORDER BY c1 desc
""" % (timing == None, medium == None, verification == None), (trial,timing,medium,verification)).fetchall()
res = list(map(lambda r: [r[0], [r[1]]], res))
return ["image", plot.barplot("nodeusage-%s-%s-%s.png" % (timing,medium,verification), res)]
def detect(seq_dir, rootdir, database, barcode, pathogen):
max_threads = cpu_count()
free = True
abs_rootdir = os.path.abspath(rootdir)
seq_rootdir = os.path.abspath(seq_dir)
batch = 0
files_old = []
while free:
batch += 1
fastq = [
y for x in os.walk(seq_rootdir)
for y in glob(os.path.join(x[0], '*.fastq')) if "analysed" not in y
]
fq = [
y for x in os.walk(seq_rootdir)
for y in glob(os.path.join(x[0], '*.fq')) if "analysed" not in y
]
files = fastq + fq
file_to_do = []
for file in files:
if file not in files_old:
file_to_do.append(file)
files_old = files_old + file_to_do
output_barcode = os.path.join(abs_rootdir, "analysed_barcode")
analysed_fasta = os.path.join(abs_rootdir, "analysed_fasta")
analysed_fastq = os.path.join(abs_rootdir, "analysed_fastq")
analysed_blastn = os.path.join(abs_rootdir, "analysed_blastn")
analysing_fasta = os.path.join(abs_rootdir, "analysing_fasta")
if not os.path.exists(output_barcode):
os.mkdir(output_barcode)
if not os.path.exists(analysed_fasta):
os.mkdir(analysed_fasta)
if not os.path.exists(analysed_fastq):
os.mkdir(analysed_fastq)
if not os.path.exists(analysed_blastn):
os.mkdir(analysed_blastn)
if not os.path.exists(analysing_fasta):
os.mkdir(analysing_fasta)
if len(file_to_do) == 0:
sys.exit("NO NEW FASTQ FILES")
fastq_combined = os.path.join(analysed_fastq,
'combined.' + str(batch) + '.fq')
with open(fastq_combined, 'wb') as wfd:
for fileIn in file_to_do:
if fileIn.endswith("fastq") or fileIn.endswith("fq"):
if not "combined" in fileIn:
with open(fileIn, 'rb') as fd:
shutil.copyfileobj(fd, wfd, 1024 * 1024 * 10)
sys.stdout.write("ANALYSING BATCH N. " + str(batch) + "\n")
#for file in file_to_do:
# shutil.move(file, os.path.join(analysed_fastq, file.split("/")[-1]))
if barcode:
with open(os.path.join(abs_rootdir, "porechop.log"),
"w") as fhlog, open(
os.path.join(abs_rootdir, "porechop.err"),
"w") as fherr:
cmd_porechop = "porechop -i %s -b %s" % (fastq_combined,
output_barcode)
p = subprocess.Popen(cmd_porechop,
stdout=fhlog,
stderr=fherr,
universal_newlines=True,
shell=True)
p.communicate()
reads_number = []
for root, dirs, files in os.walk(output_barcode):
for file in files:
if file.startswith("BC") and file.endswith("fastq"):
count = 0
file_split = os.path.join(root, file)
fasta_name = ".".join(
[file.split(".")[0],
str(batch), "fasta"])
fasta_seq = []
for record in SeqIO.parse(file_split, "fastq"):
count += 1
record.id = file.split(".")[0] + "_" + str(count)
fasta_seq.append(record)
reads_number.append([file.split(".")[0], count])
SeqIO.write(fasta_seq,
os.path.join(analysing_fasta, fasta_name),
"fasta")
fasta = [
y for x in os.walk(analysing_fasta)
for y in glob(os.path.join(x[0], '*.fasta'))
]
fasta_ready = []
for fasta_file in fasta:
fasta_ready.append([fasta_file, database, analysed_blastn])
if len(fasta) < max_threads:
max_threads = len(fasta)
with Pool(max_threads) as p:
results = p.map(blast, fasta_ready)
for file_fasta in fasta:
shutil.move(
file_fasta,
os.path.join(analysed_fasta,
file_fasta.split("/")[-1]))
plot.barplot(results) #, reads_number)
plt.show()
print("DONE")
