def test_string(self):
expected = {'type': 'COMMAND', 'type_data': 'progress', 'data': 0.1}
self.assertEqual(progress('0.1'), expected)
expected = {'type': 'COMMAND', 'type_data': 'progress', 'data': 0}
self.assertEqual(progress('0'), expected)
expected = {'type': 'COMMAND', 'type_data': 'progress', 'data': 1}
self.assertEqual(progress('1'), expected)
def test_number(self):
expected = {'type': 'COMMAND', 'type_data': 'progress', 'data': 0.1}
self.assertEqual(progress(0.1), expected)
expected = {'type': 'COMMAND', 'type_data': 'progress', 'data': 0}
self.assertEqual(progress(0), expected)
expected = {'type': 'COMMAND', 'type_data': 'progress', 'data': 1}
self.assertEqual(progress(1), expected)
def save_results(matched, notmatched, badquality, skipped, total,
_progress):
total = float(total)
send_message(
save(
"matched",
"{:,} reads ({:.2f} %)".format(matched,
100 * matched / total),
))
send_message(
save(
"notmatched",
"{:,} reads ({:.2f} %)".format(notmatched,
100 * notmatched / total),
))
send_message(
save(
"badquality",
"{:,} reads ({:.2f} %)".format(badquality,
100 * badquality / total),
))
send_message(
save(
"skipped",
"{:,} reads ({:.2f} %)".format(skipped,
100 * skipped / total),
))
send_message(progress(_progress))
def progress(self, progress):
"""Report process progress.
:param progress: A float between 0 and 1 denoting the progress
"""
report = resolwe_runtime_utils.progress(progress)
# TODO: Use the protocol to report progress.
print(report)
def progress(self, progress):
"""Report process progress.
:param progress: A float between 0 and 1 denoting the progress
"""
report = resolwe_runtime_utils.progress(progress)
# TODO: Use the protocol to report progress.
print(report)
def test_improper_input(self):
self.assertEqual(progress(None), '{"proc.warning": "Progress must be a float."}')
self.assertEqual(progress('one'), '{"proc.warning": "Progress must be a float."}')
self.assertEqual(progress('[0.1]'), '{"proc.warning": "Progress must be a float."}')
self.assertEqual(progress(-1),
'{"proc.warning": "Progress must be a float between 0 and 1."}')
self.assertEqual(progress(1.1),
'{"proc.warning": "Progress must be a float between 0 and 1."}')
self.assertEqual(progress('1.1'),
'{"proc.warning": "Progress must be a float between 0 and 1."}')
def test_improper_input(self):
self.assertEqual(progress(None),
'{"proc.warning": "Progress must be a float."}')
self.assertEqual(progress('one'),
'{"proc.warning": "Progress must be a float."}')
self.assertEqual(progress('[0.1]'),
'{"proc.warning": "Progress must be a float."}')
self.assertEqual(
progress(-1),
'{"proc.warning": "Progress must be a float between 0 and 1."}')
self.assertEqual(
progress(1.1),
'{"proc.warning": "Progress must be a float between 0 and 1."}')
self.assertEqual(
progress('1.1'),
'{"proc.warning": "Progress must be a float between 0 and 1."}')
def test_improper_input(self):
expected = {
'type': 'COMMAND',
'type_data': 'process_log',
'data': {'warning': 'Progress must be a float.'},
}
self.assertEqual(progress(None), expected)
self.assertEqual(progress('one'), expected)
self.assertEqual(progress('[0.1]'), expected)
expected = {
'type': 'COMMAND',
'type_data': 'process_log',
'data': {'warning': 'Progress must be a float between 0 and 1.'},
}
self.assertEqual(progress(-1), expected)
self.assertEqual(progress(1.1), expected)
self.assertEqual(progress('1.1'), expected)
def test_bool(self):
expected = {'type': 'COMMAND', 'type_data': 'progress', 'data': 1.0}
self.assertEqual(progress(True), expected)
def test_number(self):
self.assertEqual(progress(0.1), '{"proc.progress": 0.1}')
self.assertEqual(progress(0), '{"proc.progress": 0}')
self.assertEqual(progress(1), '{"proc.progress": 1}')
def test_string(self):
self.assertEqual(progress('0.1'), '{"proc.progress": 0.1}')
def test_bool(self):
self.assertEqual(progress(True), '{"proc.progress": 1.0}')
def read_multiplexed(reads1_file, reads2_file, barcodes_file, pool_maps,
progress_start):
"""Parse multiplexed file."""
pool_name = reads1_file.split(".")[0]
def nicename(a):
return a.replace("#", "").replace(" ",
" ").replace("/",
" ").replace(" ", "_")
files, f1, f2, fbar = {}, None, None, None
try:
barcodes = set(pool_maps.keys())
print("BARCODES: {}".format(barcodes))
for barcode in barcodes:
name = nicename(pool_maps[barcode])
if reads2_file:
filename = "{}_{}_{}_mate1.fq.gz".format(
pool_name, name, barcode)
files[barcode] = gzip.open(filename, "wb")
filename = "{}_{}_{}_mate2.fq.gz".format(
pool_name, name, barcode)
files[barcode + "2"] = gzip.open(filename, "wb")
else:
filename = "{}_{}_{}.fq.gz".format(pool_name, name, barcode)
files[barcode] = gzip.open(filename, "wb")
if reads2_file:
files["notmatched"] = gzip.open(
"Not_Matched_{}_mate1.fq.gz".format(pool_name), "wb")
files["badquality"] = gzip.open(
"Bad_Quality_{}_mate1.fq.gz".format(pool_name), "wb")
files["notmatched2"] = gzip.open(
"Not_Matched_{}_mate2.fq.gz".format(pool_name), "wb")
files["badquality2"] = gzip.open(
"Bad_Quality_{}_mate2.fq.gz".format(pool_name), "wb")
else:
files["notmatched"] = gzip.open(
"Not_Matched_{}.fq.gz".format(pool_name), "wb")
files["badquality"] = gzip.open(
"Bad_Quality_{}.fq.gz".format(pool_name), "wb")
filenames = list(sorted(set(f.name for f in files.values())))
p = subprocess.Popen(
"gzip -dc {} | wc -l".format(barcodes_file),
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
numlines, err = p.communicate()
if err:
raise Exception(err)
numlines = int(numlines)
readid, matched, notmatched, badquality, skipped = 0, 0, 0, 0, 0
send_message(progress(progress_start))
_progress = progress_start
progress_step = (0.9 - _progress) / 20.0
progress_span = numlines / 20
def save_results(matched, notmatched, badquality, skipped, total,
_progress):
total = float(total)
send_message(
save(
"matched",
"{:,} reads ({:.2f} %)".format(matched,
100 * matched / total),
))
send_message(
save(
"notmatched",
"{:,} reads ({:.2f} %)".format(notmatched,
100 * notmatched / total),
))
send_message(
save(
"badquality",
"{:,} reads ({:.2f} %)".format(badquality,
100 * badquality / total),
))
send_message(
save(
"skipped",
"{:,} reads ({:.2f} %)".format(skipped,
100 * skipped / total),
))
send_message(progress(_progress))
f1 = gzip.GzipFile(reads1_file, "r")
fbar = gzip.GzipFile(barcodes_file, "r")
if reads2_file:
f2 = gzip.GzipFile(reads2_file, "r")
while True:
readid += 1
r1 = f1.readline()
if not r1:
break
r1 = r1.decode("utf-8").rstrip("\r").rstrip("\n").split("\t")
if len(r1) != 11:
print("SKIPPED: error in {} line in r1".format(readid))
continue
s1 = r1[-3].replace(".", "N")
p1 = r1[-1]
rbar = fbar.readline()
if not rbar:
break
rbar = rbar.decode("utf-8").rstrip("\r").rstrip("\n").split("\t")
if len(rbar) != 11:
print("SKIPPED: error in {} line in rbar".format(readid))
continue
sbar = rbar[-3].replace(".", "N")[:barcode_length]
pbar = rbar[-1]
if reads2_file:
r2 = f2.readline()
if not r2:
break
r2 = r2.decode("utf-8").rstrip("\r").rstrip("\n").split("\t")
if len(r2) != 11:
print("SKIPPED: error in {} line in r2".format(readid))
continue
s2 = r2[-3].replace(".", "N")
p2 = r2[-1]
else:
r2 = r1
p2 = p1
if r1[:7] == r2[:7] == rbar[:7] and p1 == p2 == pbar:
idline = "@" + ":".join(r1[:7]) + " " + sbar
if p1 == "1" and p2 == "1":
if sbar in barcodes:
files[sbar].write(
(idline + "\n" + s1 + "\n" + "+" + "\n" + r1[-2] +
"\n").encode("utf-8"))
if reads2_file:
files[sbar + "2"].write(
(idline + "\n" + s2 + "\n" + "+" + "\n" +
r2[-2] + "\n").encode("utf-8"))
matched += 1
else:
files["notmatched"].write(
(idline + "\n" + s1 + "\n" + "+" + "\n" + r1[-2] +
"\n").encode("utf-8"))
if reads2_file:
files["notmatched2"].write(
(idline + "\n" + s2 + "\n" + "+" + "\n" +
r2[-2] + "\n").encode("utf-8"))
notmatched += 1
else:
files["badquality"].write(
(idline + "\n" + s1 + "\n" + "+" + "\n" + r1[-2] +
"\n").encode("utf-8"))
if reads2_file:
files["badquality2"].write(
(idline + "\n" + s2 + "\n" + "+" + "\n" + r2[-2] +
"\n").encode("utf-8"))
badquality += 1
else:
print("SKIPPED: {}, p1: {}, p2: {}, pbar: {}".format(
readid, p1, p2, pbar))
print("{} ? {} ? {}".format(r1[:7], r2[:7], rbar[:7]))
skipped += 1
if readid % progress_span == 0:
_progress += progress_step
save_results(matched, notmatched, badquality, skipped, readid,
_progress)
save_results(matched, notmatched, badquality, skipped, readid, 0.9)
finally:
if f1:
f1.close()
if f2:
f2.close()
if fbar:
fbar.close()
for f in files:
files[f].close()
return filenames
def test_bool(self):
self.assertEqual(progress(True), '{"proc.progress": 1.0}')
def test_string(self):
self.assertEqual(progress('0.1'), '{"proc.progress": 0.1}')
self.assertEqual(progress('0'), '{"proc.progress": 0.0}')
self.assertEqual(progress('1'), '{"proc.progress": 1.0}')
def test_number(self):
self.assertEqual(progress(0.1), '{"proc.progress": 0.1}')
self.assertEqual(progress(0), '{"proc.progress": 0.0}')
self.assertEqual(progress(1), '{"proc.progress": 1.0}')
