def test_string(self): expected = {'type': 'COMMAND', 'type_data': 'progress', 'data': 0.1} self.assertEqual(progress('0.1'), expected) expected = {'type': 'COMMAND', 'type_data': 'progress', 'data': 0} self.assertEqual(progress('0'), expected) expected = {'type': 'COMMAND', 'type_data': 'progress', 'data': 1} self.assertEqual(progress('1'), expected)
def test_number(self): expected = {'type': 'COMMAND', 'type_data': 'progress', 'data': 0.1} self.assertEqual(progress(0.1), expected) expected = {'type': 'COMMAND', 'type_data': 'progress', 'data': 0} self.assertEqual(progress(0), expected) expected = {'type': 'COMMAND', 'type_data': 'progress', 'data': 1} self.assertEqual(progress(1), expected)
def save_results(matched, notmatched, badquality, skipped, total, _progress): total = float(total) send_message( save( "matched", "{:,} reads ({:.2f} %)".format(matched, 100 * matched / total), )) send_message( save( "notmatched", "{:,} reads ({:.2f} %)".format(notmatched, 100 * notmatched / total), )) send_message( save( "badquality", "{:,} reads ({:.2f} %)".format(badquality, 100 * badquality / total), )) send_message( save( "skipped", "{:,} reads ({:.2f} %)".format(skipped, 100 * skipped / total), )) send_message(progress(_progress))
def progress(self, progress): """Report process progress. :param progress: A float between 0 and 1 denoting the progress """ report = resolwe_runtime_utils.progress(progress) # TODO: Use the protocol to report progress. print(report)
def test_improper_input(self): self.assertEqual(progress(None), '{"proc.warning": "Progress must be a float."}') self.assertEqual(progress('one'), '{"proc.warning": "Progress must be a float."}') self.assertEqual(progress('[0.1]'), '{"proc.warning": "Progress must be a float."}') self.assertEqual(progress(-1), '{"proc.warning": "Progress must be a float between 0 and 1."}') self.assertEqual(progress(1.1), '{"proc.warning": "Progress must be a float between 0 and 1."}') self.assertEqual(progress('1.1'), '{"proc.warning": "Progress must be a float between 0 and 1."}')
def test_improper_input(self): self.assertEqual(progress(None), '{"proc.warning": "Progress must be a float."}') self.assertEqual(progress('one'), '{"proc.warning": "Progress must be a float."}') self.assertEqual(progress('[0.1]'), '{"proc.warning": "Progress must be a float."}') self.assertEqual( progress(-1), '{"proc.warning": "Progress must be a float between 0 and 1."}') self.assertEqual( progress(1.1), '{"proc.warning": "Progress must be a float between 0 and 1."}') self.assertEqual( progress('1.1'), '{"proc.warning": "Progress must be a float between 0 and 1."}')
def test_improper_input(self): expected = { 'type': 'COMMAND', 'type_data': 'process_log', 'data': {'warning': 'Progress must be a float.'}, } self.assertEqual(progress(None), expected) self.assertEqual(progress('one'), expected) self.assertEqual(progress('[0.1]'), expected) expected = { 'type': 'COMMAND', 'type_data': 'process_log', 'data': {'warning': 'Progress must be a float between 0 and 1.'}, } self.assertEqual(progress(-1), expected) self.assertEqual(progress(1.1), expected) self.assertEqual(progress('1.1'), expected)
def test_bool(self): expected = {'type': 'COMMAND', 'type_data': 'progress', 'data': 1.0} self.assertEqual(progress(True), expected)
def test_number(self): self.assertEqual(progress(0.1), '{"proc.progress": 0.1}') self.assertEqual(progress(0), '{"proc.progress": 0}') self.assertEqual(progress(1), '{"proc.progress": 1}')
def test_string(self): self.assertEqual(progress('0.1'), '{"proc.progress": 0.1}')
def test_bool(self): self.assertEqual(progress(True), '{"proc.progress": 1.0}')
def read_multiplexed(reads1_file, reads2_file, barcodes_file, pool_maps, progress_start): """Parse multiplexed file.""" pool_name = reads1_file.split(".")[0] def nicename(a): return a.replace("#", "").replace(" ", " ").replace("/", " ").replace(" ", "_") files, f1, f2, fbar = {}, None, None, None try: barcodes = set(pool_maps.keys()) print("BARCODES: {}".format(barcodes)) for barcode in barcodes: name = nicename(pool_maps[barcode]) if reads2_file: filename = "{}_{}_{}_mate1.fq.gz".format( pool_name, name, barcode) files[barcode] = gzip.open(filename, "wb") filename = "{}_{}_{}_mate2.fq.gz".format( pool_name, name, barcode) files[barcode + "2"] = gzip.open(filename, "wb") else: filename = "{}_{}_{}.fq.gz".format(pool_name, name, barcode) files[barcode] = gzip.open(filename, "wb") if reads2_file: files["notmatched"] = gzip.open( "Not_Matched_{}_mate1.fq.gz".format(pool_name), "wb") files["badquality"] = gzip.open( "Bad_Quality_{}_mate1.fq.gz".format(pool_name), "wb") files["notmatched2"] = gzip.open( "Not_Matched_{}_mate2.fq.gz".format(pool_name), "wb") files["badquality2"] = gzip.open( "Bad_Quality_{}_mate2.fq.gz".format(pool_name), "wb") else: files["notmatched"] = gzip.open( "Not_Matched_{}.fq.gz".format(pool_name), "wb") files["badquality"] = gzip.open( "Bad_Quality_{}.fq.gz".format(pool_name), "wb") filenames = list(sorted(set(f.name for f in files.values()))) p = subprocess.Popen( "gzip -dc {} | wc -l".format(barcodes_file), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) numlines, err = p.communicate() if err: raise Exception(err) numlines = int(numlines) readid, matched, notmatched, badquality, skipped = 0, 0, 0, 0, 0 send_message(progress(progress_start)) _progress = progress_start progress_step = (0.9 - _progress) / 20.0 progress_span = numlines / 20 def save_results(matched, notmatched, badquality, skipped, total, _progress): total = float(total) send_message( save( "matched", "{:,} reads ({:.2f} %)".format(matched, 100 * matched / total), )) send_message( save( "notmatched", "{:,} reads ({:.2f} %)".format(notmatched, 100 * notmatched / total), )) send_message( save( "badquality", "{:,} reads ({:.2f} %)".format(badquality, 100 * badquality / total), )) send_message( save( "skipped", "{:,} reads ({:.2f} %)".format(skipped, 100 * skipped / total), )) send_message(progress(_progress)) f1 = gzip.GzipFile(reads1_file, "r") fbar = gzip.GzipFile(barcodes_file, "r") if reads2_file: f2 = gzip.GzipFile(reads2_file, "r") while True: readid += 1 r1 = f1.readline() if not r1: break r1 = r1.decode("utf-8").rstrip("\r").rstrip("\n").split("\t") if len(r1) != 11: print("SKIPPED: error in {} line in r1".format(readid)) continue s1 = r1[-3].replace(".", "N") p1 = r1[-1] rbar = fbar.readline() if not rbar: break rbar = rbar.decode("utf-8").rstrip("\r").rstrip("\n").split("\t") if len(rbar) != 11: print("SKIPPED: error in {} line in rbar".format(readid)) continue sbar = rbar[-3].replace(".", "N")[:barcode_length] pbar = rbar[-1] if reads2_file: r2 = f2.readline() if not r2: break r2 = r2.decode("utf-8").rstrip("\r").rstrip("\n").split("\t") if len(r2) != 11: print("SKIPPED: error in {} line in r2".format(readid)) continue s2 = r2[-3].replace(".", "N") p2 = r2[-1] else: r2 = r1 p2 = p1 if r1[:7] == r2[:7] == rbar[:7] and p1 == p2 == pbar: idline = "@" + ":".join(r1[:7]) + " " + sbar if p1 == "1" and p2 == "1": if sbar in barcodes: files[sbar].write( (idline + "\n" + s1 + "\n" + "+" + "\n" + r1[-2] + "\n").encode("utf-8")) if reads2_file: files[sbar + "2"].write( (idline + "\n" + s2 + "\n" + "+" + "\n" + r2[-2] + "\n").encode("utf-8")) matched += 1 else: files["notmatched"].write( (idline + "\n" + s1 + "\n" + "+" + "\n" + r1[-2] + "\n").encode("utf-8")) if reads2_file: files["notmatched2"].write( (idline + "\n" + s2 + "\n" + "+" + "\n" + r2[-2] + "\n").encode("utf-8")) notmatched += 1 else: files["badquality"].write( (idline + "\n" + s1 + "\n" + "+" + "\n" + r1[-2] + "\n").encode("utf-8")) if reads2_file: files["badquality2"].write( (idline + "\n" + s2 + "\n" + "+" + "\n" + r2[-2] + "\n").encode("utf-8")) badquality += 1 else: print("SKIPPED: {}, p1: {}, p2: {}, pbar: {}".format( readid, p1, p2, pbar)) print("{} ? {} ? {}".format(r1[:7], r2[:7], rbar[:7])) skipped += 1 if readid % progress_span == 0: _progress += progress_step save_results(matched, notmatched, badquality, skipped, readid, _progress) save_results(matched, notmatched, badquality, skipped, readid, 0.9) finally: if f1: f1.close() if f2: f2.close() if fbar: fbar.close() for f in files: files[f].close() return filenames
def test_string(self): self.assertEqual(progress('0.1'), '{"proc.progress": 0.1}') self.assertEqual(progress('0'), '{"proc.progress": 0.0}') self.assertEqual(progress('1'), '{"proc.progress": 1.0}')
def test_number(self): self.assertEqual(progress(0.1), '{"proc.progress": 0.1}') self.assertEqual(progress(0), '{"proc.progress": 0.0}') self.assertEqual(progress(1), '{"proc.progress": 1.0}')