def main(): parser = OptionParser(usage = "usage: %prog [options] <Variants gvf.gz file> ...") parser.add_option("--db", dest="db_path", help="Database path") parser.add_option("-L", "--log-level", dest="log_level", default="info", choices=["debug", "info", "warn", "error", "critical", "notset"], help="Which log level: debug, info, warn, error, critical, notset") (options, args) = parser.parse_args() logging.basicConfig( level=LOG_LEVEL[options.log_level], format="%(asctime)s %(levelname)-5s : %(message)s") log = logging.getLogger("var_db") if len(args) < 1: log.error("At least one variants file is required") parser.print_help() exit(-1) if options.db_path is None: log.error("The database path should be specified") parser.print_help() exit(-1) db_path = options.db_path log.info("Opening database ...") db = VarXrefsDb(db_path) db.open() db.begin() total_count = 0 total_start_time = time.time() src_var_count = {} src_ratio = {} chromosomes = set() chr_var_count = {} strands = set() try: partial_count = 0 partial_start_time = time.time() for xref_path in args: log.info("Reading {0} ...".format(xref_path)) if not os.path.isfile(xref_path): log.error("File not found: {0}".format(xref_path)) exit(-1) mtime = datetime.fromtimestamp(os.path.getmtime(xref_path)) f = tsv.open(xref_path, "r") src_count = 0 src_start_time = time.time() line_num = 1 # discard headers line = f.readline() while line.startswith("#"): line = f.readline() line_num += 1 src_var_count[xref_path] = 0 for line in f: try: fields = [x if len(x) > 0 else None for x in line.rstrip("\n").split("\t")] chr, source, type, start, end, _1, strand, _2, extra = fields start = int(start) end = int(end) ref = None alt = None xref = None try: for var in extra.split(";"): try: key, value = var.split("=") if key == "Dbxref": pos = value.index(":") xref = value[pos + 1:] elif key == "Reference_seq": ref = value elif key == "Variant_seq": alt = value except: continue except: pass if sum([1 if x is None else 0 for x in [chr, start, strand, ref, alt, source, xref]]) > 0: log.warn("Discarding incomplete variant: {0}".format(",".join([chr, str(start), strand, ref, alt, source, xref]))) continue src_var_count[xref_path] += 1 chromosomes.add(chr) if chr in chr_var_count: chr_var_count[chr] += 1 else: chr_var_count[chr] = 1 strands.add(strand) db.add(chr, start, ref, alt, source, xref, strand) total_count += 1 src_count += 1 partial_count += 1 elapsed_time = time.time() - partial_start_time if elapsed_time >= 10.0: ratio = float(partial_count) / elapsed_time log.debug(" {0:.1f} variants/second, {1} variants, {2} total variants".format(ratio, hsize(src_count), hsize(total_count))) partial_count = 0 partial_start_time = time.time() except Exception as ex: log.error("Error at line {0}:\n{1}".format(line_num, line.rstrip("\n"))) import sys import traceback traceback.print_exc(file=sys.stdout) continue finally: line_num += 1 elapsed_time = time.time() - src_start_time ratio = float(src_count) / elapsed_time src_ratio[xref_path] = ratio log.info(" {0:.1f} variants/second, {1} variants, {2} total variants".format(ratio, hsize(src_count), hsize(total_count))) f.close() db.commit() except KeyboardInterrupt: db.commit() log.warn("Interrupted by the user with Ctrl-C") exit(-1) except: db.rollback() raise finally: db.close() elapsed_time = time.time() - total_start_time total_ratio = float(total_count) / elapsed_time log.info("Statistics:") log.info(" Sources:") for xref_path in args: log.info(" {0}: {1} variants".format(os.path.basename(xref_path), src_var_count[xref_path])) total_size = 0 log.info(" Chromosomes:") for chr in chromosomes: log.info(" {0:>2}: {1:>7} variants".format(chr, str(chr_var_count[chr]))) log.info(" Strands: {0}".format(", ".join(strands))) log.info(" Total {0} variants ({1:.1f} variants/sec)".format(hsize(total_count), total_ratio))