def main():
parser = argparse.ArgumentParser(
description="Manipulate database indices")
cmd = DefaultCommandHelper(parser)
cmd.add_db_args()
parser.add_argument("ops", metavar="OPERATIONS", nargs="+", choices=["drop", "create"],
help="The operations to perform on the indices.")
args, logger = cmd.parse_args("index")
db = cmd.open_db()
try:
start_time = datetime.now()
for op in args.ops:
if op == "drop":
logger.info("Dropping indices ...")
db.drop_indices()
elif op == "create":
logger.info("Creating indices ...")
db.create_indices()
elapsed_time = datetime.now() - start_time
logger.info("Done. Elapsed time: {}".format(elapsed_time))
except:
return cmd.handle_error()
finally:
db.close()
return 0
def main():
parser = argparse.ArgumentParser(
description="Generate datasets needed to evaluate performance from Cosmic mutations")
cmd = DefaultCommandHelper(parser)
cmd.add_db_args()
parser.add_argument("data_path", metavar="PATH",
help="The CosmicMutantExport tsv file")
parser.add_argument("cgc_path", metavar="PATH",
help="The list of CGC genes")
parser.add_argument("tdrivers_path", metavar="PATH",
help="The list of TD drivers")
parser.add_argument("pdrivers_path", metavar="PATH",
help="The list of PD drivers")
parser.add_argument("-o", dest="prefix", metavar="PREFIX",
help="Output prefix.")
args, logger = cmd.parse_args("perf-cosmic")
fanns_db = cmd.open_db()
try:
snvs = extract_snvs(fanns_db, args.data_path, logger=logger)
create_datasets(snvs, args.cgc_path, args.tdrivers_path, args.pdrivers_path, args.prefix)
except:
cmd.handle_error()
return 0
def main():
parser = argparse.ArgumentParser(
description="Remove annotations")
cmd = DefaultCommandHelper(parser)
cmd.add_db_args()
parser.add_argument("id", metavar="ID", nargs="+",
help="Annotation identifier.")
args, logger = cmd.parse_args("ann-rm")
db = cmd.open_db()
try:
if "*" in args.id:
logger.info("Removing all the annotations ...")
for ann in db.maps():
logger.info(" {} {} ...".format(ann["id"], ann["name"]))
db.remove_map(ann["id"])
else:
for ann_id in args.id:
logger.info("Removing annotation {} ...".format(ann_id))
db.remove_map(ann_id)
db.commit()
except:
return cmd.handle_error()
finally:
db.close()
return 0
def main():
parser = argparse.ArgumentParser(
description="Create a functional scores database")
cmd = DefaultCommandHelper(parser)
cmd.add_db_args()
parser.add_argument("predictors", metavar="PREDICTORS", nargs="*",
help="Predictor identifiers")
args, logger = cmd.parse_args("create")
db = cmd.create_db()
try:
for predictor_id in args.predictors:
logger.info("Adding predictor {} ...".format(predictor_id))
db.add_predictor(predictor_id, FannsDb.SOURCE_PREDICTOR_TYPE)
db.set_initialized()
db.commit()
except:
return cmd.handle_error()
finally:
db.close()
return 0
def main():
parser = argparse.ArgumentParser(
description="Update predictors min, max and count")
cmd = DefaultCommandHelper(parser)
cmd.add_db_args()
cmd.add_selected_predictors_args()
args, logger = cmd.parse_args("pred-update")
db = cmd.open_db()
try:
predictors = cmd.get_selected_predictors(default_all=True)
logger.info("Updating predictors ...")
start_time = datetime.now()
db.update_predictors(predictors)
db.commit()
logger.info("Finished. elapsed={}".format(datetime.now() - start_time))
except:
return cmd.handle_error()
finally:
db.close()
return 0
def main():
parser = argparse.ArgumentParser(
description="List predictors")
cmd = DefaultCommandHelper(parser)
cmd.add_db_args()
parser.add_argument("--json", dest="to_json", action="store_true", default=False,
help="Print the results in json format")
args, log = cmd.parse_args("pred-list")
db = cmd.open_db()
try:
if args.to_json:
d = {}
for pred in db.predictors():
d[pred["id"]] = dict([(k,pred[k]) for k in ["type", "source", "min", "max", "count"]])
import json
print json.dumps(d, indent=True)
else:
print "\t".join(["ID", "TYPE", "SOURCE", "MIN", "MAX", "COUNT"])
for pred in db.predictors():
print "\t".join([str(pred[k]) for k in ["id", "type", "source", "min", "max", "count"]])
except:
return cmd.handle_error()
finally:
db.close()
return 0
def main():
parser = argparse.ArgumentParser(description="Add annotations")
cmd = DefaultCommandHelper(parser)
cmd.add_db_args()
parser.add_argument("id", metavar="ID", help="Annotation identifier.")
parser.add_argument("name", metavar="NAME", help="Annotation name.")
parser.add_argument(
"type", metavar="TYPE", choices=["transcript", "protein"], help="Annotation type: transcript, protein"
)
parser.add_argument("path", metavar="PATH", help="Annotation items")
parser.add_argument(
"--priority",
dest="priority",
default=0,
help="Priority for translating input annotations. 0 means not considered for translation. Default 0.",
)
parser.add_argument(
"--header",
dest="header",
action="store_true",
default=False,
help="Specify that the annotation items file have a header.",
)
args, logger = cmd.parse_args("ann-add")
db = cmd.open_db()
try:
logger.info("Creating annotation {} ...".format(args.name))
db.add_map(args.id, args.name, args.type, args.priority)
logger.info("Loading items ...")
with tsv.open(args.path) as f:
for source, value in tsv.lines(f, (str, str), header=args.header):
if len(source) > 0 and len(value) > 0:
db.add_map_item(args.id, source, value)
db.commit()
except:
return cmd.handle_error()
finally:
db.close()
return 0
def main():
parser = argparse.ArgumentParser(
description="Export SNV's")
cmd = DefaultCommandHelper(parser)
cmd.add_db_args()
parser.add_argument("dest_path", metavar="DEST",
help="The destination file. Use - for standard output.")
args, log = cmd.parse_args("export-snvs")
db = cmd.open_db()
logger.info("Exporting SNV's ...")
total_count = 0
total_start_time = time.time()
try:
progress = RatedProgress(logger, name="SNVs")
rows_count = 0
with tsv.open(args.dest_path, "w") as f:
for snv in db.snvs():
rows_count += 1
tsv.write_line(f, snv["chr"], snv["start"], snv["start"], snv["strand"], "{}>{}".format(snv["ref"], snv["alt"]), "S")
progress.update()
log.info("Finished. Total rows = {}, elapsed_time = {}".format(rows_count, progress.elapsed_time))
except:
return cmd.handle_error()
finally:
db.close()
return 0
def main():
parser = argparse.ArgumentParser(
description="List annotations")
cmd = DefaultCommandHelper(parser)
cmd.add_db_args()
args, logger = cmd.parse_args("ann-list")
db = cmd.open_db()
try:
print "\t".join(["ID", "NAME", "TYPE", "PRIORITY"])
for ann in db.maps():
print "\t".join([str(ann[k]) for k in ["id", "name", "type", "priority"]])
except:
return cmd.handle_error()
finally:
db.close()
return 0
def main():
parser = argparse.ArgumentParser(description="Map score values")
cmd = DefaultCommandHelper(parser)
cmd.add_db_args()
cmd.add_transform_args()
parser.add_argument(
"--skip-empty-scores",
dest="skip_empty_scores",
action="store_true",
default=False,
help="Skip transformation for empty scores",
)
args, logger = cmd.parse_args("scores-transform")
db = cmd.open_db()
try:
transforms = cmd.get_transforms()
predictors = transforms.keys()
logger.info("Transforming scores ...")
progress = RatedProgress(logger, name="SNVs")
rows_count = updated_count = 0
for row in db.query_scores(predictors=predictors):
rows_count += 1
scores = row["scores"]
upd_scores = {}
for predictor in transforms:
score = scores[predictor]
if args.skip_empty_scores and score is None:
continue
prev_score = score
for name, func in transforms[predictor]:
try:
score = func(score)
except:
raise Exception("Error transforming the {} score {} with {}".format(predictor, score, name))
if prev_score != score:
upd_scores[predictor] = score
if len(upd_scores) > 0:
db.update_scores(row["id"], upd_scores)
updated_count += 1
progress.update()
progress.log_totals()
logger.info("Commit ...")
db.commit()
logger.info(
"Finished. Total rows = {}, updated rows = {}, elapsed time = {}".format(
rows_count, updated_count, progress.elapsed_time
)
)
except:
return cmd.handle_error()
finally:
db.close()
return 0
def main():
parser = argparse.ArgumentParser(
description="Fetch Condel scores")
cmd = DefaultCommandHelper(parser)
cmd.add_db_args()
parser.add_argument("muts_path", metavar="SNVS_PATH",
help="SNV's to check. Use - for standard input.")
parser.add_argument("out_path", metavar="OUTPUT_PATH",
help="The results path. Use - for standard output.")
cmd.add_selected_predictors_args()
cmd.add_selected_annotations_args()
cmd.add_selected_columns_args()
args, logger = cmd.parse_args("fetch")
db = cmd.open_db()
predictors = cmd.get_selected_predictors()
annotations = cmd.get_selected_annotations()
columns = cmd.get_selected_columns()
logger.info("Reading {} ...".format(args.muts_path if args.muts_path != "-" else "from standard input"))
try:
progress = RatedProgress(logger, name="SNVs")
with tsv.open(args.muts_path) as f:
with tsv.open(args.out_path, "w") as wf:
tsv.write_line(wf, "ID", *[c.upper() for c in columns] + [a.upper() for a in annotations] + predictors)
hit = fail = 0
mut = DnaAndProtMutationParser()
for line_num, line in enumerate(f, start=1):
line = line.rstrip(" \n\r")
if len(line) == 0 or line.startswith("#"):
continue
try:
mut.parse(line)
except PrematureEnd:
logger.error("Missing fields at line {}".format(line_num))
fail += 1
continue
except UnexpectedToken as ex:
logger.error("Unexpected field '{}' at line {}".format(ex.args[0], line_num))
fail += 1
continue
exists = False
for row in query_mutation(logger, db, mut, annotations, predictors):
exists = True
ann = row["annotations"]
scores = row["scores"]
tsv.write_line(wf, mut.identifier,
*[row[c] for c in columns]
+ [ann[a] for a in annotations]
+ [scores[p] for p in predictors])
"""
if logger.isEnabledFor(logging.DEBUG):
logger.debug(" --> {} {} {} {} {} {} {} {} {} {}".format(
row["chr"], row["start"], row["ref"], row["alt"], row["transcript"],
row["protein"], row["aa_pos"], row["aa_ref"], row["aa_alt"],
mut.identifier or "*"))
"""
progress.update()
if exists:
hit += 1
else:
fail += 1
progress.log_totals()
logger.info("Finished. total={}, hits={}, fails={}, elapsed={}".format(hit + fail, hit, fail, progress.elapsed_time))
except:
return cmd.handle_error()
finally:
db.close()
def main():
parser = argparse.ArgumentParser(
description="Update scores in the database")
cmd = DefaultCommandHelper(parser)
cmd.add_db_args()
parser.add_argument("source_path", metavar="SOURCE",
help="The source file. Use - for standard input.")
cmd.add_selected_predictors_args()
parser.add_argument("--update-predictors", dest="update_predictors", action="store_true", default=False,
help="Update of the predictors.")
parser.add_argument("--ignore-errors", dest="ignore_errors", action="store_true", default=False,
help="When errors on the input file, report them but continue processing the input.")
args, logger = cmd.parse_args("update")
db = cmd.open_db()
predictors = cmd.get_selected_predictors(check_missing=False)
try:
progress = RatedProgress(logger, name="SNVs")
total_lines = 0
logger.info("Reading {} ...".format(args.source_path if args.source_path != "-" else "from standard input"))
with tsv.open(args.source_path) as f:
# Parse header
hdr_line = f.readline()
hdr = dict([(name, index) for index, name in enumerate(hdr_line.rstrip("\n").split("\t"))])
db_predictors = set([p["id"] for p in db.predictors()])
if len(predictors) == 0:
predictors = [name for name in hdr if name in db_predictors]
if len(predictors) == 0:
raise Exception("Any input file header match the available predictors in the database. Please specify them using -p.")
logger.info("Predictors: {}".format(", ".join(predictors)))
for predictor in filter(lambda p: p not in db_predictors, predictors):
logger.info("Creating predictor {} ...".format(predictor))
db.add_predictor(predictor, FannsDb.SOURCE_PREDICTOR_TYPE)
use_genome_coords = len(set(hdr.keys()) & set(GENOME_COORD_COLUMNS)) > 0
use_protein_coords = len(set(hdr.keys()) & set(PROTEIN_COORD_COLUMNS)) > 0
if not use_genome_coords and not use_protein_coords:
raise Exception("No coordinate columns found. "
"Use {} for genomic coordinates or {} for protein coordinates.".format(
GENOME_COORD_COLUMNS, PROTEIN_COORD_COLUMNS))
elif use_genome_coords and use_protein_coords:
logger.warn("Both, genomic and protein coordinates columns found. Using genomic coordinates by default")
if use_genome_coords:
coord_column_names = [n for n in hdr if n in set(GENOME_COORD_COLUMNS)]
coord_column_types = [GENOME_COORD_COLUMN_TYPE[n] for n in coord_column_names]
#get_rows = db.get_transcripts_by_dna
elif use_protein_coords:
coord_column_names = [n for n in hdr if n in set(PROTEIN_COORD_COLUMNS)]
coord_column_types = [PROTEIN_COORD_COLUMN_TYPE[n] for n in coord_column_names]
#get_rows = db.get_transcripts_by_protein
coord_column_indices = [hdr[n] for n in coord_column_names]
score_indices = [hdr[n] for n in predictors]
max_column_index = max(coord_column_indices + score_indices)
for line_num, line in enumerate(f, start=2):
fields = line.rstrip("\n").split("\t")
if len(fields) < max_column_index:
log.error("Missing columns for line {}: {}".format(line_num, " ".join(fields)))
if not args.ignore_errors:
raise
try:
coords = dict([(name.lower(), type_cast(fields[index])) for name, type_cast, index in zip(
coord_column_names, coord_column_types, coord_column_indices)])
scores = dict([(p, score_value(fields[i])) for p, i in zip(predictors, score_indices)])
except Exception as ex:
logger.error("{} at line {}: {}".format(str(ex), line_num, " ".join(fields)))
if not args.ignore_errors:
raise
try:
for row in db.query_scores(fields=[], **coords):
db.update_scores(row["id"], scores)
except Exception as ex:
logger.error("Error updating SNV at line {}: {}".format(line_num, str(ex)))
logger.error("SNV: {}".format(", ".join(["{}={}".format(k, v) for k, v in coords.items()])))
if not args.ignore_errors:
raise
progress.update()
progress.log_totals()
logger.info("Finalizing database ...")
if args.update_predictors:
logger.info("Updating predictors ...")
db.update_predictors()
logger.info("Committing ...")
db.commit()
logger.info("Finished successfully. Elapsed time: {}".format(progress.elapsed_time))
except:
return cmd.handle_error()
finally:
db.close()
return 0
def main():
parser = argparse.ArgumentParser(
description="Export Scores")
cmd = DefaultCommandHelper(parser)
cmd.add_db_args()
parser.add_argument("dest_path", metavar="OUTPUT_PATH",
help="The output file. Use - for standard output.")
cmd.add_selected_predictors_args()
cmd.add_selected_annotations_args()
cmd.add_selected_columns_args()
parser.add_argument("--json", dest="to_json", action="store_true", default=False,
help="Export the results in json format")
parser.add_argument("--sample", dest="sample", type=int, metavar="PCT",
help="Export a random sample of PCT %%")
parser.add_argument("--start", dest="start", type=int, metavar="N",
help="Start to export from the SNV number N")
parser.add_argument("--limit", dest="limit", type=int, metavar="N",
help="Limit the number of SNVs to export to N")
args, logger = cmd.parse_args("export")
db = cmd.open_db()
predictors = cmd.get_selected_predictors()
annotations = cmd.get_selected_annotations()
columns = cmd.get_selected_columns()
logger.info("Exporting ...")
random.seed(time.time())
total_count = 0
total_start_time = time.time()
try:
progress = RatedProgress(logger, name="SNVs")
to_json = args.to_json
sample = args.sample
start = args.start or 0
limit = args.limit
doc = None
last_pos = None
rows_count = 0
snvs_count = 0
with tsv.open(args.dest_path, "w") as f:
if not to_json:
tsv.write_line(f, *[c.upper() for c in columns] + [a.upper() for a in annotations] + predictors)
for row in db.query_scores(predictors=predictors, maps=annotations):
if not to_json:
if start > 0:
start -= 1
continue
if sample is not None and random.randint(1, 100) > sample:
continue
pos = (row["chr"], row["strand"], row["start"], row["ref"], row["alt"])
if last_pos != pos:
if to_json:
if start > 0:
start -= 1
continue
if limit is not None and snvs_count >= limit:
if doc is not None:
json.dump(doc, f)
f.write("\n")
break
snvs_count += 1
rows_count += 1
ann = row["annotations"]
scores = row["scores"]
if to_json:
tdoc = dict([(k,row[k]) for k in ["transcript", "protein", "aa_pos", "aa_ref", "aa_alt"]] +
[(k,scores[k]) for k in predictors])
if pos != last_pos:
if doc is not None:
if sample is None or random.randint(1, 100) <= sample:
json.dump(doc, f)
f.write("\n")
else:
snvs_count -= 1
doc = dict([(k, row[k]) for k in ["chr", "strand", "start", "ref", "alt"]] +
[("transcripts", [tdoc])])
else:
doc["transcripts"] += [tdoc]
else:
tsv.write_line(f,
*[row[c] for c in columns]
+ [ann[a] for a in annotations]
+ [scores[p] for p in predictors])
progress.update()
last_pos = pos
if not to_json and limit is not None and rows_count >= limit:
break
progress.log_totals()
logger.info("Finished. Total rows = {}, SNVs = {}, elapsed_time = {}".format(rows_count, snvs_count, progress.elapsed_time))
except:
return cmd.handle_error()
finally:
db.close()
return 0
def main():
parser = argparse.ArgumentParser(
description="Generate datasets needed to evaluate performance from Cosmic mutations")
cmd = DefaultCommandHelper(parser)
cmd.add_db_args()
parser.add_argument("data_path", metavar="PATH",
help="The CosmicMutantExport tsv file")
parser.add_argument("cgc_path", metavar="PATH",
help="The list of CGC genes")
parser.add_argument("drivers_path", metavar="PATH",
help="The list of CHASM drivers (drivers.tmps)")
parser.add_argument("-o", dest="prefix", metavar="PREFIX",
help="Output prefix.")
args, logger = cmd.parse_args("perf-cosmic")
prefix = args.prefix or "cosmic-"
fanns_db = cmd.open_db()
try:
snvs = dict()
logger.info("Counting the number of samples per mutation ...")
with tsv.open(args.data_path, "r") as df:
columns = [
#"Genome-wide screen",
"Mutation Description",
"Mutation CDS",
"Mutation AA",
"Mutation GRCh37 genome position",
"Mutation GRCh37 strand",
"Accession Number",
"ID_sample"]
total_rows = queried_rows = 0
for fields in tsv.rows(df, columns=columns, header=True):
total_rows += 1
#wide_screen, mut_desc, mut_cds, mut_aa, mut_pos, mut_strand, acc, sample_id = fields
mut_desc, mut_cds, mut_aa, mut_pos, mut_strand, acc, sample_id = fields
wide_screen = "y"
if wide_screen != "y" or mut_desc != "Substitution - Missense":
continue
queried_rows += 1
for row in get_transcripts(fanns_db, mut_cds, mut_aa, mut_pos, mut_strand, acc, logger):
k = tuple([row[k] for k in ["protein", "aa_pos", "aa_ref", "aa_alt"]])
if k not in snvs:
symbol = row["xrefs"]["symbol"]
snvs[k] = dict(
transcript=row["transcript"],
symbol=symbol,
samples=set([sample_id]))
else:
snvs[k]["samples"].add(sample_id)
logger.info("Total: total_rows={}, queried_rows={}, protein_changes={}".format(total_rows, queried_rows, len(snvs)))
logger.info("Loading CGC genes ...")
cgc_genes = set()
with open(args.cgc_path, "r") as f:
for line in f:
cgc_genes.add(line.rstrip("\n"))
logger.info("Loading CHASM drivers ...")
drivers = set()
with open(args.drivers_path, "r") as f:
for line in f:
drivers.add(line.rstrip("\n").split("\t")[0])
logger.info("Creating datasets ...")
progress = RatedProgress(logger, name="mutations")
with Dataset(prefix + "1") as rec1,\
Dataset(prefix + "2") as rec2,\
Dataset(prefix + "4") as rec4,\
Dataset(prefix + "CGC") as cgc,\
Dataset(prefix + "noCGC") as nocgc,\
Dataset(prefix + "D") as drv,\
Dataset(prefix + "O") as nodrv:
for (protein, aa_pos, aa_ref, aa_alt), snv in snvs.items():
num_samples = len(snv["samples"])
line = "\t".join([str(v) for v in [protein, aa_pos, aa_ref, aa_alt]])
if num_samples == 1:
rec1.write(line)
if num_samples >= 2:
rec2.write(line)
if num_samples >= 4:
rec4.write(line)
symbol = snv["symbol"]
if symbol is not None and ((isinstance(symbol, basestring) and symbol in cgc_genes) or len(set(symbol) & cgc_genes) > 0):
cgc.write(line)
elif num_samples == 1:
nocgc.write(line)
if snv["transcript"] in drivers:
drv.write(line)
elif num_samples == 1:
nodrv.write(line)
progress.update()
progress.log_totals()
logger.info("Datasets: {}".format(", ".join(["{}={}".format(os.path.basename(d.name), d.size) for d in [
rec1, rec2, rec4, cgc, nocgc, drv, nodrv]])))
except:
cmd.handle_error()
return 0
