def combine_calls(batch_id, samples, data):
"""Combine multiple callsets into a final set of merged calls.
"""
logger.info("Ensemble consensus calls for {0}: {1}".format(
batch_id, ",".join(x["variantcaller"] for x in samples[0]["variants"])))
edata = copy.deepcopy(data)
base_dir = utils.safe_makedir(os.path.join(edata["dirs"]["work"], "ensemble", batch_id))
caller_names, vrn_files, bam_files = _organize_variants(samples, batch_id)
exist_variants = False
for tmp_vrn_file in vrn_files:
if vcfutils.vcf_has_variants(tmp_vrn_file):
exist_variants = True
break
if exist_variants:
if "classifiers" not in edata["config"]["algorithm"]["ensemble"]:
callinfo = _run_ensemble_intersection(batch_id, vrn_files, base_dir, edata)
else:
config_file = _write_config_file(batch_id, caller_names, base_dir, edata)
callinfo = _run_ensemble(batch_id, vrn_files, config_file, base_dir,
edata["sam_ref"], edata)
callinfo["vrn_file"] = vcfutils.bgzip_and_index(callinfo["vrn_file"], data["config"])
edata["config"]["algorithm"]["variantcaller"] = "ensemble"
edata["vrn_file"] = callinfo["vrn_file"]
edata["ensemble_bed"] = callinfo["bed_file"]
callinfo["validate"] = validate.compare_to_rm(edata)[0][0].get("validate")
else:
out_vcf_file = os.path.join(base_dir, "{0}-ensemble.vcf".format(batch_id))
vcfutils.write_empty_vcf(out_vcf_file)
callinfo = {"variantcaller": "ensemble",
"vrn_file": vcfutils.bgzip_and_index(out_vcf_file, data["config"]),
"bed_file": None}
return [[batch_id, callinfo]]
def combine_calls(batch_id, samples, data):
"""Combine multiple callsets into a final set of merged calls.
"""
logger.info("Ensemble consensus calls for {0}: {1}".format(
batch_id, ",".join(x["variantcaller"] for x in samples[0]["variants"])))
edata = copy.deepcopy(data)
base_dir = utils.safe_makedir(os.path.join(edata["dirs"]["work"], "ensemble", batch_id))
caller_names, vrn_files, bam_files = _organize_variants(samples, batch_id)
exist_variants = False
for tmp_vrn_file in vrn_files:
if vcfutils.vcf_has_variants(tmp_vrn_file):
exist_variants = True
break
if exist_variants:
if "classifiers" not in edata["config"]["algorithm"]["ensemble"]:
callinfo = _run_ensemble_intersection(batch_id, vrn_files, base_dir, edata)
else:
config_file = _write_config_file(batch_id, caller_names, base_dir, edata)
callinfo = _run_ensemble(batch_id, vrn_files, config_file, base_dir,
edata["sam_ref"], edata)
edata["config"]["algorithm"]["variantcaller"] = "ensemble"
edata["vrn_file"] = callinfo["vrn_file"]
edata["ensemble_bed"] = callinfo["bed_file"]
callinfo["validate"] = validate.compare_to_rm(edata)[0][0].get("validate")
else:
out_vcf_file = os.path.join(base_dir, "{0}-ensemble.vcf".format(batch_id))
vcfutils.write_empty_vcf(out_vcf_file)
callinfo = {"variantcaller": "ensemble",
"vrn_file": out_vcf_file,
"bed_file": None}
return [[batch_id, callinfo]]
def combine_calls(*args):
"""Combine multiple callsets into a final set of merged calls.
"""
if len(args) == 3:
is_cwl = False
batch_id, samples, data = args
caller_names, vrn_files = _organize_variants(samples, batch_id)
else:
is_cwl = True
samples = [utils.to_single_data(x) for x in args]
samples = [cwlutils.unpack_tarballs(x, x) for x in samples]
data = samples[0]
batch_id = data["batch_id"]
caller_names = data["variants"]["variantcallers"]
vrn_files = data["variants"]["calls"]
logger.info("Ensemble consensus calls for {0}: {1}".format(
batch_id, ",".join(caller_names)))
edata = copy.deepcopy(data)
base_dir = utils.safe_makedir(os.path.join(edata["dirs"]["work"], "ensemble", batch_id))
if any([vcfutils.vcf_has_variants(f) for f in vrn_files]):
# Decompose multiallelic variants and normalize
passonly = not tz.get_in(["config", "algorithm", "ensemble", "use_filtered"], edata, False)
vrn_files = [normalize.normalize(f, data, passonly=passonly, rerun_effects=False, remove_oldeffects=True,
nonrefonly=True,
work_dir=utils.safe_makedir(os.path.join(base_dir, c)))
for c, f in zip(caller_names, vrn_files)]
if "classifiers" not in (dd.get_ensemble(edata) or {}):
callinfo = _run_ensemble_intersection(batch_id, vrn_files, caller_names, base_dir, edata)
else:
config_file = _write_config_file(batch_id, caller_names, base_dir, edata)
callinfo = _run_ensemble(batch_id, vrn_files, config_file, base_dir,
dd.get_ref_file(edata), edata)
callinfo["vrn_file"] = vcfutils.bgzip_and_index(callinfo["vrn_file"], data["config"])
# After decomposing multiallelic variants and normalizing, re-evaluate effects
ann_ma_file, _ = effects.add_to_vcf(callinfo["vrn_file"], data)
if ann_ma_file:
callinfo["vrn_file"] = ann_ma_file
edata["config"]["algorithm"]["variantcaller"] = "ensemble"
edata["vrn_file"] = callinfo["vrn_file"]
edata["ensemble_bed"] = callinfo["bed_file"]
callinfo["validate"] = validate.compare_to_rm(edata)[0][0].get("validate")
else:
out_vcf_file = os.path.join(base_dir, "{0}-ensemble.vcf".format(batch_id))
vcfutils.write_empty_vcf(out_vcf_file, samples=[dd.get_sample_name(d) for d in samples])
callinfo = {"variantcaller": "ensemble",
"vrn_file": vcfutils.bgzip_and_index(out_vcf_file, data["config"]),
"bed_file": None}
if is_cwl:
callinfo["batch_samples"] = data["batch_samples"]
callinfo["batch_id"] = batch_id
return [{"ensemble": callinfo}]
else:
return [[batch_id, callinfo]]
def postprocess_variants(data):
"""Provide post-processing of variant calls.
"""
logger.info("Finalizing variant calls: %s" % str(data["name"]))
if data["work_bam"]:
data["vrn_file"] = finalize_genotyper(data["vrn_file"], data["work_bam"],
data["sam_ref"], data["config"])
logger.info("Calculating variation effects for %s" % str(data["name"]))
ann_vrn_file = effects.snpeff_effects(data["vrn_file"], data["genome_build"],
data["config"])
if ann_vrn_file:
data["vrn_file"] = ann_vrn_file
data = validate.compare_to_rm(data)
return [[data]]
def postprocess_variants(data):
"""Provide post-processing of variant calls.
"""
logger.info("Finalizing variant calls: %s" % str(data["name"]))
if data["work_bam"]:
data["vrn_file"] = finalize_genotyper(data["vrn_file"],
data["work_bam"],
data["sam_ref"], data["config"])
logger.info("Calculating variation effects for %s" % str(data["name"]))
ann_vrn_file = effects.snpeff_effects(data["vrn_file"],
data["genome_build"],
data["config"])
if ann_vrn_file:
data["vrn_file"] = ann_vrn_file
data = validate.compare_to_rm(data)
return [[data]]
def combine_calls(batch_id, samples, data):
"""Combine multiple callsets into a final set of merged calls.
"""
logger.info("Ensemble consensus calls for {0}: {1}".format(
batch_id, ",".join(x["variantcaller"] for x in samples[0]["variants"])))
edata = copy.deepcopy(data)
base_dir = utils.safe_makedir(os.path.join(edata["dirs"]["work"], "ensemble", batch_id))
caller_names, vrn_files = _organize_variants(samples, batch_id)
config_file = _write_config_file(batch_id, caller_names, base_dir, edata)
callinfo = _run_ensemble(batch_id, vrn_files, config_file, base_dir,
edata["sam_ref"], edata["config"])
edata["config"]["algorithm"]["variantcaller"] = "ensemble"
edata["vrn_file"] = callinfo["vrn_file"]
edata["ensemble_bed"] = callinfo["bed_file"]
callinfo["validate"] = validate.compare_to_rm(edata)[0][0].get("validate")
return [[batch_id, callinfo]]
def combine_calls(data):
"""Combine multiple callsets into a final set of merged calls.
"""
if _has_ensemble(data):
logger.info("Ensemble consensus calls for {0}: {1}".format(
",".join(x["variantcaller"] for x in data["variants"]), data["work_bam"]))
edata = copy.deepcopy(data)
edata["config"]["algorithm"]["variantcaller"] = "ensemble"
sample = edata["name"][-1].replace(" ", "_")
base_dir = utils.safe_makedir(os.path.join(edata["dirs"]["work"], "ensemble"))
config_file = _write_config_file(edata, sample, base_dir, "ensemble")
callinfo = _run_bcbio_variation(config_file, base_dir, sample, edata)
from bcbio.variation import validate
edata["vrn_file"] = callinfo["vrn_file"]
edata["ensemble_bed"] = callinfo["bed_file"]
callinfo["validate"] = validate.compare_to_rm(edata)[0][0].get("validate")
data["variants"].insert(0, callinfo)
_write_config_file(data, sample, base_dir, "compare")
return [[data]]
def combine_calls(batch_id, samples, data):
"""Combine multiple callsets into a final set of merged calls.
"""
logger.info("Ensemble consensus calls for {0}: {1}".format(
batch_id, ",".join(x["variantcaller"] for x in samples[0]["variants"])))
edata = copy.deepcopy(data)
base_dir = utils.safe_makedir(os.path.join(edata["dirs"]["work"], "ensemble", batch_id))
caller_names, vrn_files, bam_files = _organize_variants(samples, batch_id)
exist_variants = False
for tmp_vrn_file in vrn_files:
if vcfutils.vcf_has_variants(tmp_vrn_file):
exist_variants = True
break
if exist_variants:
# Decompose multiallelic variants and normalize
passonly = not tz.get_in(["config", "algorithm", "ensemble", "use_filtered"], edata, False)
vrn_files = [normalize.normalize(f, data, passonly=passonly, rerun_effects=False, remove_oldeffects=True)
for f in vrn_files]
if "classifiers" not in edata["config"]["algorithm"]["ensemble"]:
callinfo = _run_ensemble_intersection(batch_id, vrn_files, caller_names, base_dir, edata)
else:
config_file = _write_config_file(batch_id, caller_names, base_dir, edata)
callinfo = _run_ensemble(batch_id, vrn_files, config_file, base_dir,
edata["sam_ref"], edata)
callinfo["vrn_file"] = vcfutils.bgzip_and_index(callinfo["vrn_file"], data["config"])
# After decomposing multiallelic variants and normalizing, re-evaluate effects
ann_ma_file, _ = effects.add_to_vcf(callinfo["vrn_file"], data)
if ann_ma_file:
callinfo["vrn_file"] = ann_ma_file
edata["config"]["algorithm"]["variantcaller"] = "ensemble"
edata["vrn_file"] = callinfo["vrn_file"]
edata["ensemble_bed"] = callinfo["bed_file"]
callinfo["validate"] = validate.compare_to_rm(edata)[0][0].get("validate")
else:
out_vcf_file = os.path.join(base_dir, "{0}-ensemble.vcf".format(batch_id))
vcfutils.write_empty_vcf(out_vcf_file, samples=[dd.get_sample_name(d) for d in samples])
callinfo = {"variantcaller": "ensemble",
"vrn_file": vcfutils.bgzip_and_index(out_vcf_file, data["config"]),
"bed_file": None}
return [[batch_id, callinfo]]
def combine_calls(data):
"""Combine multiple callsets into a final set of merged calls.
"""
if _has_ensemble(data):
logger.info("Ensemble consensus calls for {0}: {1}".format(
",".join(x["variantcaller"] for x in data["variants"]),
data["work_bam"]))
edata = copy.deepcopy(data)
edata["config"]["algorithm"]["variantcaller"] = "ensemble"
sample = edata["name"][-1].replace(" ", "_")
base_dir = utils.safe_makedir(
os.path.join(edata["dirs"]["work"], "ensemble"))
config_file = _write_config_file(edata, sample, base_dir, "ensemble")
callinfo = _run_bcbio_variation(config_file, base_dir, sample, edata)
from bcbio.variation import validate
edata["vrn_file"] = callinfo["vrn_file"]
edata["ensemble_bed"] = callinfo["bed_file"]
callinfo["validate"] = validate.compare_to_rm(edata)[0][0].get(
"validate")
data["variants"].insert(0, callinfo)
_write_config_file(data, sample, base_dir, "compare")
return [[data]]
def compare_to_rm(*args):
return validate.compare_to_rm(*args)
def compare_to_rm(*args):
return validate.compare_to_rm(*args)
