def find_cluster_files(cluster_path):
# Return a dictionary of:
# "cdt" : cdt_filename
# "atr" : atr_filename
# "gtr" : gtr_filename
# "kag" : kag_filename
# "kgg" : kgg_filename
# Any of these files can be missing.
import os
from genomicode import filelib
filelib.assert_exists(cluster_path)
opj = os.path.join
cdt = opj(cluster_path, "signal.cdt")
atr = opj(cluster_path, "array_tree.atr")
gtr = opj(cluster_path, "gene_tree.gtr")
kag = opj(cluster_path, "array_cluster.kag")
kgg = opj(cluster_path, "gene_cluster.kgg")
cluster_files = {}
if filelib.exists_nz(cdt):
cluster_files["cdt"] = cdt
if filelib.exists_nz(atr):
cluster_files["atr"] = atr
if filelib.exists_nz(gtr):
cluster_files["gtr"] = gtr
if filelib.exists_nz(kag):
cluster_files["kag"] = kag
if filelib.exists_nz(kgg):
cluster_files["kgg"] = kgg
assert "cdt" in cluster_files, "No clustered file."
return cluster_files
def get_config(name,
which_assert_file=False,
assert_exists=False,
quote=False):
from genomicode import filelib
from genomicode import config
assert hasattr(config, name), "Not configured for genomicode: %s" % name
x = getattr(config, name)
if which_assert_file:
x = filelib.which_assert(x)
elif assert_exists:
filelib.assert_exists(x)
if quote:
x = sq(x)
return x
def run(self, network, in_data, out_attributes, user_options, num_cores,
out_filename):
from genomicode import filelib
vcf_node = in_data
# Some callers, like jointsnvmix, will create vcf files for
# each chromosome. To avoid picking these up, only accept
# .vcf files from the top level.
vcf_filenames = filelib.list_files_in_path(vcf_node.identifier,
endswith=".vcf",
toplevel_only=True)
assert vcf_filenames, "No .vcf files: %s" % vcf_node.identifier
metadata = {}
tmp_path = "indexed.vcf"
m = merge_vcf_files(vcf_filenames, out_filename, num_cores, tmp_path)
metadata.update(m)
filelib.assert_exists(out_filename) # may be size 0
return metadata
def _run_filterRadia_with_restart(cmd, cancer_sample, chrom, logfile):
# Sometimes samtools crashes in the middle of a run. Detect this
# case, and re-run the analysis if needed.
from genomicode import parallel
from genomicode import filelib
num_tries = 0
while num_tries <= 3:
num_tries += 1
parallel.sshell(cmd, ignore_nonzero_exit=True)
filelib.assert_exists(logfile)
log = open(logfile).read()
# Empty logfile means cmd completed successfully.
if not log.strip():
break
# Look for evidence that samtools died. If this occurs, try again.
# 06/29/2016 09:57:16 AM ERROR The return code of '1' from the
# following filter command indicates an error.
# 06/29/2016 09:57:16 AM ERROR Error from /usr/bin/python
# /usr/local/radia/scripts/createBlatFile.pyc 196C-lung2
# radia2.tmp/196C-lung2_dnaFiltered_chr1.vcf
# radia2.tmp/196C-lung2_mpileup_rna_origin_chr1.vcf
# -o radia2.tmp/196C-lung2_blatInput_chr1.fa
# --allVCFCalls --blatRnaNormalReads --blatRnaTumorReads:
# <Traceback>
# [...]
# samtoolsCall.kill()
# [...]
# OSError: [Errno 3] No such process
if log.find("samtoolsCall.kill") >= 0 \
and log.find("No such process") >= 0:
continue
# Otherwise, the process failed for some other reason. Raise
# an exception.
raise AssertionError, "Problem filtering: %s %s\n%s" % (cancer_sample,
chrom, log)
def run(self, network, antecedents, out_attributes, user_options,
num_cores, out_path):
import os
from genomicode import filelib
from genomicode import alignlib
from genomicode import parallel
from genomicode import hashlib
from Betsy import module_utils as mlib
fastq_node, sample_node, strand_node, reference_node = antecedents
fastq_files = mlib.find_merged_fastq_files(sample_node.identifier,
fastq_node.identifier)
assert fastq_files, "I could not find any FASTQ files."
ref = alignlib.create_reference_genome(reference_node.identifier)
stranded = mlib.read_stranded(strand_node.identifier)
filelib.safe_mkdir(out_path)
metadata = {}
metadata["tool"] = "RSEM %s" % alignlib.get_rsem_version()
# Figure out whether to align to genome or transcriptome.
x = out_attributes["align_to"]
assert x in ["genome", "transcriptome"]
align_to_genome = (x == "genome")
# RSEM makes files:
# <sample_name>.genome.bam
# <sample_name>.transcript.bam
# <sample_name>.genes.results
# <sample_name>.isoforms.results
# <sample_name>.stat
#
# Does not work right if there is a space in the sample name.
# Therefore, give a hashed sample name, and then re-name
# later.
# Make a list of the jobs to run.
jobs = []
for x in fastq_files:
sample, pair1, pair2 = x
sample_h = hashlib.hash_var(sample)
x1, x2, x3 = mlib.splitpath(pair1)
x = "%s%s" % (hashlib.hash_var(x2), x3)
pair1_h = os.path.join(out_path, x)
if pair2:
x1, x2, x3 = mlib.splitpath(pair2)
x = "%s%s" % (hashlib.hash_var(x2), x3)
pair2_h = os.path.join(out_path, x)
results_filename = os.path.join(out_path,
"%s.genes.results" % sample)
log_filename = os.path.join(out_path, "%s.log" % sample)
x = filelib.GenericObject(sample=sample,
sample_h=sample_h,
pair1=pair1,
pair2=pair2,
pair1_h=pair1_h,
pair2_h=pair2_h,
results_filename=results_filename,
log_filename=log_filename)
jobs.append(x)
# Make sure hashed samples are unique.
seen = {}
for j in jobs:
assert j.sample_h not in seen, \
"Dup (%d): %s" % (len(jobs), j.sample_h)
assert j.pair1_h not in seen
assert j.pair2_h not in seen
seen[j.sample_h] = 1
seen[j.pair1_h] = 1
seen[j.pair2_h] = 1
# Symlink the fastq files.
for j in jobs:
os.symlink(j.pair1, j.pair1_h)
if j.pair2:
os.symlink(j.pair2, j.pair2_h)
s2fprob = {
"unstranded": None,
"firststrand": 0.0,
"secondstrand": 1.0,
}
assert stranded.stranded in s2fprob, "Unknown stranded: %s" % \
stranded.stranded
forward_prob = s2fprob[stranded.stranded]
# How much memory for bowtie. May need to increase this if
# there are lots of memory warnings in the log files:
# Warning: Exhausted best-first chunk memory for read
# ST-J00106:110:H5NY5BBXX:6:1101:18203:44675 1:N:0:1/1
# (patid 2076693); skipping read
# Default is 64.
# Seems like too high a value can cause problems.
#chunkmbs = 4*1024 # Generates warnings.
chunkmbs = 512
# Get lots of warnings with bowtie:
# Warning: Detected a read pair whose two mates have different names
# Use STAR aligner instead.
use_STAR = True
sq = parallel.quote
commands = []
for j in jobs:
# Debug: If the results file exists, don't run it again.
if filelib.exists_nz(j.results_filename) and \
filelib.exists(j.log_filename):
continue
# If using the STAR aligner, then most memory efficient
# way is to let STAR take care of the multiprocessing.
nc = max(1, num_cores / len(jobs))
if use_STAR:
nc = num_cores
keywds = {}
if use_STAR:
keywds["align_with_star"] = True
else:
keywds["align_with_bowtie2"] = True
x = alignlib.make_rsem_command(ref.fasta_file_full,
j.sample_h,
j.pair1_h,
fastq_file2=j.pair2_h,
forward_prob=forward_prob,
output_genome_bam=align_to_genome,
bowtie_chunkmbs=chunkmbs,
num_threads=nc,
**keywds)
x = "%s >& %s" % (x, sq(j.log_filename))
commands.append(x)
metadata["commands"] = commands
metadata["num cores"] = num_cores
# Need to run in out_path. Otherwise, files will be everywhere.
nc = num_cores
if use_STAR:
nc = 1
parallel.pshell(commands, max_procs=nc, path=out_path)
# Rename the hashed sample names back to the original unhashed
# ones.
files = os.listdir(out_path)
rename_files = [] # list of (src, dst)
for j in jobs:
if j.sample == j.sample_h:
continue
for f in files:
if not f.startswith(j.sample_h):
continue
src = os.path.join(out_path, f)
x = j.sample + f[len(j.sample_h):]
dst = os.path.join(out_path, x)
rename_files.append((src, dst))
for src, dst in rename_files:
filelib.assert_exists(src)
os.rename(src, dst)
# Delete the symlinked fastq files.
for j in jobs:
filelib.safe_unlink(j.pair1_h)
filelib.safe_unlink(j.pair2_h)
# Make sure the analysis completed successfully.
x1 = [x.results_filename for x in jobs]
x2 = [x.log_filename for x in jobs]
filelib.assert_exists_nz_many(x1 + x2)
return metadata
def run(self, network, antecedents, out_attributes, user_options,
num_cores, out_path):
import os
import zipfile
#import subprocess
import arrayio
from genomicode import parallel
from genomicode import filelib
from Betsy import module_utils as mlib
#from genomicode import config
in_data = antecedents
if not os.path.exists(out_path):
os.mkdir(out_path)
metadata = {}
module_name = 'IlluminaExpressionFileCreator'
params = {}
if zipfile.is_zipfile(in_data.identifier):
params['idat.zip'] = in_data.identifier
else:
# Add ".zip" to the end of the file.
zipfile_name = os.path.split(in_data.identifier)[-1] + '.zip'
zip_directory(in_data.identifier, zipfile_name)
params['idat.zip'] = os.path.join(".", zipfile_name)
x = user_options.get("illu_manifest", MP.DEFAULT_MANIFEST)
assert x in MP.ILLU_MANIFEST, "Unknown manifest: %s" % x
params['manifest'] = x
x = user_options.get("illu_chip", MP.DEFAULT_CHIP)
assert x in MP.ILLU_CHIP, "Unknown chip: %s" % x
params['chip'] = x
x = user_options.get("illu_bg_mode")
if x is not None:
assert x in ['true', 'false'], \
'illu_bg_mode should be true or false'
params['background.subtraction.mode'] = x
x = user_options.get("illu_coll_mode")
if x is not None:
assert x in ['none', 'max', 'median'], \
'ill_coll_mode is not correct'
params['collapse.mode'] = str(x)
if 'illu_clm' in user_options:
params['clm'] = str(user_options['illu_clm'])
if 'illu_custom_chip' in user_options:
params['chip'] = str(user_options['illu_custom_chip'])
if 'illu_custom_manifest' in user_options:
params['custom.manifest'] = str(
user_options['illu_custom_manifest'])
gp_module = mlib.get_config("genepattern", which_assert_file=True)
download_directory = 'illumina_result'
sq = parallel.quote
cmd = [
sq(gp_module),
module_name,
'-o',
sq(download_directory),
]
for key in params.keys():
x = ['--parameters', key + ':' + params[key]]
cmd.extend(x)
cmd = " ".join(cmd)
parallel.sshell(cmd)
metadata["commands"] = [cmd]
filelib.assert_exists(download_directory)
result_files = os.listdir(download_directory)
#assert 'stderr.txt' not in result_files,('gene_pattern get error '+
# 'The contents of stderr.txt is:'+
# file(os.path.join(download_directory,'stderr.txt')).read())
for result_file in result_files:
if result_file == 'System.out':
continue
# BUG: What if there are duplicate sample names?
M = arrayio.read(os.path.join(download_directory, result_file))
a = M._col_names['_SAMPLE_NAME']
b = sorted(a)
index = []
for i in b:
index.append(a.index(i))
M_new = M.matrix(None, index)
result_path = os.path.join(out_path, result_file)
f = file(result_path, 'w')
arrayio.gct_format.write(M_new, f)
f.close()
return metadata
def run(self, network, antecedents, out_attributes, user_options,
num_cores, out_path):
import os
from genomicode import filelib
from genomicode import parallel
from genomicode import vcflib
from Betsy import module_utils as mlib
vcf_node, nc_node = antecedents
vcf_filenames = filelib.list_files_in_path(vcf_node.identifier,
endswith=".vcf")
assert vcf_filenames, "No .vcf files."
nc_match = mlib.read_normal_cancer_file(nc_node.identifier)
filelib.safe_mkdir(out_path)
metadata = {}
# Filenames:
# <caller>.vcf
wgs_or_wes = mlib.get_user_option(user_options,
"wgs_or_wes",
not_empty=True,
allowed_values=["wgs", "wes"])
genome = mlib.get_user_option(user_options,
"snpeff_genome",
not_empty=True)
databases = list_snpeff_databases()
assert genome in databases, "Unknown genome database: %s" % genome
# For each caller, do the SnpEFF calls. Some callers include
# the somatic information, others do not. If germline samples
# are present, then do with _cancer. Otherwise, do not.
# java -Xmx16g -jar $SNPEFF -v -cancer -cancerSamples vcf03.txt
# GRCh37.75 vcf02.txt 1> test03.txt 2> test03.log
# Don't bother annotating positions that do not pass filter.
# Filter them out first based on FILTER column.
opj = os.path.join
jobs = []
for in_filename in vcf_filenames:
path, stem, ext = mlib.splitpath(in_filename)
samples_file = opj(out_path, "%s.cancerSamples.txt" % stem)
filtered_filename = opj(out_path, "%s.filtered_input" % stem)
out_filename = opj(out_path, "%s.vcf" % stem)
log_filename = opj(out_path, "%s.log" % stem)
x = filelib.GenericObject(in_filename=in_filename,
samples_file=samples_file,
filtered_filename=filtered_filename,
out_filename=out_filename,
log_filename=log_filename)
jobs.append(x)
# First, filter each of the VCF files.
commands = []
for j in jobs:
# For debugging. If this file exists, don't filter it again.
if os.path.exists(j.filtered_filename):
continue
args = j.in_filename, j.filtered_filename, wgs_or_wes
x = vcflib.filter_vcf_file, args, {}
commands.append(x)
parallel.pyfun(commands, num_procs=num_cores)
# Make the cancer_samples files.
for j in jobs:
# Will generate this if there are cancer samples.
make_cancer_samples_file(j.filtered_filename, nc_match,
j.samples_file)
# Make a list of commands.
commands = []
for j in jobs:
cancer = False
if os.path.exists(j.samples_file):
cancer = True
x = make_snpeff_command(j.filtered_filename,
genome,
j.out_filename,
j.log_filename,
is_cancer=cancer,
cancer_samples_file=j.samples_file)
commands.append(x)
nc = mlib.calc_max_procs_from_ram(16, upper_max=num_cores)
parallel.pshell(commands, max_procs=nc)
metadata["commands"] = commands
metadata["num_cores"] = nc
# Make sure the analysis completed successfully.
x = [x.out_filename for x in jobs]
filelib.assert_exists_nz_many(x)
# Log files should be empty.
for j in jobs:
filelib.assert_exists(j.log_filename)
assert not filelib.exists_nz(j.log_filename), \
"Error with %s.\n%s" % (j.stem, j.log_filename)
filelib.safe_unlink(j.log_filename)
return metadata
def run(self, network, antecedents, out_attributes, user_options,
num_cores, outfile):
import os
import subprocess
#from Betsy import read_label_file
from genomicode import filelib
from genomicode import config
#from genomicode import arraysetlib
cls_node_train, data_node_train = antecedents
#result, label_line, class_name = read_label_file.read(
# cls_node_train.identifier)
#x = arraysetlib.read_cls_file(cls_node_train.identifier)
#class_names, classes = x
metadata = {}
module_name = 'WeightedVotingXValidation'
module_id_version = '00028:2'
gp_params = dict()
gp_params['data.filename'] = data_node_train.identifier
gp_params['class.filename'] = cls_node_train.identifier
if 'wv_num_features' in user_options:
gp_params['num.features'] = str(user_options['wv_num_features'])
## if 'wv_minstd' in user_input:
## assert module_utils.is_number(
## user_input['wv_minstd']), 'the sv_minstd should be number'
## gp_parameters['min.std'] = str(user_input['wv_minstd'])
##
## wv_feature_stat = ['wv_snr', 'wv_ttest', 'wv_snr_median',
## 'wv_ttest_median', 'wv_snr_minstd',
## 'wv_ttest_minstd', 'wv_snr_median_minstd',
## 'wv_ttest_median_minstd']
##
## assert parameters['wv_feature_stat'] in wv_feature_stat, (
## 'the wv_feature_stat is invalid')
## gp_parameters['feature.selection.statistic'] = str(
## wv_feature_stat.index(parameters[
## 'wv_feature_stat']))
gp_path = config.genepattern
gp_module = filelib.which(gp_path)
assert gp_module, 'cannot find the %s' % gp_path
download_directory = os.path.join(".", 'wv_result')
command = [
gp_module,
module_name,
'--id_and_version',
module_id_version,
'-o',
download_directory,
]
for key in gp_params:
x = ['--parameters', "%s:%s" % (key, gp_params[key])]
command.extend(x)
x = " ".join(map(str, x))
metadata["commands"] = [x]
# DEBUG: If this is already run, don't run it again.
if not os.path.exists(download_directory):
process = subprocess.Popen(command,
shell=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
process.wait()
error_message = process.communicate()[1]
# Ignore warning:
# /opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/
# python2.7/site-packages/rpy2/rinterface/__init__.py:185:
# RRuntimeWarning: Loading required package: rJava
#
# warnings.warn(x, RRuntimeWarning)
x = error_message.strip()
if x.endswith("warnings.warn(x, RRuntimeWarning)"):
pass
else:
assert not x, error_message
filelib.assert_exists(download_directory)
# Find the prediction file.
x = os.listdir(download_directory)
assert 'stderr.txt' not in x, 'gene_pattern get error'
x = [x for x in x if x.endswith("pred.odf")]
assert x, "Missing predictions file"
assert len(x) == 1, "Too many prediction files: %s" % repr(x)
gp_file = x[0]
gp_file = os.path.join(download_directory, gp_file)
text = open(gp_file).readlines()
#os.rename(os.path.join(download_directory, gp_file),
# os.path.join(download_directory, 'prediction.odf'))
assert text[1][0:12] == 'HeaderLines='
start = int(text[1][12:-1])
newresult = [[
'Sample_name', 'Predicted_class', 'Confidence', 'Actual_class',
'Correct?'
]]
for i in text[start + 2:]:
line = i.split()
n = len(line)
newline = [
' '.join(line[0:n - 4]), line[n - 3], line[n - 2], line[n - 4],
line[n - 1]
]
newresult.append(newline)
f = file(outfile, 'w')
for i in newresult:
f.write('\t'.join(i))
f.write('\n')
f.close()
return metadata
def main():
import os
import argparse
import itertools
from genomicode import filelib
from genomicode import config
from genomicode import parallel
from genomicode import alignlib
parser = argparse.ArgumentParser(description="")
parser.add_argument("reference_genome", help="fasta file")
parser.add_argument("-j",
dest="num_procs",
type=int,
default=1,
help="Number of jobs to run in parallel.")
parser.add_argument(
"--dry_run",
action="store_true",
help="Just display the commands, and don't generate the alignment.")
parser.add_argument("--window",
default=80,
type=int,
help="Number of bases in alignment. Default: 80")
group = parser.add_argument_group(title="Input")
group.add_argument("--bam_file", help="Indexed BAM file.")
group.add_argument("--bam_path", help="Path to BAM files.")
group.add_argument(
"--position",
action="append",
default=[],
help="Specify a position to view, "
"e.g. chr20:45,927,663 or chr20:45927663. 1-based coordinates")
group.add_argument("--position_file",
help="Tab-delimited text file with two columns. "
"Column 1 is chromosome, column 2 is position.")
group = parser.add_argument_group(title="Output")
group.add_argument("--prefix", help="Pre-pend a prefix to each outfile.")
group.add_argument(
"--outpath",
help="If multiple alignments are generated, this option "
"directs where to save the output files.")
group.add_argument(
"--noclobber",
action="store_true",
help="If an output file already exists, don't overwrite it.")
# Parse the input arguments.
args = parser.parse_args()
filelib.assert_exists_nz(args.reference_genome)
assert args.bam_file or args.bam_path, \
"Either --bam_file or --bam_path must be provided."
assert not (args.bam_file and args.bam_path), \
"Cannot specify both --bam_file or --bam_path."
if args.bam_file:
filelib.assert_exists_nz(args.bam_file)
if args.bam_path:
assert os.path.exists(args.bam_path)
if args.position_file:
filelib.assert_exists_nz(args.position_file)
if args.outpath and not os.path.exists(args.outpath):
os.mkdir(args.outpath)
if args.num_procs < 1 or args.num_procs > 100:
parser.error("Please specify between 1 and 100 processes.")
assert args.window >= 1 and args.window < 500
bam_filenames = []
if args.bam_file:
bam_filenames.append(args.bam_file)
else:
x = os.listdir(args.bam_path)
x = [x for x in x if x.endswith(".bam")]
x = [os.path.join(args.bam_path, x) for x in x]
bam_filenames = x
assert bam_filenames, "No bam files found."
positions = [] # list of (chrom, pos)
for x in args.position:
chrom, pos = _parse_position(x)
positions.append((chrom, pos))
if args.position_file and os.path.exists(args.position_file):
for cols in filelib.read_cols(args.position_file):
assert len(cols) == 2, "Position file should have 2 columns"
chrom, pos = cols
pos = int(pos)
assert pos >= 1
positions.append((chrom, pos))
assert positions, "No positions specified."
# Make the commands.
assert hasattr(config, "samtools")
filelib.assert_exists(config.samtools)
# Make sure we have the right version of samtools.
# 1.2 (using htslib 1.2.1)
# 0.1.18 (r982:295)
version = alignlib.get_samtools_version()
x = version.split(".")
assert len(x) >= 2
major = x[0]
assert major in ["0", "1"], "Unknown samtools version: %s" % version
major = int(major)
assert major >= 1, "Requires samtools >= 1 (Current version: %s)" % version
commands = []
for x in itertools.product(bam_filenames, positions):
bam_filename, (chrom, pos) = x
p, f = os.path.split(bam_filename)
sample, e = os.path.splitext(f)
left = max(pos - args.window / 2, 1)
pos_str = "%s:%s" % (chrom, left)
x = "%2s.%9s.%s.html" % (chrom, pos, sample)
if args.prefix:
x = "%s.%s" % (args.prefix, x)
if args.outpath:
x = os.path.join(args.outpath, x)
out_filename = x
if args.noclobber and os.path.exists(out_filename):
continue
# samtools tview -d t -p 7:100550778 bam01/196B-lung.bam $FA
sq = parallel.quote
x = [
sq(config.samtools),
"tview",
"-d",
"h",
"-p",
pos_str,
sq(bam_filename),
sq(args.reference_genome),
]
x = " ".join(x)
x = "%s >& %s" % (x, sq(out_filename))
commands.append(x)
if args.dry_run:
for x in commands:
print x
return
parallel.pshell(commands, max_procs=args.num_procs)
def run(self, network, antecedents, out_attributes, user_options,
num_cores, out_path):
import os
import shutil
from genomicode import filelib
from genomicode import parallel
from genomicode import parselib
from Betsy import module_utils as mlib
mpileup_node, nc_node = antecedents
mpileup_filenames = filelib.list_files_in_path(mpileup_node.identifier,
endswith=".pileup")
assert mpileup_filenames, "No .pileup files."
nc_match = mlib.read_normal_cancer_file(nc_node.identifier)
#ref = alignlib.create_reference_genome(ref_node.identifier)
filelib.safe_mkdir(out_path)
# Figure out whether the purpose is to get coverage. Change
# the parameters if it is.
assert "vartype" in out_attributes
vartype = out_attributes["vartype"]
assert vartype in ["snp", "indel"]
sample2pufile = {} # sample -> mpileup filename
for filename in mpileup_filenames:
path, sample, ext = mlib.splitpath(filename)
sample2pufile[sample] = filename
# Make sure files exist for all the samples.
all_samples = []
for (normal_sample, cancer_sample) in nc_match:
if normal_sample not in all_samples:
all_samples.append(normal_sample)
if cancer_sample not in all_samples:
all_samples.append(cancer_sample)
missing = [x for x in all_samples if x not in sample2pufile]
x = parselib.pretty_list(missing, max_items=5)
assert not missing, "Missing BAM files for samples: %s" % x
# list of (sample, normal_pileup, cancer_pileup,
# tmp1_normal, tmp1_cancer, log_filename, out_filename)
opj = os.path.join
jobs = []
for (normal_sample, cancer_sample) in nc_match:
normal_pileup = sample2pufile[normal_sample]
cancer_pileup = sample2pufile[cancer_sample]
p, sample, ext = mlib.splitpath(cancer_pileup)
tmp1_normal = opj(out_path, "%s.normal.tmp1" % sample)
tmp1_cancer = opj(out_path, "%s.cancer.tmp1" % sample)
log_filename = opj(out_path, "%s.log" % sample)
out_filename = opj(out_path, "%s.vcf" % sample)
x = sample, normal_sample, cancer_sample, \
normal_pileup, cancer_pileup, \
tmp1_normal, tmp1_cancer, log_filename, out_filename
jobs.append(x)
# VarScan will generate a "Parsing Exception" if there are 0
# reads in a location. Will be either "0" or blank. Filter
# those lines out.
sq = parallel.quote
commands = []
for x in jobs:
sample, normal_sample, cancer_sample, \
normal_pileup, cancer_pileup, \
tmp1_normal, tmp1_cancer, log_filename, out_filename = x
x1 = "awk -F'\t' '$4 >= 1 {print}' %s > %s" % (normal_pileup,
tmp1_normal)
x2 = "awk -F'\t' '$4 >= 1 {print}' %s > %s" % (cancer_pileup,
tmp1_cancer)
commands.extend([x1, x2])
parallel.pshell(commands, max_procs=num_cores)
x = [x[3] for x in jobs] + [x[4] for x in jobs]
filelib.assert_exists_nz_many(x)
# java -jar VarScan.jar somatic [normal_pileup] [tumor_pileup]
# [output] OPTIONS
varscan = mlib.findbin("varscan_jar")
# Use parameters from:
# Using VarScan 2 for Germline Variant Calling and Somatic
# Mutation Detection
# Make a list of commands.
commands = []
for x in jobs:
sample, normal_sample, cancer_sample, \
normal_pileup, cancer_pileup, \
tmp1_normal, tmp1_cancer, log_filename, out_filename = x
x = [
"java",
"-jar",
sq(varscan),
"somatic",
sq(tmp1_normal),
sq(tmp1_cancer),
sample,
"--min-coverage",
10,
"--min-avg-qual",
15,
"--min-normal-coverage",
10,
"--min-tumor-coverage",
10,
"--min-var-freq",
0.05,
"--somatic-p-value",
0.05,
"--output-vcf",
1,
]
x = " ".join(map(str, x))
x = "%s >& %s" % (x, log_filename)
commands.append(x)
parallel.pshell(commands, max_procs=num_cores)
x = [x[5] for x in jobs]
filelib.assert_exists_nz_many(x)
# Files in out_path can get very big. Clean them up.
# <sample>.normal.tmp1 Very big (10's Gb).
# <sample>.cancer.tmp1 Very big (10's to 100 Gb).
for x in jobs:
sample, normal_sample, cancer_sample, \
normal_pileup, cancer_pileup, \
tmp1_normal, tmp1_cancer, log_filename, out_filename = x
if os.path.exists(tmp1_normal):
os.unlink(tmp1_normal)
if os.path.exists(tmp1_cancer):
os.unlink(tmp1_cancer)
# Copy the final file to the right place.
for x in jobs:
sample, normal_sample, cancer_sample, \
normal_pileup, cancer_pileup, \
tmp1_normal, tmp1_cancer, log_filename, out_filename = x
# Will be written in current directory.
varscan_out = "%s.snp.vcf" % sample
if vartype == "indel":
varscan_out = "%s.indel.vcf" % sample
filelib.assert_exists(varscan_out)
shutil.copy2(varscan_out, out_filename)
# VarScan names the samples "NORMAL" and "TUMOR". Replace
# them with the actual names.
for x in jobs:
sample, normal_sample, cancer_sample, \
normal_pileup, cancer_pileup, \
tmp1_normal, tmp1_cancer, log_filename, out_filename = x
_fix_normal_cancer_names(out_filename, normal_sample,
cancer_sample)
def run_module(network,
input_ids,
module_id,
out_data_node,
all_user_options,
pool,
transitions,
user,
job_name='',
clean_up=True,
num_cores=8,
verbosity=0):
# Return tuple of (output_path, IdentifiedDataNode, node_id,
# elapsed time) for the node that was created. Returns None if
# this module fails (no compatible output nodes, or all output
# nodes already generated).
import os
import sys
import time
import logging
from genomicode import filelib
from Betsy import config
from Betsy import bie3
assert user
output_path = config.CACHE_PATH
filelib.assert_exists(output_path)
# Import module.
module_node, module = _import_module(network, module_id)
assert len(module_node.in_datatypes) == len(input_ids)
module_name = module_node.name
# Get the antecedents from the pool.
antecedents = [pool[x] for x in input_ids]
if len(antecedents) == 1:
antecedents = antecedents[0]
# Get the user_options for this module. all_user_options contains
# all options provided by the user. Pull out the ones relevant
# for this module. Use the defaults when necessary.
user_options = {}
for option in module_node.option_defs:
value = all_user_options.get(option.name)
if value is None:
value = option.default
assert value is not None, "Missing input: %s" % option.name
user_options[option.name] = value
# Set up the directories and outfile.
# Unfortunately, can't use timestamp in pathname, or else this
# will never re-use prior analyses. Have to be more clever about
# this.
h = _hash_module(module_name, antecedents, out_data_node.attributes,
user_options)
## Get milliseconds.
#x = time.time()
#ms = int((x-int(x))*100)
#ts = time.strftime("%y%m%d.%H%M%S", time.localtime())
#x = "%s.%02d__%s__B%03d__%s" % (ts, ms, module_name, VERSION, h)
x = "%s__B%03d__%s" % (module_name, VERSION, h)
result_dir = os.path.join(output_path, x)
outfile = module.name_outfile(antecedents, user_options)
# Create the IdentifiedDataNode that will be the output once the
# module has been run.
full_outfile = os.path.join(result_dir, outfile)
#x = bie3.DataNode(network.nodes[next_id].datatype, **out_data.attributes)
out_identified_data_node = bie3.IdentifiedDataNode(out_data_node,
full_outfile)
#time_str = "[%s]" % time.strftime('%I:%M %p')
time_str = "[%s]" % time.strftime('%a %I:%M %p')
# Check if this has already been run.
if _is_module_output_complete(result_dir):
# Update timestamp on LAST_ACCESSED_FILE.
open(os.path.join(result_dir, LAST_ACCESSED_FILE), 'w')
# Read parameter file.
filename = os.path.join(result_dir, BETSY_PARAMETER_FILE)
assert os.path.exists(filename)
params = _read_parameter_file(filename)
elapsed = params["elapsed"]
run_time = params["elapsed_pretty"]
if run_time == "instant":
x = "ran instantly"
else:
x = "took %s" % run_time
x = "%s %s (CACHED, previously %s)" % (time_str, module_name, x)
#parselib.print_split(x, prefixn=2)
print x
if verbosity >= 1:
# Print out the output directory.
indent = len("[Thu 10:06 PM] ")
x = os.path.split(result_dir)[1]
print "%s%s" % (" " * indent, x)
sys.stdout.flush()
return result_dir, out_identified_data_node, elapsed
#_debug_is_module_output_complete(
# module_name, antecedents, out_data_node.attributes, user_options,
# VERSION, h, result_dir)
# Running this module now.
x = "%s %s" % (time_str, module_name)
#parselib.print_split(x, prefixn=2)
print x
if verbosity >= 1:
# Print out the output directory.
indent = len("[Thu 10:06 PM] ")
x = os.path.split(result_dir)[1]
print "%s%s" % (" " * indent, x)
sys.stdout.flush()
# Run the analysis. If someone else is currently running the same
# analysis, then wait for them to finish. However, if they have
# somehow failed, then delete the incomplete results and start
# over.
#
# 1. Create directory.
# 2. Write in_progress.txt.
# 3. Run the analysis. Refresh in_progress.txt every 5 sec.
# 4. Write finished.txt.
# 5. Stop refreshing in_progress.txt.
#
# IN_PROGRESS FINISHED INTERPRETATION
# missing missing Starting analysis? Wait 5 sec, check again.
# If everything still missing, then overwrite.
# missing present Complete analysis.
# <5 sec old missing Still copying. Wait.
# <5 sec old present Finishing up. Consider complete.
# >5 sec old missing Abandoned. Overwrite.
# >5 sec old present Should not happen. rm copying.txt, check
# after 5 sec. If missing, consider
# complete. If back, consider error.
REFRESH = 5 # number of seconds to refresh copying.txt file.
success_file = os.path.join(result_dir, FINISHED_FILE)
last_accessed_file = os.path.join(result_dir, LAST_ACCESSED_FILE)
copying_file = os.path.join(result_dir, IN_PROGRESS_FILE)
exists = os.path.exists
i_run_analysis = None
while i_run_analysis is None:
# Try to make the result_dir. If I make it, then I should run the
# analysis. Otherwise, someone else has priority. Let them run
# the analysis.
if not os.path.exists(result_dir):
try:
os.mkdir(result_dir)
i_run_analysis = True
break
except OSError, x:
pass
# For debugging. If I want to re-run the module over
# the old one, then just keep this directory.
if not CLEAN_UP_PATH_FOR_NEW_MODULE:
i_run_analysis = True
break
last_refresh = None
if exists(copying_file):
last_refresh = time.time() - os.path.getctime(copying_file)
if not exists(copying_file) and not exists(success_file):
# BUG: This doesn't work. What if this was abandoned, but
# somebody else just happens to create the directory again
# while I'm checking? Will have result_dir, but nothing
# inside it.
# SOLUTION: Give them a cycle to create something.
# DEBUG: Skip this.
#i_run_analysis = True; break
time.sleep(REFRESH + 1)
if not exists(copying_file) and not exists(success_file):
# Abandoned. Delete the result dir and try again.
if CLEAN_UP_PATH_FOR_NEW_MODULE:
_rmtree_multi(result_dir)
elif not exists(copying_file) and exists(success_file):
# Previous run is now finished.
i_run_analysis = False
# From here on down, copying_file should exist.
elif last_refresh < REFRESH and not exists(success_file):
# Still copying. Wait.
time.sleep(REFRESH)
elif last_refresh < REFRESH and exists(success_file):
# Finishing up. Consider complete.
i_run_analysis = False
elif last_refresh >= REFRESH * 2 and not exists(success_file):
# Steal the file. This can cause a lot of problems, so
# don't do this unless we're sure the other process is
# really dead.
# Abandoned. Delete the result dir and try again.
if CLEAN_UP_PATH_FOR_NEW_MODULE:
_rmtree_multi(result_dir)
elif last_refresh >= REFRESH * 2 and exists(success_file):
os.unlink(copying_file)
time.sleep(REFRESH)
# Should not be coming back if analysis has already
# completed successfully.
assert not exists(copying_file), "Zombie in progress file"
# At this point, no copying_file, but there is a
# success_file. Consider this analysis complete.
i_run_analysis = False
else:
# Does not fit one of these.
time.sleep(REFRESH)