def runAllMappedSplit(start_idx=0, stop_idx=100000, part='[]', map_dir_ext=''):
if part == 'all': parts = ["[%d]" % x for x in range(50)]
else: parts = [part]
map_dir, out_label = '/mapped_reads%s' % map_dir_ext, 'out_split%s' % map_dir_ext
all_dir, out_dir = getAllDataDirs(), getLogDir()
for part in parts:
idx = 0
for dirname in all_dir:
print(getShortDir(dirname), idx)
extra_cmd = '' if (
part != '[]' and 0 not in eval(part)
) or getRunLocal() else 'rm -rf %s/%s' % (dirname, map_dir)
if getRunLocal() and 'NULL' not in dirname and os.path.isdir(
'%s/%s' % (dirname, map_dir)):
shutil.rmtree('%s/%s' % (dirname, map_dir))
cmd = getPythonCmd(
) + ' split_mapped_reads_by_id.py %s %s %s/%s "%s"' % (
dirname, getExpOligoFile(dirname), dirname, map_dir, part)
idx = runCmdCheckIdx(cmd,
idx,
start_idx,
stop_idx,
out_dir,
out_label,
extra_cmd=extra_cmd,
queue='normal')
def runAllIndelMap(start_idx=0,
stop_idx=10000000,
overbeek_only=False,
queue='normal',
map_dir='/mapped_reads/',
max_cut_dist=4,
num_parts=1,
order_by_incomplete=False):
all_dir, out_dir = getAllDataDirs(), getLogDir()
if overbeek_only: print('Computing for Overbeek guides only')
completed_lookup = {}
if order_by_incomplete:
f = io.open('../quality_checks/status.log')
completed_lookup = {
toks[0]: min([eval(x) for x in toks[1:]]) != 0
for toks in csv.reader(f, delimiter='\t')
}
f.close()
completed = [
x for x in all_dir if getDirLabel(x) in completed_lookup
and completed_lookup[getDirLabel(x)]
]
not_completed = [x for x in all_dir if x not in completed]
max_files_per_dir = 20
file_per_part = int(max_files_per_dir / num_parts + 0.99)
i, idx = 0, 0
for dirname in not_completed + completed:
if len(not_completed) == i:
print('-------------------------------------------------')
i += 1
print(getShortDir(dirname), idx)
if not os.path.isdir(dirname + map_dir): continue
for subdir in getSubdirs(dirname, withpath=False):
if overbeek_only and subdir != 'Oligos_71': continue
args = (dirname, getNullDir(dirname), subdir, file_per_part,
map_dir, max_cut_dist, getIndelMapExe(), getPythonCmd())
cmd = getPythonCmd(
) + ' indelmap_subdir.py %s %s %s %d - 0 %s %d %s %s' % args
idx = runCmdCheckIdx(cmd,
idx,
start_idx,
stop_idx,
out_dir,
'out_indelmap_%s' % getDirLabel(dirname),
numj=num_parts,
queue=queue)
def runAllPartition(start_idx=0, stop_idx=1000000, nump=50):
all_dir, out_dir = getAllDataDirs(), getLogDir()
idx = 0
for dirname in all_dir:
print(getShortDir(dirname), idx)
filenames = [
x for x in os.listdir(dirname)
if x.split('_')[-1] == 'pear.assembled.fastq'
]
for filename in filenames:
cmd = getPythonCmd() + ' partition_pear.py %s/%s %d' % (
dirname, filename, nump)
idx = runCmdCheckIdx(cmd, idx, start_idx, stop_idx, out_dir,
'out_part')
def runPerSubdir(python_script,
out_label,
caller,
extra_args='',
include_null=False):
idx = 0
for dirname in [
x for x in getAllDataDirs() if include_null or not isNullDir(x)
]:
if not os.path.isdir(dirname + '/mapped_reads'):
print(getShortDir(dirname)), 'No mapped_reads directory'
else:
subdirs = getSubdirs(dirname)
idx = runSubdir(idx,
subdirs,
getShortDir(dirname),
python_script,
out_label,
caller,
extra_args=extra_args)
def runAllMap(start_idx=0,
stop_idx=100000000,
recompute=True,
unassembled_only=False,
max_cut_dist=4,
map_dir='mapping_files'):
all_dir, out_dir = getAllDataDirs(), getLogDir()
idx = 0
for dirname in all_dir:
exp_file = getExpOligoFile(dirname)
print(getShortDir(dirname), idx)
check_str = '_pear.unassembled_pear.assembled._' if unassembled_only else '_pear.assembled._'
filenames = [x for x in os.listdir(dirname) if check_str in x]
for filename in filenames:
cmd_args = (dirname, filename, exp_file, dirname, map_dir,
filename[:-6], max_cut_dist)
cmd = getIndelMapExe(
) + ' %s/%s %s %s/%s/%s_mappings.txt 1 %d' % cmd_args
extra_cmd = ''
if not os.path.isdir(dirname + '/' + map_dir):
if getRunLocal(): os.mkdir(dirname + '/' + map_dir)
else: extra_cmd = 'mkdir %s' % (dirname + '/' + map_dir)
if not recompute and os.path.isfile(dirname + '/' + map_dir + '/' +
filename[:-6] +
'_mappings.txt'):
continue
idx = runCmdCheckIdx(cmd,
idx,
start_idx,
stop_idx,
out_dir,
'out_map',
extra_cmd=extra_cmd)
def runAllPear(start_idx=0, stop_idx=100000):
all_dir, out_dir = getAllDataDirs(), getLogDir()
idx = 0
for dirname in all_dir:
print(getShortDir(dirname), idx)
r1_fasta_files = [
x for x in os.listdir(dirname)
if x[-9:] == '_R1.fastq' or x[-13:] == '_R1_001.fastq'
]
for r1_file in r1_fasta_files:
file_prefix = r1_file[:r1_file.index('_R1')]
file_suffix = r1_file[len(file_prefix):].replace('1', '2', 1)
if not os.path.isfile(dirname + '/' + file_prefix + file_suffix):
print('Could not find matching R2 file:', dirname, file_prefix,
file_suffix)
continue
cargs = (dirname, r1_file, dirname, file_prefix, file_suffix,
dirname, file_prefix)
cmd = getPearExe(
) + ' -f %s/%s -r %s/%s%s -o %s/%s_pear -n 20 -p 0.01' % cargs
idx = runCmdCheckIdx(cmd, idx, start_idx, stop_idx, out_dir,
'out_pear')
from selftarget.oligo import getOligoIdsFromMappedFastaFile, splitOligoFastaFile, getFileForOligoIdx
start_idx, stop_idx = -1, -1
queue = 'normal'
source = 'mapped_counts'
indelmap_only = False
if len(sys.argv) != 3 and len(sys.argv) != 4 and len(sys.argv) != 5 and len(
sys.argv) != 6:
print 'Usage: run_all_indelmap.py start_idx stop_idx (opt)queue (opt)indelmap_only (opt)source'
else:
start_idx, stop_idx = eval(sys.argv[1]), eval(sys.argv[2])
if len(sys.argv) >= 4: queue = sys.argv[3]
if len(sys.argv) >= 5: indelmap_only = eval(sys.argv[4])
if len(sys.argv) >= 6: source = sys.argv[5]
all_dir, out_dir = getAllDataDirs(), getLogDir()
idx = 0
if source == 'date_modified':
f = io.open('../quality_checks/log_files/out_modified.log')
for line in f:
if 'mapped_reads' not in line: continue
#e.g. /lustre/scratch117/cellgen/team227/fa9/self-target/indel_analysis/ST_June_2017/data/K562_800x_LV7B_DPI3/mapped_reads/Oligos_70/Oligos_70850-70899_mappedindelsummary.txt
full_filename = line.split()[0]
toks = full_filename.split('/')
fasta_file = toks[-1][:-23] + '.fasta'
filepath = '/'.join(toks[:-1])
dirname = '/'.join(toks[:-3])
nulldir = getNullDir(dirname)
from selftarget.util import runSubdir
from selftarget.data import getAllDataDirs
if __name__ == '__main__':
idx = 0
idx = runSubdir(idx, getAllDataDirs(), 'All',
'fetch_mh_mismatch_frequencies.py', 'out_get_mm_freq',
__file__)