def main():
usage = 'usage: %prog [options] <fasta_file> <seqs_bed_file> <seqs_cov_dir> <tfr_file>'
parser = OptionParser(usage)
parser.add_option('-s',
dest='start_i',
default=0,
type='int',
help='Sequence start index [Default: %default]')
parser.add_option('-e',
dest='end_i',
default=None,
type='int',
help='Sequence end index [Default: %default]')
parser.add_option('--te',
dest='target_extend',
default=None,
type='int',
help='Extend targets vector [Default: %default]')
parser.add_option(
'--ts',
dest='target_start',
default=0,
type='int',
help='Write targets into vector starting at index [Default: %default')
parser.add_option('-u',
dest='umap_npy',
help='Unmappable array numpy file')
parser.add_option(
'--umap_clip',
dest='umap_clip',
default=1,
type='float',
help=
'Clip values at unmappable positions to distribution quantiles, eg 0.25. [Default: %default]'
)
parser.add_option(
'--umap_tfr',
dest='umap_tfr',
default=False,
action='store_true',
help='Save umap array into TFRecords [Default: %default]')
(options, args) = parser.parse_args()
if len(args) != 4:
parser.error('Must provide input arguments.')
else:
fasta_file = args[0]
seqs_bed_file = args[1]
seqs_cov_dir = args[2]
tfr_file = args[3]
################################################################
# read model sequences
model_seqs = []
for line in open(seqs_bed_file):
a = line.split()
model_seqs.append(ModelSeq(a[0], int(a[1]), int(a[2]), None))
if options.end_i is None:
options.end_i = len(model_seqs)
num_seqs = options.end_i - options.start_i
################################################################
# determine sequence coverage files
seqs_cov_files = []
ti = 0
seqs_cov_file = '%s/%d.h5' % (seqs_cov_dir, ti)
while os.path.isfile(seqs_cov_file):
seqs_cov_files.append(seqs_cov_file)
ti += 1
seqs_cov_file = '%s/%d.h5' % (seqs_cov_dir, ti)
if len(seqs_cov_files) == 0:
print('Sequence coverage files not found, e.g. %s' % seqs_cov_file,
file=sys.stderr)
exit(1)
seq_pool_len = h5py.File(seqs_cov_files[0], 'r')['targets'].shape[1]
num_targets = len(seqs_cov_files)
################################################################
# read targets
# extend targets
num_targets_tfr = num_targets
if options.target_extend is not None:
assert (options.target_extend >= num_targets_tfr)
num_targets_tfr = options.target_extend
# initialize targets
targets = np.zeros((num_seqs, seq_pool_len, num_targets_tfr),
dtype='float16')
# read each target
for ti in range(num_targets):
seqs_cov_open = h5py.File(seqs_cov_files[ti], 'r')
tii = options.target_start + ti
targets[:, :, tii] = seqs_cov_open['targets'][
options.start_i:options.end_i, :]
seqs_cov_open.close()
################################################################
# modify unmappable
if options.umap_npy is not None and options.umap_clip < 1:
unmap_mask = np.load(options.umap_npy)
for si in range(num_seqs):
msi = options.start_i + si
# determine unmappable null value
seq_target_null = np.percentile(targets[si],
q=[100 * options.umap_clip],
axis=0)[0]
# set unmappable positions to null
targets[si, unmap_mask[msi, :], :] = np.minimum(
targets[si, unmap_mask[msi, :], :], seq_target_null)
elif options.umap_npy is not None and options.umap_tfr:
unmap_mask = np.load(options.umap_npy)
################################################################
# write TFRecords
# open FASTA
fasta_open = pysam.Fastafile(fasta_file)
# define options
tf_opts = tf.io.TFRecordOptions(compression_type='ZLIB')
with tf.io.TFRecordWriter(tfr_file, tf_opts) as writer:
for si in range(num_seqs):
msi = options.start_i + si
mseq = model_seqs[msi]
# read FASTA
seq_dna = fasta_open.fetch(mseq.chr, mseq.start, mseq.end)
# one hot code
seq_1hot = dna_1hot(seq_dna)
# seq_1hot = dna_1hot_index(seq_dna) # more efficient, but fighting inertia
# hash to bytes
features_dict = {
'sequence': feature_bytes(seq_1hot),
'target': feature_bytes(targets[si, :, :])
}
# add unmappability
if options.umap_tfr:
features_dict['umap'] = feature_bytes(unmap_mask[msi, :])
# write example
example = tf.train.Example(features=tf.train.Features(
feature=features_dict))
writer.write(example.SerializeToString())
fasta_open.close()
def main():
usage = 'usage: %prog [options] <genome_hic_file> <seqs_bed_file> <seqs_hic_file>'
parser = OptionParser(usage)
parser.add_option('-b', dest='blacklist_bed',
help='Set blacklist nucleotides to a baseline value.')
parser.add_option('--clip', dest='clip',
default=None, type='float',
help='Clip values post-summary to a maximum [Default: %default]')
parser.add_option('--crop', dest='crop_bp',
default=0, type='int',
help='Crop bp off each end [Default: %default]')
parser.add_option('-d', dest='diagonal_offset',
default=2, type='int',
help='Positions on the diagonal to ignore [Default: %default]')
parser.add_option('-k', dest='kernel_stddev',
default=0, type='int',
help='Gaussian kernel stddev to smooth values [Default: %default]')
# parser.add_option('-s', dest='scale',
# default=1., type='float',
# help='Scale values by [Default: %default]')
parser.add_option('-w',dest='pool_width',
default=1, type='int',
help='Average pooling width [Default: %default]')
parser.add_option('--as_obsexp',dest='as_obsexp',
default=False,action="store_true",
help='save targets as obsexp profiles')
parser.add_option('--global_obsexp',dest='global_obsexp',
default=False,action="store_true",
help='use global obs/exp')
parser.add_option('--no_log',dest='no_log',
default=False,action="store_true",
help='no not take log for obs/exp')
(options, args) = parser.parse_args()
if len(args) != 3:
parser.error('')
else:
genome_hic_file = args[0]
seqs_bed_file = args[1]
seqs_hic_file = args[2]
# read model sequences
model_seqs = []
for line in open(seqs_bed_file):
a = line.split()
model_seqs.append(ModelSeq(a[0],int(a[1]),int(a[2]),None))
# read blacklist regions
black_chr_trees = read_blacklist(options.blacklist_bed)
# compute dimensions
num_seqs = len(model_seqs)
seq_len_nt = model_seqs[0].end - model_seqs[0].start
seq_len_pool = seq_len_nt // options.pool_width
if options.crop_bp == 0:
seq_len_crop = seq_len_pool
else:
crop_start = options.crop_bp // options.pool_width
crop_end = seq_len_pool - crop_start
seq_len_crop = seq_len_pool - 2*crop_start
# compute upper triangular indexes
triu_tup = np.triu_indices(seq_len_crop, options.diagonal_offset)
seq_len_nodiag = seq_len_crop - options.diagonal_offset
seq_len_hic = seq_len_nodiag*(seq_len_nodiag + 1) // 2
# initialize sequences coverage file
seqs_hic_open = h5py.File(seqs_hic_file, 'w')
seqs_hic_open.create_dataset('targets', shape=(num_seqs, seq_len_hic), dtype='float16')
if options.kernel_stddev > 0:
# initialize Gaussian kernel
kernel = Gaussian2DKernel(x_stddev=options.kernel_stddev)
else:
kernel = None
# open genome coverage file
genome_hic_cool = cooler.Cooler(genome_hic_file)
if options.global_obsexp:
try:
print('loading by-chromosome expected')
genome_hic_expected = pd.read_csv(genome_hic_file.replace('.cool','.expected'), sep='\t')
except:
print('not found: '+genome_hic_file.replace('cool','expected'))
raise ValueError('invalid expected file')
# check for "chr" prefix
chr_pre = 'chr1' in genome_hic_cool.chromnames
# assert that resolution matches
assert(options.pool_width == genome_hic_cool.info['bin-size'])
# for each model sequence
for si in range(num_seqs):
mseq = model_seqs[si]
try:
# pull hic values
if chr_pre:
mseq_str = '%s:%d-%d' % (mseq.chr, mseq.start, mseq.end)
else:
mseq_str = '%s:%d-%d' % (mseq.chr[3:], mseq.start, mseq.end)
#print('mseq_str:', mseq_str)
seq_hic_raw = genome_hic_cool.matrix(balance=True).fetch(mseq_str)
seq_hic_nan = np.isnan(seq_hic_raw)
num_filtered_bins = np.sum(np.sum(seq_hic_nan,axis=0) == len(seq_hic_nan))
if num_filtered_bins > (.5*len(seq_hic_nan)):
print("WARNING: %s >50% bins filtered, check: %s. " % (genome_hic_file, mseq_str))
# set blacklist to NaNs
if mseq.chr in black_chr_trees:
for black_interval in black_chr_trees[mseq.chr][mseq.start:mseq.end]:
# adjust for sequence indexes
black_seq_start = (black_interval.begin - mseq.start)// options.pool_width
black_seq_end = int( np.ceil( (black_interval.end - mseq.start)/ options.pool_width ) )
seq_hic_raw[:,black_seq_start:black_seq_end] = np.nan
seq_hic_raw[black_seq_start:black_seq_end,:] = np.nan
seq_hic_nan = np.isnan(seq_hic_raw)
# clip first diagonals and high values
clipval = np.nanmedian(np.diag(seq_hic_raw,options.diagonal_offset))
for i in range(-options.diagonal_offset+1,options.diagonal_offset):
set_diag(seq_hic_raw, clipval, i)
seq_hic_raw = np.clip(seq_hic_raw, 0, clipval)
seq_hic_raw[seq_hic_nan] = np.nan
# adaptively coarsegrain based on raw counts
seq_hic_smoothed = adaptive_coarsegrain(
seq_hic_raw,
genome_hic_cool.matrix(balance=False).fetch(mseq_str),
cutoff=2, max_levels=8)
seq_hic_nan = np.isnan(seq_hic_smoothed)
#todo: pass an option to add a certain pseudocount value, or the minimum nonzero value
if options.as_obsexp:
# compute obs/exp
if options.global_obsexp: # compute global obs/exp
exp_chr = genome_hic_expected.iloc[ genome_hic_expected['chrom'].values ==mseq.chr][0:seq_len_pool]
if len(exp_chr) ==0:
raise ValueError('no expected values found for chr:'+mseq.chr)
exp_map= np.zeros((seq_len_pool,seq_len_pool))
for i in range(seq_len_pool):
set_diag(exp_map,exp_chr['balanced.avg'].values[i],i)
set_diag(exp_map,exp_chr['balanced.avg'].values[i],-i)
seq_hic_obsexp = seq_hic_smoothed / exp_map
for i in range(-options.diagonal_offset+1,options.diagonal_offset): set_diag(seq_hic_obsexp,1.0,i)
seq_hic_obsexp[seq_hic_nan] = np.nan
else: # compute local obs/exp
seq_hic_obsexp = observed_over_expected(seq_hic_smoothed, ~seq_hic_nan)[0]
# log
if options.no_log==False:
seq_hic_obsexp = np.log(seq_hic_obsexp)
if options.clip is not None:
seq_hic_obsexp = np.clip(seq_hic_obsexp, -options.clip, options.clip)
seq_hic_obsexp = interp_nan(seq_hic_obsexp)
for i in range(-options.diagonal_offset+1, options.diagonal_offset): set_diag(seq_hic_obsexp, 0,i)
else:
if options.clip is not None:
seq_hic_obsexp = np.clip(seq_hic_obsexp, 0, options.clip)
seq_hic_obsexp = interp_nan(seq_hic_obsexp)
for i in range(-options.diagonal_offset+1, options.diagonal_offset): set_diag(seq_hic_obsexp, 1,i)
# apply kernel
if kernel is not None:
seq_hic = convolve(seq_hic_obsexp, kernel)
else:
seq_hic = seq_hic_obsexp
else:
# interpolate all missing bins
seq_hic_interpolated = interp_nan(seq_hic_smoothed)
# rescale, reclip
seq_hic = 100000*seq_hic_interpolated
clipval = np.nanmedian(np.diag(seq_hic,options.diagonal_offset))
for i in range(-options.diagonal_offset+1, options.diagonal_offset):
set_diag(seq_hic,clipval,i)
seq_hic = np.clip(seq_hic, 0, clipval)
#extra smoothing. todo pass kernel specs
if kernel is not None:
seq_hic = convolve(seq_hic, kernel)
except ValueError:
print("WARNING: %s doesn't see %s. Setting to all zeros." % (genome_hic_file, mseq_str))
seq_hic = np.zeros((seq_len_pool,seq_len_pool), dtype='float16')
# crop
if options.crop_bp > 0:
seq_hic = seq_hic[crop_start:crop_end,:]
seq_hic = seq_hic[:,crop_start:crop_end]
# unroll upper triangular
seq_hic = seq_hic[triu_tup]
# write
seqs_hic_open['targets'][si,:] = seq_hic.astype('float16')
# close sequences coverage file
seqs_hic_open.close()
def main():
usage = 'usage: %prog [options] <genome_cov_file> <seqs_bed_file> <seqs_cov_file>'
parser = OptionParser(usage)
parser.add_option('-b',
dest='blacklist_bed',
help='Set blacklist nucleotides to a baseline value.')
parser.add_option(
'-c',
dest='clip',
default=None,
type='float',
help='Clip values post-summary to a maximum [Default: %default]')
parser.add_option('--crop',
dest='crop_bp',
default=0,
type='int',
help='Crop bp off each end [Default: %default]')
parser.add_option('-i',
dest='interp_nan',
default=False,
action='store_true',
help='Interpolate NaNs [Default: %default]')
parser.add_option('-s',
dest='scale',
default=1.,
type='float',
help='Scale values by [Default: %default]')
parser.add_option(
'--soft',
dest='soft_clip',
default=False,
action='store_true',
help=
'Soft clip values, applying sqrt to the execess above the threshold [Default: %default]'
)
parser.add_option(
'-u',
dest='sum_stat',
default='sum',
help='Summary statistic to compute in windows [Default: %default]')
parser.add_option('-w',
dest='pool_width',
default=1,
type='int',
help='Average pooling width [Default: %default]')
(options, args) = parser.parse_args()
if len(args) != 3:
parser.error('')
else:
genome_cov_file = args[0]
seqs_bed_file = args[1]
seqs_cov_file = args[2]
assert (options.crop_bp >= 0)
# read model sequences
model_seqs = []
for line in open(seqs_bed_file):
a = line.split()
model_seqs.append(ModelSeq(a[0], int(a[1]), int(a[2]), None))
# read blacklist regions
black_chr_trees = read_blacklist(options.blacklist_bed)
# compute dimensions
num_seqs = len(model_seqs)
seq_len_nt = model_seqs[0].end - model_seqs[0].start
seq_len_nt -= 2 * options.crop_bp
target_length = seq_len_nt // options.pool_width
assert (target_length > 0)
# initialize sequences coverage file
seqs_cov_open = h5py.File(seqs_cov_file, 'w')
seqs_cov_open.create_dataset('targets',
shape=(num_seqs, target_length),
dtype='float16')
# open genome coverage file
genome_cov_open = CovFace(genome_cov_file)
# for each model sequence
for si in range(num_seqs):
mseq = model_seqs[si]
# read coverage
seq_cov_nt = genome_cov_open.read(mseq.chr, mseq.start, mseq.end)
# interpolate NaN
if options.interp_nan:
seq_cov_nt = interp_nan(seq_cov_nt)
# determine baseline coverage
baseline_cov = np.percentile(seq_cov_nt, 10)
baseline_cov = np.nan_to_num(baseline_cov)
# set blacklist to baseline
if mseq.chr in black_chr_trees:
for black_interval in black_chr_trees[
mseq.chr][mseq.start:mseq.end]:
# adjust for sequence indexes
black_seq_start = black_interval.begin - mseq.start
black_seq_end = black_interval.end - mseq.start
seq_cov_nt[black_seq_start:black_seq_end] = baseline_cov
# set NaN's to baseline
if not options.interp_nan:
nan_mask = np.isnan(seq_cov_nt)
seq_cov_nt[nan_mask] = baseline_cov
# crop
if options.crop_bp > 0:
seq_cov_nt = seq_cov_nt[options.crop_bp:-options.crop_bp]
# sum pool
seq_cov = seq_cov_nt.reshape(target_length, options.pool_width)
if options.sum_stat == 'sum':
seq_cov = seq_cov.sum(axis=1, dtype='float32')
elif options.sum_stat in ['mean', 'avg']:
seq_cov = seq_cov.mean(axis=1, dtype='float32')
elif options.sum_stat == 'median':
seq_cov = seq_cov.median(axis=1, dtype='float32')
elif options.sum_stat == 'max':
seq_cov = seq_cov.max(axis=1, dtype='float32')
else:
print('ERROR: Unrecognized summary statistic "%s".' %
options.sum_stat,
file=sys.stderr)
exit(1)
# clip
if options.clip is not None:
if options.soft_clip:
clip_mask = (seq_cov > options.clip)
seq_cov[clip_mask] = options.clip + np.sqrt(
seq_cov[clip_mask] - options.clip)
else:
seq_cov = np.clip(seq_cov, 0, options.clip)
# scale
seq_cov = options.scale * seq_cov
# write
seqs_cov_open['targets'][si, :] = seq_cov.astype('float16')
# close genome coverage file
genome_cov_open.close()
# close sequences coverage file
seqs_cov_open.close()
def main():
usage = 'usage: %prog [options] <genome_cov_file> <seqs_bed_file> <seqs_cov_file>'
parser = OptionParser(usage)
# add option for thresholding
parser.add_option('-b',
dest='blacklist_bed',
help='Set blacklist nucleotides to a baseline value.')
parser.add_option(
'-c',
dest='clip',
default=None,
type='float',
help='Clip values post-summary to a maximum [Default: %default]')
parser.add_option(
'--clip_soft',
dest='clip_soft',
default=None,
type='float',
help=
'Soft clip values, applying sqrt to the execess above the threshold [Default: %default]'
)
parser.add_option('--crop',
dest='crop_bp',
default=0,
type='int',
help='Crop bp off each end [Default: %default]')
parser.add_option('-i',
dest='interp_nan',
default=False,
action='store_true',
help='Interpolate NaNs [Default: %default]')
parser.add_option('-s',
dest='scale',
default=1.,
type='float',
help='Scale values by [Default: %default]')
parser.add_option(
'-u',
dest='sum_stat',
default='sum',
help='Summary statistic to compute in windows [Default: %default]')
parser.add_option('-w',
dest='pool_width',
default=1,
type='int',
help='Average pooling width [Default: %default]')
parser.add_option('--norm',
dest='norm',
default='',
type='str',
help='Normalize coverage values')
# parser.add_option('--step_fr', dest='step_fr',
# default=1., type='float',
# help='Stride using fraction of bin size [Default: %default]')
parser.add_option('--step_bp',
dest='step_bp',
default=0,
type='int',
help='Stride using bp step size [Default: %default]')
parser.add_option('--padding',
dest='padding',
default='valid',
type='str',
help='Padding method for sliding window approach')
(options, args) = parser.parse_args()
if len(args) != 3:
parser.error('')
else:
genome_cov_file = args[0]
seqs_bed_file = args[1]
seqs_cov_file = args[2]
assert (options.crop_bp >= 0)
# read model sequences
model_seqs = []
for line in open(seqs_bed_file):
a = line.split()
model_seqs.append(ModelSeq(a[0], int(a[1]), int(a[2]), None))
# read blacklist regions
black_chr_trees = read_blacklist(options.blacklist_bed)
# compute dimensions
num_seqs = len(model_seqs)
seq_len_nt = model_seqs[0].end - model_seqs[0].start
seq_len_nt -= 2 * options.crop_bp
target_length = seq_len_nt // options.pool_width
assert (target_length > 0)
# initialize sequences coverage file
seqs_cov_open = h5py.File(seqs_cov_file, 'w')
# seqs_cov_open.create_dataset('targets', shape=(num_seqs, target_length), dtype='float16')
targets_list = []
# open genome coverage file
genome_cov_open = CovFace(genome_cov_file)
# for each model sequence
for si in range(num_seqs):
mseq = model_seqs[si]
# read coverage
seq_cov_nt = genome_cov_open.read(mseq.chr, mseq.start, mseq.end)
# interpolate NaN
if options.interp_nan:
seq_cov_nt = interp_nan(seq_cov_nt)
# determine baseline coverage
if target_length >= 8:
baseline_cov = np.percentile(seq_cov_nt, 10)
baseline_cov = np.nan_to_num(baseline_cov)
else:
baseline_cov = 0
# set blacklist to baseline
if mseq.chr in black_chr_trees:
for black_interval in black_chr_trees[
mseq.chr][mseq.start:mseq.end]:
# adjust for sequence indexes
black_seq_start = black_interval.begin - mseq.start
black_seq_end = black_interval.end - mseq.start
seq_cov_nt[black_seq_start:black_seq_end] = baseline_cov
# set NaN's to baseline
if not options.interp_nan:
nan_mask = np.isnan(seq_cov_nt)
seq_cov_nt[nan_mask] = baseline_cov
# crop
if options.crop_bp > 0:
seq_cov_nt = seq_cov_nt[options.crop_bp:-options.crop_bp]
# sliding window
if options.step_bp > 0:
if options.padding == 'same':
seq_cov_nt = np.pad(seq_cov_nt,
(int(options.pool_width / 2 - 1),
int(options.pool_width / 2)), 'edge')
#
seq_cov = np.array(
list(
more_itertools.windowed(seq_cov_nt,
n=options.pool_width,
step=options.step_bp)))
# sum pool
else:
seq_cov = seq_cov_nt.reshape(target_length, options.pool_width)
if options.sum_stat == 'sum':
seq_cov = seq_cov.sum(axis=1, dtype='float32')
elif options.sum_stat in ['mean', 'avg']:
seq_cov = seq_cov.mean(axis=1, dtype='float32')
elif options.sum_stat == 'median':
seq_cov = seq_cov.median(axis=1)
elif options.sum_stat == 'max':
seq_cov = seq_cov.max(axis=1)
elif options.sum_stat == 'peak':
seq_cov = seq_cov.mean(axis=1, dtype='float32')
seq_cov = np.clip(np.sqrt(seq_cov * 4), 0, 1)
else:
print('ERROR: Unrecognized summary statistic "%s".' %
options.sum_stat,
file=sys.stderr)
exit(1)
# clip
if options.clip_soft is not None:
clip_mask = (seq_cov > options.clip_soft)
seq_cov[clip_mask] = options.clip_soft + np.sqrt(
seq_cov[clip_mask] - options.clip_soft)
if options.clip is not None:
seq_cov = np.clip(seq_cov, 0, options.clip)
# # threshold
# if options.threshold > 0:
# print('Filtering using threshold {}'.format(options.threshold))
# print('~~~')
# print(seq_cov.shape)
# print('~~~')
# scale
seq_cov = options.scale * seq_cov
if options.norm == 'log':
seq_cov = np.log(seq_cov + 1)
# print('LOG NORMALIZING THE DATA')
# save
targets_list.append(seq_cov.astype('float16'))
# print('!!!!!')
# print(np.array(targets_list, dtype='float16').shape)
# exit()
# write
# seqs_cov_open['targets'][si,:] = seq_cov.astype('float16')
# write all
seqs_cov_open.create_dataset('targets',
dtype='float16',
data=np.array(targets_list, dtype='float16'))
# close genome coverage file
genome_cov_open.close()
# close sequences coverage file
seqs_cov_open.close()
def main():
usage = (
"usage: %prog [options] <fasta_file> <seqs_bed_file> <seqs_cov_dir> <tfr_file>"
)
parser = OptionParser(usage)
parser.add_option("-g",
dest="genome_index",
default=None,
type="int",
help="Genome index")
parser.add_option(
"-s",
dest="start_i",
default=0,
type="int",
help="Sequence start index [Default: %default]",
)
parser.add_option(
"-e",
dest="end_i",
default=None,
type="int",
help="Sequence end index [Default: %default]",
)
parser.add_option(
"--te",
dest="target_extend",
default=None,
type="int",
help="Extend targets vector [Default: %default]",
)
parser.add_option(
"--ts",
dest="target_start",
default=0,
type="int",
help="Write targets into vector starting at index [Default: %default",
)
parser.add_option("-u",
dest="umap_npy",
help="Unmappable array numpy file")
parser.add_option(
"--umap_set",
dest="umap_set",
default=None,
type="float",
help=
"Sequence distribution value to set unmappable positions to, eg 0.25.",
)
(options, args) = parser.parse_args()
if len(args) != 4:
parser.error("Must provide input arguments.")
else:
fasta_file = args[0]
seqs_bed_file = args[1]
seqs_cov_dir = args[2]
tfr_file = args[3]
################################################################
# read model sequences
model_seqs = []
for line in open(seqs_bed_file):
a = line.split()
model_seqs.append(ModelSeq(a[0], int(a[1]), int(a[2]), None))
if options.end_i is None:
options.end_i = len(model_seqs)
num_seqs = options.end_i - options.start_i
################################################################
# determine sequence coverage files
seqs_cov_files = []
ti = 0
if options.genome_index is None:
seqs_cov_file = "%s/%d.h5" % (seqs_cov_dir, ti)
else:
seqs_cov_file = "%s/%d-%d.h5" % (seqs_cov_dir, options.genome_index,
ti)
while os.path.isfile(seqs_cov_file):
seqs_cov_files.append(seqs_cov_file)
ti += 1
if options.genome_index is None:
seqs_cov_file = "%s/%d.h5" % (seqs_cov_dir, ti)
else:
seqs_cov_file = "%s/%d-%d.h5" % (seqs_cov_dir,
options.genome_index, ti)
if len(seqs_cov_files) == 0:
print(
"Sequence coverage files not found, e.g. %s" % seqs_cov_file,
file=sys.stderr,
)
exit(1)
seq_pool_len = h5py.File(seqs_cov_files[0], "r")["seqs_cov"].shape[1]
num_targets = len(seqs_cov_files)
################################################################
# read targets
# extend targets
num_targets_tfr = num_targets
if options.target_extend is not None:
assert options.target_extend >= num_targets_tfr
num_targets_tfr = options.target_extend
# initialize targets
targets = np.zeros((num_seqs, seq_pool_len, num_targets_tfr),
dtype="float16")
# read each target
for ti in range(num_targets):
seqs_cov_open = h5py.File(seqs_cov_files[ti], "r")
tii = options.target_start + ti
targets[:, :, tii] = seqs_cov_open["seqs_cov"][
options.start_i:options.end_i, :]
seqs_cov_open.close()
################################################################
# modify unmappable
if options.umap_npy is not None and options.umap_set is not None:
unmap_mask = np.load(options.umap_npy)
for si in range(num_seqs):
msi = options.start_i + si
# determine unmappable null value
seq_target_null = np.percentile(targets[si],
q=[100 * options.umap_set],
axis=0)[0]
# set unmappable positions to null
targets[si, unmap_mask[msi, :], :] = np.minimum(
targets[si, unmap_mask[msi, :], :], seq_target_null)
################################################################
# write TFRecords
# open FASTA
fasta_open = pysam.Fastafile(fasta_file)
# define options
tf_opts = tf.python_io.TFRecordOptions(
tf.python_io.TFRecordCompressionType.ZLIB)
with tf.python_io.TFRecordWriter(tfr_file, tf_opts) as writer:
for si in range(num_seqs):
msi = options.start_i + si
mseq = model_seqs[msi]
# read FASTA
seq_dna = fasta_open.fetch(mseq.chr, mseq.start, mseq.end)
# one hot code
seq_1hot = dna_1hot(seq_dna)
if options.genome_index is None:
example = tf.train.Example(features=tf.train.Features(
feature={
"genome":
_int_feature(0),
"sequence":
_bytes_feature(seq_1hot.flatten().tostring()),
"target":
_bytes_feature(targets[si, :, :].flatten().tostring()),
}))
else:
example = tf.train.Example(features=tf.train.Features(
feature={
"genome":
_int_feature(options.genome_index),
"sequence":
_bytes_feature(seq_1hot.flatten().tostring()),
"target":
_bytes_feature(targets[si, :, :].flatten().tostring()),
}))
writer.write(example.SerializeToString())
fasta_open.close()
def main():
usage = 'usage: %prog [options] <genome_hic_file> <seqs_bed_file> <seqs_hic_file>'
parser = OptionParser(usage)
parser.add_option('-b',
dest='blacklist_bed',
help='Set blacklist nucleotides to a baseline value.')
parser.add_option(
'-c',
dest='clip',
default=None,
type='float',
help='Clip values post-summary to a maximum [Default: %default]')
parser.add_option('-s',
dest='scale',
default=1.,
type='float',
help='Scale values by [Default: %default]')
parser.add_option(
'--soft',
dest='soft_clip',
default=False,
action='store_true',
help=
'Soft clip values, applying sqrt to the execess above the threshold [Default: %default]'
)
parser.add_option(
'-u',
dest='sum_stat',
default='sum',
help='Summary statistic to compute in windows [Default: %default]')
parser.add_option('-w',
dest='pool_width',
default=1,
type='int',
help='Average pooling width [Default: %default]')
parser.add_option('--as_obsexp',
dest='as_obsexp',
default=False,
action="store_true",
help='save targets as obsexp profiles')
(options, args) = parser.parse_args()
if len(args) != 3:
parser.error('')
else:
genome_hic_file = args[0]
seqs_bed_file = args[1]
seqs_hic_file = args[2]
print('saving TFRs as obsexp:', options.as_obsexp)
# read model sequences
model_seqs = []
for line in open(seqs_bed_file):
a = line.split()
model_seqs.append(ModelSeq(a[0], int(a[1]), int(a[2]), None))
# read blacklist regions
black_chr_trees = read_blacklist(options.blacklist_bed)
# compute dimensions
num_seqs = len(model_seqs)
seq_len_nt = model_seqs[0].end - model_seqs[0].start
seq_len_pool = seq_len_nt // options.pool_width
# initialize sequences coverage file
seqs_hic_open = h5py.File(seqs_hic_file, 'w')
seqs_hic_open.create_dataset('seqs_hic',
shape=(num_seqs, seq_len_pool, seq_len_pool),
dtype='float16')
# open genome coverage file
genome_hic_cool = cooler.Cooler(genome_hic_file)
# check for "chr" prefix
chr_pre = 'chr1' in genome_hic_cool.chromnames
# assert that resolution matches
assert (options.pool_width == genome_hic_cool.info['bin-size'])
# for each model sequence
for si in range(num_seqs):
mseq = model_seqs[si]
try:
# pull hic values
if chr_pre:
mseq_str = '%s:%d-%d' % (mseq.chr, mseq.start, mseq.end)
else:
mseq_str = '%s:%d-%d' % (mseq.chr[3:], mseq.start, mseq.end)
#print('mseq_str:', mseq_str)
seq_hic_raw = genome_hic_cool.matrix(balance=True).fetch(mseq_str)
seq_hic_nan = np.isnan(seq_hic_raw)
if np.sum(seq_hic_nan[len(seq_hic_nan) // 2 -
1:len(seq_hic_nan) // 2 + 1,
len(seq_hic_nan) // 2 -
2:len(seq_hic_nan) // 2 + 2]) > 4:
print(
"WARNING: %s lots of zeros, check that umap_midpoint is correct %s. "
% (genome_hic_file, mseq_str))
# set blacklist to NaNs
if mseq.chr in black_chr_trees:
for black_interval in black_chr_trees[
mseq.chr][mseq.start:mseq.end]:
# adjust for sequence indexes
black_seq_start = (black_interval.begin -
mseq.start) // options.pool_width
black_seq_end = int(
np.ceil((black_interval.end - mseq.start) /
options.pool_width))
seq_hic_raw[:, black_seq_start:black_seq_end] = np.nan
seq_hic_raw[black_seq_start:black_seq_end, :] = np.nan
seq_hic_nan = np.isnan(seq_hic_raw)
# clip first diagonals and high values
clipval = np.nanmedian(np.diag(seq_hic_raw, 2))
for i in [-1, 0, 1]:
set_diag(seq_hic_raw, clipval, i)
seq_hic_raw = np.clip(seq_hic_raw, 0, seq_hic_raw)
seq_hic_raw[seq_hic_nan] = np.nan
# adaptively coarsegrain based on raw counts
seq_hic_smoothed = adaptive_coarsegrain(
seq_hic_raw,
genome_hic_cool.matrix(balance=False).fetch(mseq_str),
cutoff=2,
max_levels=8)
#todo: pass an option to add a certain pseudocount value, or the minimum nonzero value
if options.as_obsexp == True:
# interpolate single missing bins
seq_hic_interpolated = interpolate_bad_singletons(
seq_hic_smoothed,
mask=(~seq_hic_nan),
fillDiagonal=True,
returnMask=False,
secondPass=True,
verbose=False)
seq_hic_nan = np.isnan(seq_hic_interpolated)
# compute observed/expected
seq_hic_obsexp = observed_over_expected(
seq_hic_interpolated, ~seq_hic_nan)[0]
# todo: allow passing a global expected rather than computing locally
# log
seq_hic_obsexp = np.log(seq_hic_obsexp)
# set nan to 0
seq_hic_obsexp = np.nan_to_num(seq_hic_obsexp)
# todo: make obsexp_clip an option for obs/exp
seq_hic = np.clip(seq_hic_obsexp, -2, 2)
else:
# interpolate all missing bins
seq_hic_interpolated = interp_nan(seq_hic_smoothed)
# rescale
seq_hic = 100000 * seq_hic_interpolated
# todo add extra smoothing
except ValueError:
print("WARNING: %s doesn't see %s. Setting to all zeros." %
(genome_hic_file, mseq_str))
seq_hic = np.zeros((seq_len_pool, seq_len_pool), dtype='float16')
# write
seqs_hic_open['seqs_hic'][si, :, :] = seq_hic.astype('float16')
# close sequences coverage file
seqs_hic_open.close()
def main():
usage = 'usage: %prog [options] <fasta_file> <seqs_bed_file> <seqs_cov_dir> <tfr_file>'
parser = OptionParser(usage)
parser.add_option('-s',
dest='start_i',
default=0,
type='int',
help='Sequence start index [Default: %default]')
parser.add_option('-e',
dest='end_i',
default=None,
type='int',
help='Sequence end index [Default: %default]')
parser.add_option('-u',
dest='umap_npy',
help='Unmappable array numpy file')
parser.add_option(
'--umap_set',
dest='umap_set',
default=None,
type='float',
help=
'Sequence distribution value to set unmappable positions to, eg 0.25.')
(options, args) = parser.parse_args()
if len(args) != 4:
parser.error('Must provide input arguments.')
else:
fasta_file = args[0]
seqs_bed_file = args[1]
seqs_cov_dir = args[2]
tfr_file = args[3]
################################################################
# read model sequences
model_seqs = []
for line in open(seqs_bed_file):
a = line.split()
model_seqs.append(ModelSeq(a[0], int(a[1]), int(a[2])))
if options.end_i is None:
options.end_i = len(model_seqs)
num_seqs = options.end_i - options.start_i
################################################################
# determine sequence coverage files
seqs_cov_files = []
ti = 0
seqs_cov_file = '%s/%d.h5' % (seqs_cov_dir, ti)
while os.path.isfile(seqs_cov_file):
seqs_cov_files.append(seqs_cov_file)
ti += 1
seqs_cov_file = '%s/%d.h5' % (seqs_cov_dir, ti)
seq_pool_len = h5py.File(seqs_cov_files[0], 'r')['seqs_cov'].shape[1]
num_targets = len(seqs_cov_files)
################################################################
# read targets
# initialize targets
targets = np.zeros((num_seqs, seq_pool_len, num_targets), dtype='float16')
# read each target
for ti in range(num_targets):
seqs_cov_open = h5py.File(seqs_cov_files[ti], 'r')
targets[:, :, ti] = seqs_cov_open['seqs_cov'][
options.start_i:options.end_i, :]
seqs_cov_open.close()
################################################################
# modify unmappable
if options.umap_npy is not None and options.umap_set is not None:
unmap_mask = np.load(options.umap_npy)
for si in range(num_seqs):
msi = options.start_i + si
# determine unmappable null value
seq_target_null = np.percentile(targets[si],
q=[100 * options.umap_set],
axis=0)[0]
# set unmappable positions to null
targets[si, unmap_mask[msi, :], :] = np.minimum(
targets[si, unmap_mask[msi, :], :], seq_target_null)
################################################################
# write TFRecords
# open FASTA
fasta_open = pysam.Fastafile(fasta_file)
# define options
tf_opts = tf.python_io.TFRecordOptions(
tf.python_io.TFRecordCompressionType.ZLIB)
with tf.python_io.TFRecordWriter(tfr_file, tf_opts) as writer:
for si in range(num_seqs):
msi = options.start_i + si
mseq = model_seqs[msi]
# read FASTA
seq_dna = fasta_open.fetch(mseq.chr, mseq.start, mseq.end)
# one hot code
seq_1hot = dna_1hot(seq_dna)
# example = tf.train.Example(features=tf.train.Features(feature={
# 'sequence': _bytes_feature(seq_1hot.flatten().tostring()),
# 'target': _float_feature(targets[si,:,:].flatten())}))
example = tf.train.Example(features=tf.train.Features(
feature={
'sequence':
_bytes_feature(seq_1hot.flatten().tostring()),
'target':
_bytes_feature(targets[si, :, :].flatten().tostring())
}))
writer.write(example.SerializeToString())
fasta_open.close()
def main():
usage = "usage: %prog [options] <genome_cov_file> <seqs_bed_file> <seqs_cov_file>"
parser = OptionParser(usage)
parser.add_option(
"-b",
dest="blacklist_bed",
help="Set blacklist nucleotides to a baseline value.",
)
parser.add_option(
"-c",
dest="clip",
default=None,
type="float",
help="Clip values post-summary to a maximum [Default: %default]",
)
parser.add_option(
"-s",
dest="scale",
default=1.0,
type="float",
help="Scale values by [Default: %default]",
)
parser.add_option(
"--soft",
dest="soft_clip",
default=False,
action="store_true",
help="Soft clip values, applying sqrt to the execess above the threshold [Default: %default]",
)
parser.add_option(
"-u",
dest="sum_stat",
default="sum",
help="Summary statistic to compute in windows [Default: %default]",
)
parser.add_option(
"-w",
dest="pool_width",
default=1,
type="int",
help="Average pooling width [Default: %default]",
)
(options, args) = parser.parse_args()
if len(args) != 3:
parser.error("")
else:
genome_cov_file = args[0]
seqs_bed_file = args[1]
seqs_cov_file = args[2]
# read model sequences
model_seqs = []
for line in open(seqs_bed_file):
a = line.split()
model_seqs.append(ModelSeq(a[0], int(a[1]), int(a[2]), None))
# read blacklist regions
black_chr_trees = read_blacklist(options.blacklist_bed)
# compute dimensions
num_seqs = len(model_seqs)
seq_len_nt = model_seqs[0].end - model_seqs[0].start
seq_len_pool = seq_len_nt // options.pool_width
# initialize sequences coverage file
seqs_cov_open = h5py.File(seqs_cov_file, "w")
seqs_cov_open.create_dataset(
"seqs_cov", shape=(num_seqs, seq_len_pool), dtype="float16"
)
# open genome coverage file
genome_cov_open = CovFace(genome_cov_file)
# for each model sequence
for si in range(num_seqs):
mseq = model_seqs[si]
# read coverage
seq_cov_nt = genome_cov_open.read(mseq.chr, mseq.start, mseq.end)
# determine baseline coverage
baseline_cov = np.percentile(seq_cov_nt, 10)
baseline_cov = np.nan_to_num(baseline_cov)
# set blacklist to baseline
if mseq.chr in black_chr_trees:
for black_interval in black_chr_trees[mseq.chr][mseq.start : mseq.end]:
# adjust for sequence indexes
black_seq_start = black_interval.begin - mseq.start
black_seq_end = black_interval.end - mseq.start
seq_cov_nt[black_seq_start:black_seq_end] = baseline_cov
# set NaN's to baseline
nan_mask = np.isnan(seq_cov_nt)
seq_cov_nt[nan_mask] = baseline_cov
# sum pool
seq_cov = seq_cov_nt.reshape(seq_len_pool, options.pool_width)
if options.sum_stat == "sum":
seq_cov = seq_cov.sum(axis=1, dtype="float32")
elif options.sum_stat in ["mean", "avg"]:
seq_cov = seq_cov.mean(axis=1, dtype="float32")
elif options.sum_stat == "median":
seq_cov = seq_cov.median(axis=1, dtype="float32")
elif options.sum_stat == "max":
seq_cov = seq_cov.max(axis=1, dtype="float32")
else:
print(
'ERROR: Unrecognized summary statistic "%s".' % options.sum_stat,
file=sys.stderr,
)
exit(1)
# clip
if options.clip is not None:
if options.soft_clip:
clip_mask = seq_cov > options.clip
seq_cov[clip_mask] = options.clip + np.sqrt(
seq_cov[clip_mask] - options.clip
)
else:
seq_cov = np.clip(seq_cov, 0, options.clip)
# scale
seq_cov = options.scale * seq_cov
# write
seqs_cov_open["seqs_cov"][si, :] = seq_cov.astype("float16")
# close genome coverage file
genome_cov_open.close()
# close sequences coverage file
seqs_cov_open.close()
def main():
usage = 'usage: %prog [options] <fasta_file> <targets_file>'
parser = OptionParser(usage)
parser.add_option('-b', dest='blacklist_bed',
help='Set blacklist nucleotides to a baseline value.')
parser.add_option('--break', dest='break_t',
default=8388608, type='int',
help='Break in half contigs above length [Default: %default]')
parser.add_option('-c', '--crop', dest='crop_bp',
default=0, type='int',
help='Crop bp off each end [Default: %default]')
parser.add_option('-d', dest='diagonal_offset',
default=2, type='int',
help='Positions on the diagonal to ignore [Default: %default]')
parser.add_option('-f', dest='folds',
default=None, type='int',
help='Generate cross fold split [Default: %default]')
parser.add_option('-g', dest='gaps_file',
help='Genome assembly gaps BED [Default: %default]')
parser.add_option('-k', dest='kernel_stddev',
default=0, type='int',
help='Gaussian kernel stddev to smooth values [Default: %default]')
parser.add_option('-l', dest='seq_length',
default=131072, type='int',
help='Sequence length [Default: %default]')
parser.add_option('--limit', dest='limit_bed',
help='Limit to segments that overlap regions in a BED file')
parser.add_option('--local', dest='run_local',
default=False, action='store_true',
help='Run jobs locally as opposed to on SLURM [Default: %default]')
parser.add_option('-o', dest='out_dir',
default='data_out',
help='Output directory [Default: %default]')
parser.add_option('-p', dest='processes',
default=None, type='int',
help='Number parallel processes [Default: %default]')
parser.add_option('-r', dest='seqs_per_tfr',
default=128, type='int',
help='Sequences per TFRecord file [Default: %default]')
parser.add_option('--restart', dest='restart',
default=False, action='store_true',
help='Continue progress from midpoint. [Default: %default]')
parser.add_option('--sample', dest='sample_pct',
default=1.0, type='float',
help='Down-sample the segmenDown-sample the segments')
parser.add_option('--seed', dest='seed',
default=44, type='int',
help='Random seed [Default: %default]')
parser.add_option('--stride_train', dest='stride_train',
default=1., type='float',
help='Stride to advance train sequences [Default: seq_length]')
parser.add_option('--stride_test', dest='stride_test',
default=1., type='float',
help='Stride to advance valid and test sequences [Default: seq_length]')
parser.add_option('--st', '--split_test', dest='split_test',
default=False, action='store_true',
help='Exit after split. [Default: %default]')
parser.add_option('-t', dest='test_pct_or_chr',
default=0.05, type='str',
help='Proportion of the data for testing [Default: %default]')
parser.add_option('-u', dest='umap_bed',
help='Unmappable regions in BED format')
parser.add_option('--umap_midpoints', dest='umap_midpoints',
help='Regions with midpoints to exclude in BED format. Used for 4C/HiC.')
parser.add_option('--umap_t', dest='umap_t',
default=0.3, type='float',
help='Remove sequences with more than this unmappable bin % [Default: %default]')
parser.add_option('--umap_set', dest='umap_set',
default=None, type='float',
help='Set unmappable regions to this percentile in the sequences\' distribution of values')
parser.add_option('-w', dest='pool_width',
default=128, type='int',
help='Sum pool width [Default: %default]')
parser.add_option('-v', dest='valid_pct_or_chr',
default=0.05, type='str',
help='Proportion of the data for validation [Default: %default]')
parser.add_option('--snap', dest='snap',
default=None, type='int',
help='snap stride to multiple for binned targets in bp, if not None seq_length must be a multiple of snap')
parser.add_option('--as_obsexp', dest='as_obsexp',
action="store_true", default=False,
help='save targets as obsexp profiles')
parser.add_option('--global_obsexp', dest='global_obsexp',
action="store_true", default=False,
help='use pre-calculated by-chromosome obs/exp')
parser.add_option('--no_log', dest='no_log',
action="store_true", default=False,
help='do not take log for obs/exp')
(options, args) = parser.parse_args()
if len(args) != 2:
parser.error('Must provide FASTA and sample coverage labels and paths.')
else:
fasta_file = args[0]
targets_file = args[1]
random.seed(options.seed)
np.random.seed(options.seed)
# transform proportion strides to base pairs
if options.stride_train <= 1:
print('stride_train %.f'%options.stride_train, end='')
options.stride_train = options.stride_train*options.seq_length
print(' converted to %f' % options.stride_train)
options.stride_train = int(np.round(options.stride_train))
if options.stride_test <= 1:
print('stride_test %.f'%options.stride_test, end='')
options.stride_test = options.stride_test*options.seq_length
print(' converted to %f' % options.stride_test)
options.stride_test = int(np.round(options.stride_test))
if options.snap != None:
if np.mod(options.seq_length, options.snap) != 0:
raise ValueError('seq_length must be a multiple of snap')
if np.mod(options.stride_train, options.snap) != 0:
raise ValueError('stride_train must be a multiple of snap')
if np.mod(options.stride_test, options.snap) != 0:
raise ValueError('stride_test must be a multiple of snap')
if os.path.isdir(options.out_dir) and not options.restart:
print('Remove output directory %s or use --restart option.' % options.out_dir)
exit(1)
elif not os.path.isdir(options.out_dir):
os.mkdir(options.out_dir)
# dump options
with open('%s/options.json' % options.out_dir, 'w') as options_json_out:
json.dump(options.__dict__, options_json_out, sort_keys=True, indent=4)
################################################################
# define genomic contigs
################################################################
if not options.restart:
chrom_contigs = genome.load_chromosomes(fasta_file)
# remove gaps
if options.gaps_file:
chrom_contigs = genome.split_contigs(chrom_contigs,
options.gaps_file)
# ditch the chromosomes for contigs
contigs = []
for chrom in chrom_contigs:
contigs += [Contig(chrom, ctg_start, ctg_end)
for ctg_start, ctg_end in chrom_contigs[chrom]]
# limit to a BED file
if options.limit_bed is not None:
contigs = limit_contigs(contigs, options.limit_bed)
# filter for large enough
contigs = [ctg for ctg in contigs if ctg.end - ctg.start >= options.seq_length]
# break up large contigs
if options.break_t is not None:
contigs = break_large_contigs(contigs, options.break_t)
# print contigs to BED file
ctg_bed_file = '%s/contigs.bed' % options.out_dir
write_seqs_bed(ctg_bed_file, contigs)
################################################################
# divide between train/valid/test
################################################################
# label folds
if options.folds is not None:
fold_labels = ['fold%d' % fi for fi in range(options.folds)]
num_folds = options.folds
else:
fold_labels = ['train', 'valid', 'test']
num_folds = 3
if not options.restart:
if options.folds is not None:
# divide by fold pct
fold_contigs = divide_contigs_folds(contigs, options.folds)
else:
try:
# convert to float pct
valid_pct = float(options.valid_pct_or_chr)
test_pct = float(options.test_pct_or_chr)
assert(0 <= valid_pct <= 1)
assert(0 <= test_pct <= 1)
# divide by pct
fold_contigs = divide_contigs_pct(contigs, test_pct, valid_pct)
except (ValueError, AssertionError):
# divide by chr
valid_chrs = options.valid_pct_or_chr.split(',')
test_chrs = options.test_pct_or_chr.split(',')
fold_contigs = divide_contigs_chr(contigs, test_chrs, valid_chrs)
# rejoin broken contigs within set
for fi in range(len(fold_contigs)):
fold_contigs[fi] = rejoin_large_contigs(fold_contigs[fi])
# write labeled contigs to BED file
ctg_bed_file = '%s/contigs.bed' % options.out_dir
ctg_bed_out = open(ctg_bed_file, 'w')
for fi in range(len(fold_contigs)):
for ctg in fold_contigs[fi]:
line = '%s\t%d\t%d\t%s' % (ctg.chr, ctg.start, ctg.end, fold_labels[fi])
print(line, file=ctg_bed_out)
ctg_bed_out.close()
if options.split_test:
exit()
################################################################
# define model sequences
################################################################
if not options.restart:
fold_mseqs = []
for fi in range(num_folds):
if fold_labels[fi] in ['valid','test']:
stride_fold = options.stride_test
else:
stride_fold = options.stride_train
# stride sequences across contig
fold_mseqs_fi = contig_sequences(fold_contigs[fi], options.seq_length,
stride_fold, options.snap, fold_labels[fi])
fold_mseqs.append(fold_mseqs_fi)
# shuffle
random.shuffle(fold_mseqs[fi])
# down-sample
if options.sample_pct < 1.0:
fold_mseqs[fi] = random.sample(fold_mseqs[fi], int(options.sample_pct*len(fold_mseqs[fi])))
# merge into one list
mseqs = [ms for fm in fold_mseqs for ms in fm]
################################################################
# mappability
################################################################
if not options.restart:
if (options.umap_bed is not None) or (options.umap_midpoints is not None):
if shutil.which('bedtools') is None:
print('Install Bedtools to annotate unmappable sites', file=sys.stderr)
exit(1)
if options.umap_bed is not None:
# annotate unmappable positions
mseqs_unmap = annotate_unmap(mseqs, options.umap_bed,
options.seq_length, options.pool_width)
# filter unmappable
mseqs_map_mask = (mseqs_unmap.mean(axis=1, dtype='float64') < options.umap_t)
mseqs = [mseqs[i] for i in range(len(mseqs)) if mseqs_map_mask[i]]
mseqs_unmap = mseqs_unmap[mseqs_map_mask,:]
# write to file
unmap_npy = '%s/mseqs_unmap.npy' % options.out_dir
np.save(unmap_npy, mseqs_unmap)
if options.umap_midpoints is not None:
# annotate unmappable midpoints for 4C/HiC
mseqs_unmap = annotate_unmap(mseqs, options.umap_midpoints,
options.seq_length, options.pool_width)
# filter unmappable
seqmid = mseqs_unmap.shape[1]//2 #int( options.seq_length / options.pool_width /2)
mseqs_map_mask = (np.sum(mseqs_unmap[:,seqmid-1:seqmid+1],axis=1) == 0)
mseqs = [mseqs[i] for i in range(len(mseqs)) if mseqs_map_mask[i]]
mseqs_unmap = mseqs_unmap[mseqs_map_mask,:]
# write to file
unmap_npy = '%s/mseqs_unmap_midpoints.npy' % options.out_dir
np.save(unmap_npy, mseqs_unmap)
# write sequences to BED
print('writing sequences to BED')
seqs_bed_file = '%s/sequences.bed' % options.out_dir
write_seqs_bed(seqs_bed_file, mseqs, True)
else:
# read from directory
seqs_bed_file = '%s/sequences.bed' % options.out_dir
unmap_npy = '%s/mseqs_unmap.npy' % options.out_dir
mseqs = []
fold_mseqs = []
for fi in range(num_folds):
fold_mseqs.append([])
for line in open(seqs_bed_file):
a = line.split()
msg = ModelSeq(a[0], int(a[1]), int(a[2]), a[3])
mseqs.append(msg)
if a[3] == 'train':
fi = 0
elif a[3] == 'valid':
fi = 1
elif a[3] == 'test':
fi = 2
else:
fi = int(a[3].replace('fold',''))
fold_mseqs[fi].append(msg)
################################################################
# read sequence coverage values
################################################################
# read target datasets
targets_df = pd.read_csv(targets_file, index_col=0, sep='\t')
seqs_cov_dir = '%s/seqs_cov' % options.out_dir
if not os.path.isdir(seqs_cov_dir):
os.mkdir(seqs_cov_dir)
read_jobs = []
for ti in range(targets_df.shape[0]):
genome_cov_file = targets_df['file'].iloc[ti]
seqs_cov_stem = '%s/%d' % (seqs_cov_dir, ti)
seqs_cov_file = '%s.h5' % seqs_cov_stem
clip_ti = None
if 'clip' in targets_df.columns:
clip_ti = targets_df['clip'].iloc[ti]
# scale_ti = 1
# if 'scale' in targets_df.columns:
# scale_ti = targets_df['scale'].iloc[ti]
if options.restart and os.path.isfile(seqs_cov_file):
print('Skipping existing %s' % seqs_cov_file, file=sys.stderr)
else:
cmd = 'python ~/nn_anopheles/basenji/bin/akita_data_read.py'
cmd += ' --crop %d' % options.crop_bp
cmd += ' -d %s' % options.diagonal_offset
cmd += ' -k %d' % options.kernel_stddev
cmd += ' -w %d' % options.pool_width
if clip_ti is not None:
cmd += ' --clip %f' % clip_ti
# cmd += ' -s %f' % scale_ti
if options.blacklist_bed:
cmd += ' -b %s' % options.blacklist_bed
if options.as_obsexp:
cmd += ' --as_obsexp'
if options.global_obsexp:
cmd += ' --global_obsexp'
if options.no_log:
cmd += ' --no_log'
cmd += ' %s' % genome_cov_file
cmd += ' %s' % seqs_bed_file
cmd += ' %s' % seqs_cov_file
if options.run_local:
# breaks on some OS
# cmd += ' &> %s.err' % seqs_cov_stem
read_jobs.append(cmd)
else:
j = slurm.Job(cmd,
name='read_t%d' % ti,
out_file='%s.out' % seqs_cov_stem,
err_file='%s.err' % seqs_cov_stem,
queue='standard', mem=15000, time='12:0:0')
read_jobs.append(j)
if options.run_local:
util.exec_par(read_jobs, options.processes, verbose=True)
else:
slurm.multi_run(read_jobs, options.processes, verbose=True,
launch_sleep=1, update_sleep=5)
################################################################
# write TF Records
################################################################
# copy targets file
shutil.copy(targets_file, '%s/targets.txt' % options.out_dir)
# initialize TF Records dir
tfr_dir = '%s/tfrecords' % options.out_dir
if not os.path.isdir(tfr_dir):
os.mkdir(tfr_dir)
write_jobs = []
for fold_set in fold_labels:
fold_set_indexes = [i for i in range(len(mseqs)) if mseqs[i].label == fold_set]
fold_set_start = fold_set_indexes[0]
fold_set_end = fold_set_indexes[-1] + 1
tfr_i = 0
tfr_start = fold_set_start
tfr_end = min(tfr_start+options.seqs_per_tfr, fold_set_end)
while tfr_start <= fold_set_end:
tfr_stem = '%s/%s-%d' % (tfr_dir, fold_set, tfr_i)
cmd = 'python ~/nn_anopheles/basenji/bin/basenji_data_write.py'
cmd += ' -s %d' % tfr_start
cmd += ' -e %d' % tfr_end
# do not use
# if options.umap_bed is not None:
# cmd += ' -u %s' % unmap_npy
# if options.umap_set is not None:
# cmd += ' --umap_set %f' % options.umap_set
cmd += ' %s' % fasta_file
cmd += ' %s' % seqs_bed_file
cmd += ' %s' % seqs_cov_dir
cmd += ' %s.tfr' % tfr_stem
if options.run_local:
# breaks on some OS
# cmd += ' &> %s.err' % tfr_stem
write_jobs.append(cmd)
else:
j = slurm.Job(cmd,
name='write_%s-%d' % (fold_set, tfr_i),
out_file='%s.out' % tfr_stem,
err_file='%s.err' % tfr_stem,
queue='standard', mem=15000, time='12:0:0')
write_jobs.append(j)
# update
tfr_i += 1
tfr_start += options.seqs_per_tfr
tfr_end = min(tfr_start+options.seqs_per_tfr, fold_set_end)
if options.run_local:
util.exec_par(write_jobs, options.processes, verbose=True)
else:
slurm.multi_run(write_jobs, options.processes, verbose=True,
launch_sleep=1, update_sleep=5)
################################################################
# stats
################################################################
stats_dict = {}
stats_dict['num_targets'] = targets_df.shape[0]
stats_dict['seq_length'] = options.seq_length
stats_dict['pool_width'] = options.pool_width
stats_dict['crop_bp'] = options.crop_bp
stats_dict['diagonal_offset'] = options.diagonal_offset
target1_length = options.seq_length - 2*options.crop_bp
target1_length = target1_length // options.pool_width
target1_length = target1_length - options.diagonal_offset
target_length = target1_length*(target1_length+1) // 2
stats_dict['target_length'] = target_length
for fi in range(num_folds):
stats_dict['%s_seqs' % fold_labels[fi]] = len(fold_mseqs[fi])
with open('%s/statistics.json' % options.out_dir, 'w') as stats_json_out:
json.dump(stats_dict, stats_json_out, indent=4)
def main():
usage = 'usage: %prog [options] <fasta_file> <seqs_bed_file> <seqs_cov_dir> <tfr_file>'
parser = OptionParser(usage)
parser.add_option('-g',
dest='genome_index',
default=None,
type='int',
help='Genome index')
parser.add_option('-s',
dest='start_i',
default=0,
type='int',
help='Sequence start index [Default: %default]')
parser.add_option('-e',
dest='end_i',
default=None,
type='int',
help='Sequence end index [Default: %default]')
parser.add_option('--te',
dest='target_extend',
default=None,
type='int',
help='Extend targets vector [Default: %default]')
parser.add_option(
'--ts',
dest='target_start',
default=0,
type='int',
help='Write targets into vector starting at index [Default: %default')
parser.add_option('-u',
dest='umap_npy',
help='Unmappable array numpy file')
parser.add_option(
'--umap_set',
dest='umap_set',
default=None,
type='float',
help=
'Sequence distribution value to set unmappable positions to, eg 0.25.')
(options, args) = parser.parse_args()
if len(args) != 4:
parser.error('Must provide input arguments.')
else:
fasta_file = args[0]
seqs_bed_file = args[1]
seqs_cov_dir = args[2]
tfr_file = args[3]
################################################################
# read model sequences
model_seqs = []
for line in open(seqs_bed_file):
a = line.split()
model_seqs.append(ModelSeq(a[0], int(a[1]), int(a[2]), None))
if options.end_i is None:
options.end_i = len(model_seqs)
num_seqs = options.end_i - options.start_i
################################################################
# determine sequence coverage files
seqs_cov_files = []
ti = 0
if options.genome_index is None:
seqs_cov_file = '%s/%d.h5' % (seqs_cov_dir, ti)
else:
seqs_cov_file = '%s/%d-%d.h5' % (seqs_cov_dir, options.genome_index,
ti)
while os.path.isfile(seqs_cov_file):
seqs_cov_files.append(seqs_cov_file)
ti += 1
if options.genome_index is None:
seqs_cov_file = '%s/%d.h5' % (seqs_cov_dir, ti)
else:
seqs_cov_file = '%s/%d-%d.h5' % (seqs_cov_dir,
options.genome_index, ti)
if len(seqs_cov_files) == 0:
print('Sequence coverage files not found, e.g. %s' % seqs_cov_file,
file=sys.stderr)
exit(1)
seq_pool_len_hic = h5py.File(seqs_cov_files[0], 'r')['targets'].shape[1]
num_targets = len(seqs_cov_files)
################################################################
# read targets
# extend targets
num_targets_tfr = num_targets
if options.target_extend is not None:
assert (options.target_extend >= num_targets_tfr)
num_targets_tfr = options.target_extend
# initialize targets
targets = np.zeros((num_seqs, seq_pool_len_hic, num_targets_tfr),
dtype='float16')
# read each target
for ti in range(num_targets):
seqs_cov_open = h5py.File(seqs_cov_files[ti], 'r')
tii = options.target_start + ti
targets[:, :, tii] = seqs_cov_open['targets'][
options.start_i:options.end_i, :]
seqs_cov_open.close()
################################################################
# write TFRecords
# open FASTA
fasta_open = pysam.Fastafile(fasta_file)
# define options
tf_opts = tf.io.TFRecordOptions(compression='ZLIB')
with tf.io.TFRecordWriter(tfr_file, tf_opts) as writer:
for si in range(num_seqs):
msi = options.start_i + si
mseq = model_seqs[msi]
# read FASTA
seq_dna = fasta_open.fetch(mseq.chr, mseq.start, mseq.end)
# one hot code
seq_1hot = dna_1hot(seq_dna)
if options.genome_index is None:
example = tf.train.Example(features=tf.train.Features(
feature={
'genome':
_int_feature(0),
'sequence':
_bytes_feature(seq_1hot.flatten().tostring()),
'target':
_bytes_feature(targets[si, :, :].flatten().tostring())
}))
else:
example = tf.train.Example(features=tf.train.Features(
feature={
'genome':
_int_feature(options.genome_index),
'sequence':
_bytes_feature(seq_1hot.flatten().tostring()),
'target':
_bytes_feature(targets[si, :, :].flatten().tostring())
}))
writer.write(example.SerializeToString())
fasta_open.close()
def main():
usage = 'usage: %prog [options] <fasta_file> <seqs_bed_file> <seqs_cov_dir> <tfr_file> <fold_set>'
parser = OptionParser(usage)
parser.add_option('--threshold',
dest='threshold',
default=0,
type='float',
help='Set a minimum threshold for activity.')
parser.add_option(
'--test_threshold',
dest='test_threshold',
type='float',
help='Set a minimum threshold for activity for test set.')
parser.add_option('-s',
dest='start_i',
default=0,
type='int',
help='Sequence start index [Default: %default]')
parser.add_option('-e',
dest='end_i',
default=None,
type='int',
help='Sequence end index [Default: %default]')
parser.add_option('--te',
dest='target_extend',
default=None,
type='int',
help='Extend targets vector [Default: %default]')
parser.add_option(
'--ts',
dest='target_start',
default=0,
type='int',
help='Write targets into vector starting at index [Default: %default')
parser.add_option('-u',
dest='umap_npy',
help='Unmappable array numpy file')
parser.add_option(
'--umap_clip',
dest='umap_clip',
default=1,
type='float',
help=
'Clip values at unmappable positions to distribution quantiles, eg 0.25. [Default: %default]'
)
parser.add_option(
'--umap_tfr',
dest='umap_tfr',
default=False,
action='store_true',
help='Save umap array into TFRecords [Default: %default]')
parser.add_option('-o',
dest='out_dir',
default='data_out',
help='Output directory [Default: %default]')
(options, args) = parser.parse_args()
if len(args) != 5:
parser.error('Must provide input arguments.')
else:
fasta_file = args[0]
seqs_bed_file = args[1]
seqs_cov_dir = args[2]
tfr_file = args[3]
fold_set = args[4]
if fold_set == 'test':
options.threshold = options.test_threshold
################################################################
# read model sequences
model_seqs = []
for line in open(seqs_bed_file):
a = line.split()
model_seqs.append(ModelSeq(a[0], int(a[1]), int(a[2]), None))
if options.end_i is None:
options.end_i = len(model_seqs)
num_seqs = options.end_i - options.start_i
################################################################
# determine sequence coverage files
seqs_cov_files = []
ti = 0
seqs_cov_file = '%s/%d.h5' % (seqs_cov_dir, ti)
while os.path.isfile(seqs_cov_file):
seqs_cov_files.append(seqs_cov_file)
ti += 1
seqs_cov_file = '%s/%d.h5' % (seqs_cov_dir, ti)
if len(seqs_cov_files) == 0:
print('Sequence coverage files not found, e.g. %s' % seqs_cov_file,
file=sys.stderr)
exit(1)
seq_pool_len = h5py.File(seqs_cov_files[0], 'r')['targets'].shape[1]
num_targets = len(seqs_cov_files)
################################################################
# read targets
# extend targets
num_targets_tfr = num_targets
if options.target_extend is not None:
assert (options.target_extend >= num_targets_tfr)
num_targets_tfr = options.target_extend
# initialize targets
targets = np.zeros((num_seqs, seq_pool_len, num_targets_tfr),
dtype='float16')
# read each target
for ti in range(num_targets):
seqs_cov_open = h5py.File(seqs_cov_files[ti], 'r')
tii = options.target_start + ti
targets[:, :, tii] = seqs_cov_open['targets'][
options.start_i:options.end_i, :]
seqs_cov_open.close()
# threshold each sequence using an arbitrary threshold
mask_by_thr = np.any(np.any(targets > options.threshold, axis=1), axis=-1)
idx_filt_seqs = np.argwhere(mask_by_thr).flatten()
num_seqs_to_add = len(idx_filt_seqs)
for i in range(5):
print('*')
print(num_seqs_to_add)
for i in range(5):
print('*')
# current_json = open('%s/statistics.json' % options.out_dir, 'r')
# current_stats = json.load(current_json)
# current_stats['%s_seqs'%fold_set] += num_seqs_to_add # update number of seqs
# with open('%s/statistics.json' % options.out_dir, 'w') as stats_json_out:
# json.dump(current_stats, stats_json_out, indent=4)
count_dir = os.path.join(options.out_dir, 'counts')
if not os.path.isdir(count_dir):
os.mkdir(count_dir)
file_id = fold_set + '_' + uuid.uuid4().hex
file_path = os.path.join(count_dir, file_id)
f = open(file_path, 'w')
f.write(str(num_seqs_to_add))
f.close()
################################################################
# modify unmappable
#
# if options.umap_npy is not None and options.umap_clip < 1:
# unmap_mask = np.load(options.umap_npy)
#
# for si in idx_filt_seqs:
# msi = options.start_i + si
#
# # determine unmappable null value
# seq_target_null = np.percentile(targets[si], q=[100*options.umap_clip], axis=0)[0]
#
# # set unmappable positions to null
# targets[si,unmap_mask[msi,:],:] = np.minimum(targets[si,unmap_mask[msi,:],:], seq_target_null)
#
# elif options.umap_npy is not None and options.umap_tfr:
# unmap_mask = np.load(options.umap_npy)
################################################################
# write TFRecords
# open FASTA
fasta_open = pysam.Fastafile(fasta_file)
# define options
tf_opts = tf.io.TFRecordOptions(compression_type='ZLIB')
with tf.io.TFRecordWriter(tfr_file, tf_opts) as writer:
for si in idx_filt_seqs:
msi = options.start_i + si
mseq = model_seqs[msi]
# read FASTA
seq_dna = fasta_open.fetch(mseq.chr, mseq.start, mseq.end)
# one hot code
seq_1hot = dna_1hot(seq_dna)
# seq_1hot = dna_1hot_index(seq_dna) # more efficient, but fighting inertia
# hash to bytes
features_dict = {
'coordinate':
feature_str('{}_{}_{}'.format(mseq.chr, mseq.start,
mseq.end).encode()),
# 'sequence': feature_bytes(seq_1hot),
'sequence':
feature_bytes(seq_1hot),
'target':
feature_bytes(targets[si, :, :])
}
# features_dict = {
# 'chrom': feature_str(mseq.chr.encode()),
# # 'sequence': feature_bytes(seq_1hot),
# 'start': feature_floats(mseq.start),
# 'end': feature_floats(mseq.end),
# 'sequence': feature_bytes(seq_1hot),
# 'target': feature_bytes(targets[si,:,:])
# }
# add unmappability
if options.umap_tfr:
features_dict['umap'] = feature_bytes(unmap_mask[msi, :])
# write example
example = tf.train.Example(features=tf.train.Features(
feature=features_dict))
writer.write(example.SerializeToString())
fasta_open.close()
def main():
usage = 'usage: %prog [options] <genome_cov_file> <seqs_bed_file> <seqs_cov_file>'
parser = OptionParser(usage)
parser.add_option('-w',
dest='pool_width',
default=1,
type='int',
help='Average pooling width [Default: %default]')
(options, args) = parser.parse_args()
if len(args) != 3:
parser.error('')
else:
genome_cov_file = args[0]
seqs_bed_file = args[1]
seqs_cov_file = args[2]
# read model sequences
model_seqs = []
for line in open(seqs_bed_file):
a = line.split()
model_seqs.append(ModelSeq(a[0], int(a[1]), int(a[2])))
# compute dimensions
num_seqs = len(model_seqs)
seq_len_nt = model_seqs[0].end - model_seqs[0].start
seq_len_pool = seq_len_nt // options.pool_width
# initialize sequences coverage file
seqs_cov_open = h5py.File(seqs_cov_file, 'w')
seqs_cov_open.create_dataset('seqs_cov',
shape=(num_seqs, seq_len_pool),
dtype='float16')
# open genome coverage file
genome_cov_open = h5py.File(genome_cov_file, 'r')
# for each model sequence
for si in range(num_seqs):
mseq = model_seqs[si]
# read coverage
if mseq.chr in genome_cov_open:
seq_cov_nt = genome_cov_open[mseq.chr][mseq.start:mseq.end]
else:
print("WARNING: %s doesn't see %s:%d-%d. Setting to all zeros." %
(cov_file, mseq.chr, mseq.start, mseq.end))
seq_cov_nt = np.zeros(mseq.end - mseq.start, dtype='float16')
# set NaN's to zero
seq_cov_nt = np.nan_to_num(seq_cov_nt)
# sum pool
seq_cov = seq_cov_nt.reshape(seq_len_pool, options.pool_width)
seq_cov = seq_cov.sum(axis=1, dtype='float32')
# write
seqs_cov_open['seqs_cov'][si, :] = seq_cov
# close genome coverage file
genome_cov_open.close()
# close sequences coverage file
seqs_cov_open.close()