def indel_anavar(arg_tuple):
ins_sfs, ins_m, del_sfs, del_m, bootstrap, n, region, c, out_stem = arg_tuple
anavar_path = '/shared/evolgen1/shared_data/program_files/iceberg/'
anavar_cmd = '{path}anavar1.2 {ctl} {rslts} {log}'
results = []
for i in [0] + range(1, bootstrap + 1):
# sort sfs
if i == 0:
sfs_i = ins_sfs
sfs_d = del_sfs
else:
sfs_i = resample_replace(ins_sfs)
sfs_d = resample_replace(del_sfs)
# convert to correct format for anavar
sfs_i = sfs2counts(sfs_i, n)
sfs_d = sfs2counts(sfs_d, n)
# sort file names
ctl_name = out_stem + '.rep{}.control.txt'.format(i)
result_name = out_stem + '.rep{}.results.txt'.format(i)
log_name = out_stem + '.rep{}.log.txt'.format(i)
# construct control file
sfs_m = {'INS': (sfs_i, ins_m), 'DEL': (sfs_d, del_m)}
ctl = an.Indel1ControlFile()
ctl.set_data(sfs_m, n, dfe='discrete', c=c)
ctl_contents = ctl.construct()
with open(ctl_name, 'w') as control:
control.write(ctl_contents)
# call anavar
rep_cmd = anavar_cmd.format(path=anavar_path,
ctl=ctl_name,
rslts=result_name,
log=log_name)
subprocess.call(rep_cmd, shell=True)
# process results
with open(result_name) as rep_results:
results_data = an.ResultsFile(rep_results)
header = list(results_data.header()) + ['rep', 'region']
ml_est = results_data.ml_estimate(
as_string=True) + '\t{}\t{}'.format(i, region)
if i == 0:
results.append('\t'.join(header))
results.append(ml_est)
return results
def main():
res = sys.argv[1]
results = an.ResultsFile(open(res))
dn_truth = (0.000121257528807, 0.000200465669246) # (ins, del)
ds_truth = (0.00183918200407, 0.00383421025726) # (ins, del)
print('percent_bias_adjust',
'alpha',
'var_type',
'rDI_cds',
'rDI_nc',
sep='\t')
# estimate alpha assuming different % errors of div polarisation ie 1% error
for i in range(0, 26):
if i == 0:
dn_adjuster = 0
ds_adjuster = 0
else:
percent_error = i / float(100)
dn_adjuster = sum(dn_truth) * percent_error
ds_adjuster = sum(ds_truth) * percent_error
for direction in [-1, 1]:
dn_adj = adjust_div(dn_truth, dn_adjuster, direction)
ds_adj = adjust_div(ds_truth, ds_adjuster, direction)
ins_alpha = results.get_alpha(dn=dn_adj[0],
ds=ds_adj[0],
var_type='ins')
del_alpha = results.get_alpha(dn=dn_adj[1],
ds=ds_adj[1],
var_type='del')
print(i * direction,
ins_alpha,
'ins',
dn_adj[1] / dn_adj[0],
ds_adj[1] / ds_adj[0],
sep='\t')
print(i * direction,
del_alpha,
'del',
dn_adj[1] / dn_adj[0],
ds_adj[1] / ds_adj[0],
sep='\t')
def main():
header = [
'run', 'imp', 'exit_code', 'neu_theta_1', 'neu_gamma_1', 'neu_e_1',
'sel_theta', 'sel_shape', 'sel_scale', 'sel_e', 'lnL'
]
additional = ['converged', 'bounds', 'rep', 'region']
head_out = header + additional
print(*head_out, sep=',')
res_list = []
for res in sys.stdin:
res_stem = res.split('/')[-1]
bs = int(res_stem.split('.bsrep')[-1].split('.')[0])
region = res_stem.split('_')[1]
res = res.rstrip()
results = an.ResultsFile(open(res))
mle = results.ml_estimate()
converged = results.converged()
bounds = results.bounds_hit(gamma_r=(-500, 100),
theta_r=(1e-14, 0.1),
r_r=(0.01, 100),
scale_r=(0.1, 5000.0))
current_res = [mle[x] for x in header
] + [converged, ';'.join(bounds), bs, region]
res_list.append([bs, current_res])
for processed_res in sorted(res_list, key=lambda x: x[0]):
print(*processed_res[1], sep=',')
def sel_v_neu_anavar(mode, vcf, call, sel_region, constraint, n, c, dfe, alg,
nnoimp, maximp, out_stem, search, degree, spread, evolgen,
start_index, given, ar_ref):
"""
submits anavar jobs to cluster after writing required files etc
:param mode: str
:param vcf: str
:param call: dict
:param sel_region: str
:param constraint: str
:param n: int
:param c: int
:param dfe: str
:param alg: str
:param nnoimp: int
:param maximp: int
:param out_stem: str
:param search: int
:param degree: int
:param spread: int
:param evolgen: bool
:param start_index: int
:param given: bool
:param ar_ref: bool
:return: None
"""
anavar_path = '/shared/evolgen1/shared_data/program_files/sharc/'
anavar_cmd = '{path}anavar1.4 {ctl} {rslts} {log} {seed}'
# sort file names
ctl_name = out_stem + '.control.txt'
merge_out = out_stem + '.merged.results.txt'
# catch given on first run
init = ()
if given:
if not os.path.isfile(merge_out):
sys.exit(
'Given True but no previous runs completed to take besty res from'
)
else:
# get best result from merged out
best_res = an.ResultsFile(
open(merge_out)).ml_estimate(as_string=True)
init = tuple(best_res.split()[3:-1])
# region combinations
region_combs = {
'CDS': ['CDS_frameshift', 'CDS_non_frameshift'],
'intron': ['intron'],
'intergenic': ['intergenic'],
'noncoding': ['intergenic', 'intron']
}
# make control file
if mode == 'snp':
sfs_data = prepare_snp_sfs(vcf,
call,
n,
sel_sfs_regions=region_combs[sel_region],
call_sel_reg=sel_region)
ctl = an.SNPNeuSelControlFile()
else:
sfs_data = prepare_indel_sfs(vcf,
call,
n,
sel_sfs_regions=region_combs[sel_region],
call_sel_reg=sel_region,
ar_ref=ar_ref)
ctl = an.IndelNeuSelControlFile()
ctl.set_alg_opts(search=search,
alg=alg,
key=3,
epsabs=1e-20,
epsrel=1e-9,
rftol=1e-9,
maxtime=3600,
optional=True,
maximp=maximp,
nnoimp=nnoimp,
init=init)
ctl.set_data(sfs_data,
n,
dfe=dfe,
c=c,
gamma_r=(-5e4, 1e5),
theta_r=(1e-14, 0.1),
r_r=(0.01, 100),
scale_r=(0.1, 5000.0))
if degree != 50:
ctl.set_dfe_optional_opts(degree=degree, optional=True)
ctl.set_constraint(constraint)
ctl_contents = ctl.construct()
with open(ctl_name, 'w') as control:
control.write(ctl_contents)
res_file_list = out_stem + '.allres.list.txt'
hjids = []
with open(res_file_list, 'a') as res_list:
# split into requested jobs
for i in range(start_index, start_index + spread):
split_stem = '{}.split{}'.format(out_stem, i)
result_name = split_stem + '.results.txt'
log_name = split_stem + '.log.txt'
print(result_name, file=res_list)
# call anavar
rep_cmd = anavar_cmd.format(path=anavar_path,
ctl=ctl_name,
rslts=result_name,
log=log_name,
seed=i)
q_sub([rep_cmd],
out=split_stem,
jid=split_stem.split('/')[-1] + '.sh',
t=48,
evolgen=evolgen)
hjids.append(split_stem.split('/')[-1] + '.sh')
# hold job to merge outputs
gather = 'cat {} | ~/parus_indel/anavar_analyses/gather_searches.py {}'.format(
res_file_list, merge_out)
q_sub([gather], out=out_stem + '.merge', hold=hjids, evolgen=evolgen)
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
'-file_pattern',
help=
'takes a regular expression in order to extract a custom ID from a file'
'name, along with a column name eg) degree,_degree(\d+)\.')
parser.add_argument('-neu_ref',
help='sets neutral reference type to determine ds',
default='ar',
choices=['ar', 'nc'])
args = parser.parse_args()
counter = 0
if args.file_pattern is not None:
spec_col = args.file_pattern.split(',')[0]
spec_pattern = args.file_pattern.split(',')[1]
else:
spec_col, spec_pattern = None, None
all_res = []
for res in sys.stdin:
# sort custom col contents
if spec_col is not None:
spec_val = re.search(spec_pattern, res).group(1)
else:
spec_val = None
res = res.rstrip()
if 'equal_t' in res:
constraint = '_equal_t'
else:
constraint = ''
results = an.ResultsFile(open(res))
mle = results.ml_estimate()
n_class = results.num_class()
variant = results.data_type()
dfe = results.dfe()
free_params = len(results.free_parameters())
if constraint == '_equal_t':
if args.neu_ref == 'ar':
i_ds = 0.00183918200407
d_ds = 0.00383421025726
else:
i_ds = 0.00171438749928
d_ds = 0.00308343831154
ins_alpha = results.get_alpha(dn=0.000121257528807,
ds=i_ds,
var_type='ins')
del_alpha = results.get_alpha(dn=0.000200465669246,
ds=d_ds,
var_type='del')
else:
ins_alpha = 'NA'
del_alpha = 'NA'
mod_name = '{}_{}_{}class{}'.format(variant, dfe, n_class, constraint)
reformed = reformat_mle(mle,
n_class,
variant,
results.converged(),
results.bounds_hit(gamma_r=(-5e4, 1e5),
theta_r=(1e-12, 0.1),
r_r=(0.01, 100),
scale_r=(0.1, 5000.0)),
mod_name,
free_params,
dfe,
results.num_runs(),
alpha_ins=ins_alpha,
alpha_del=del_alpha,
custom_col=spec_col,
custom_val=spec_val)
if counter == 0:
all_res.append(reformed[0])
counter += 1
for x in reformed[1]:
all_res.append(x)
# calc AIC
aics = sorted([z[-1] for z in all_res[1:]])
best_aic = aics[0]
print(*all_res[0] + ['delta_AIC'], sep=',')
out_data = []
for line in all_res[1:]:
delta = delta_aic(line[-1], best_aic)
out_data.append((delta, line + [delta]))
for line in sorted(out_data, reverse=True):
print(*line[1], sep=',')
def sel_v_neu_anavar(sfs_dat, constraint, n, c, dfe, alg, nnoimp, maximp,
out_stem, search, degree, spread, start_index, given):
"""
submits anavar jobs to cluster after writing required files etc
:param sfs_dat: dict
:param constraint: str
:param n: int
:param c: int
:param dfe: str
:param alg: str
:param nnoimp: int
:param maximp: int
:param out_stem: str
:param search: int
:param degree: int
:param spread: int
:param start_index: int
:param given: bool
:return: None
"""
anavar_path = ''
anavar_cmd = '{path}anavar {ctl} {rslts} {log} {seed}'
# sort file names
ctl_name = out_stem + '.control.txt'
merge_out = out_stem + '.merged.results.txt'
# catch given on first run
init = ()
if given:
if not os.path.isfile(merge_out):
sys.exit('Given True but no previous runs completed to take besty res from')
else:
# get best result from merged out
best_res = an.ResultsFile(open(merge_out)).ml_estimate(as_string=True)
init = tuple(best_res.split()[3:-1])
# make control file
ctl = an.SNPNeuSelControlFile()
ctl.set_alg_opts(search=search, alg=alg, key=3,
epsabs=1e-20, epsrel=1e-9, rftol=1e-9,
maxtime=3600, optional=True,
maximp=maximp, nnoimp=nnoimp, init=init)
ctl.set_data(sfs_dat, n, dfe=dfe, c=c, gamma_r=(-500, 100), theta_r=(1e-14, 0.1), r_r=(0.01, 100),
scale_r=(0.1, 5000.0), snp_fold=False)
if degree != 50:
ctl.set_dfe_optional_opts(degree=degree, optional=True)
ctl.set_constraint(constraint)
ctl_contents = ctl.construct()
with open(ctl_name, 'w') as control:
control.write(ctl_contents)
res_file_list = out_stem + '.allres.list.txt'
with open(res_file_list, 'a') as res_list:
# split into requested jobs
for i in range(start_index, start_index+spread):
split_stem = '{}.split{}'.format(out_stem, i)
result_name = split_stem + '.results.txt'
log_name = split_stem + '.log.txt'
print(result_name, file=res_list)
# call anavar
rep_cmd = anavar_cmd.format(path=anavar_path, ctl=ctl_name, rslts=result_name, log=log_name, seed=i)
q_sub([rep_cmd], out=split_stem, jid=split_stem.split('/')[-1] + '.sh', t=48, scheduler='SLURM')
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
'-file_pattern',
help=
'takes a regular expression in order to extract a custom ID from a file'
'name, along with a column name eg) degree,_degree(\d+)\.')
args = parser.parse_args()
counter = 0
if args.file_pattern is not None:
spec_col = args.file_pattern.split(',')[0]
spec_pattern = args.file_pattern.split(',')[1]
else:
spec_col, spec_pattern = None, None
all_res = []
for res in sys.stdin:
# sort custom col contents
if spec_col is not None:
spec_val = re.search(spec_pattern, res).group(1)
else:
spec_val = None
res = res.rstrip()
if 'equal_t' in res:
constraint = '_equal_t'
else:
constraint = ''
results = an.ResultsFile(open(res))
mle = results.ml_estimate()
n_class = results.num_class()
variant = results.data_type()
dfe = results.dfe()
free_params = len(results.free_parameters())
mod_name = '{}_{}_{}class{}'.format(variant, dfe, n_class, constraint)
reformed = reformat_mle(mle,
n_class,
variant,
results.converged(),
results.bounds_hit(gamma_r=(-5e4, 1e3),
theta_r=(1e-10, 0.1),
r_r=(0.01, 100),
scale_r=(0.1, 5000.0)),
mod_name,
free_params,
dfe,
results.num_runs(),
custom_col=spec_col,
custom_val=spec_val)
if counter == 0:
all_res.append(reformed[0])
counter += 1
for x in reformed[1]:
all_res.append(x)
# calc AIC
aics = sorted([z[-1] for z in all_res[1:]])
best_aic = aics[0]
print(*all_res[0] + ['delta_AIC'], sep=',')
out_data = []
for line in all_res[1:]:
delta = delta_aic(line[-1], best_aic)
out_data.append((delta, line + [delta]))
for line in sorted(out_data, reverse=True):
print(*line[1], sep=',')