def _run_vep(self, input_vcf, output_vcf):
"""
Just run VEP to the xbrowse configurations
"""
vep_command = [
self._vep_perl_path,
"--offline",
"--cache",
"--everything", # http://useast.ensembl.org/info/docs/tools/vep/script/vep_options.html#opt_everything
"--vcf",
"--fork", "4",
"--fasta", os.path.join(self._vep_cache_dir,
"homo_sapiens/78_GRCh37/Homo_sapiens.GRCh37.75.dna.primary_assembly.fa"),
# "--filter", "transcript_ablation,splice_donor_variant,splice_acceptor_variant,frameshift_variant,"
# "stop_gained,stop_lost,initiator_codon_variant,transcript_amplification,"
# "inframe_insertion,inframe_deletion,missense_variant,splice_region_variant,"
# "incomplete_terminal_codon_variant,stop_retained_variant,synonymous_variant,coding_sequence_variant,"
# "mature_miRNA_variant,5_prime_UTR_variant,3_prime_UTR_variant,intron_variant,NMD_transcript_variant,"
# "TFBS_ablation,TFBS_amplification,TF_binding_site_variant",
"--force_overwrite",
"--dir", self._vep_cache_dir,
"-i", input_vcf,
"-o", output_vcf,
]
if self._human_ancestor_fa is not None:
vep_command += [
"--plugin", "LoF,human_ancestor_fa:{}".format(self._human_ancestor_fa),
]
print("Running VEP:\n" + " ".join(vep_command))
sh.perl(vep_command)
def _run_vep(self, input_vcf, output_vcf):
"""
Just run VEP to the xbrowse configurations
"""
vep_command = [
self._vep_perl_path,
"--offline",
"--cache",
"--everything", # http://useast.ensembl.org/info/docs/tools/vep/script/vep_options.html#opt_everything
"--vcf",
"--fork", "4",
"--fasta", os.path.join(self._vep_cache_dir,
"homo_sapiens/78_GRCh37/Homo_sapiens.GRCh37.75.dna.primary_assembly.fa"),
# "--filter", "transcript_ablation,splice_donor_variant,splice_acceptor_variant,frameshift_variant,"
# "stop_gained,stop_lost,initiator_codon_variant,transcript_amplification,"
# "inframe_insertion,inframe_deletion,missense_variant,splice_region_variant,"
# "incomplete_terminal_codon_variant,stop_retained_variant,synonymous_variant,coding_sequence_variant,"
# "mature_miRNA_variant,5_prime_UTR_variant,3_prime_UTR_variant,intron_variant,NMD_transcript_variant,"
# "TFBS_ablation,TFBS_amplification,TF_binding_site_variant",
"--force_overwrite",
"--dir", self._vep_cache_dir,
"-i", input_vcf,
"-o", output_vcf,
]
if self._human_ancestor_fa is not None:
vep_command += [
"--plugin", "LoF,human_ancestor_fa:{}".format(self._human_ancestor_fa),
]
print("Running VEP:\n" + " ".join(vep_command))
sh.perl(vep_command)
def build_arch(self, arch):
options_iphoneos = (
"-isysroot {}".format(arch.sysroot),
"-DOPENSSL_THREADS",
"-D_REENTRANT",
"-DDSO_DLFCN",
"-DHAVE_DLFCN_H",
"-fomit-frame-pointer",
"-fno-common",
"-O3"
)
build_env = arch.get_env()
target = arch_mapper[arch.arch]
shprint(sh.env, _env=build_env)
sh.perl(join(self.build_dir, "Configure"),
target,
_env=build_env)
if target == 'iphoneos-cross':
sh.sed("-ie", "s!^CFLAG=.*!CFLAG={} {}!".format(build_env['CFLAGS'],
" ".join(options_iphoneos)),
"Makefile")
sh.sed("-ie", "s!static volatile sig_atomic_t intr_signal;!static volatile intr_signal;! ",
"crypto/ui/ui_openssl.c")
else:
sh.sed("-ie", "s!^CFLAG=!CFLAG={} !".format(build_env['CFLAGS']),
"Makefile")
shprint(sh.make, "clean")
shprint(sh.make, "-j4", "build_libs")
def build_arch(self, arch):
options_iphoneos = ("-isysroot {}".format(arch.sysroot),
"-DOPENSSL_THREADS", "-D_REENTRANT", "-DDSO_DLFCN",
"-DHAVE_DLFCN_H", "-DOPENSSL_NO_ASYNC",
"-fomit-frame-pointer", "-fno-common", "-O3")
build_env = arch.get_env()
build_env["C_INCLUDE_PATH"] = join(arch.sysroot, "usr", "include")
target = arch_mapper[arch.arch]
shprint(sh.env, _env=build_env)
sh.perl(join(self.build_dir, "Configure"),
"-DOPENSSL_NO_ASYNC",
target,
_env=build_env)
if target == 'iphoneos-cross':
sh.sed(
"-ie",
"s!^CFLAG=.*!CFLAG={} {}!".format(build_env['CFLAGS'],
" ".join(options_iphoneos)),
"Makefile")
sh.sed(
"-ie",
"s!static volatile sig_atomic_t intr_signal;!static volatile intr_signal;! ",
"crypto/ui/ui_openssl.c")
else:
sh.sed("-ie", "s!^CFLAG=!CFLAG={} !".format(build_env['CFLAGS']),
"Makefile")
shprint(sh.make, "clean")
shprint(sh.make,
self.ctx.concurrent_make,
"build_libs",
_env=build_env)
def _run_vep(self, input_vcf, output_vcf):
"""
Just run VEP to the xbrowse configurations
"""
vep_command = [
self._vep_perl_path,
"--offline",
"--protein",
"--vcf",
"--force_overwrite",
"--dir", self._vep_cache_dir,
"-i", input_vcf,
"-o", output_vcf,
]
if self._human_ancestor_fa is not None:
vep_command += [
"--plugin",
"LoF,human_ancestor_fa:{}".format(self._human_ancestor_fa),
]
if platform.system() == 'Darwin':
vep_command.append("--compress")
vep_command.append("gunzip -c")
sh.perl(vep_command)
def build_x86_64(self):
arch = self.archs[0]
sdk_path = sh.xcrun("--sdk", "macosx", "--show-sdk-path").strip()
dist_dir = join(self.ctx.dist_dir, "hostopenssl")
print("OpenSSL for host to be installed at: {}").format(dist_dir)
sh.perl(join(self.build_dir, "Configure"), "darwin64-x86_64-cc",
"--openssldir={}".format(dist_dir),
"--prefix={}".format(dist_dir))
shprint(sh.make, "clean")
shprint(sh.make, "-j4", "build_libs")
def _run_vep(self, in_file):
out_file = append_stem(in_file, "vep")
if file_exists(out_file):
return out_file
with file_transaction(out_file) as tmp_out_file:
sh.perl(self.vep, "-i", in_file, "-o", tmp_out_file,
species=self.species, _convert_underscore=False,
**self.options)
return out_file
def build_arch(self, arch):
build_env = arch.get_env()
target = arch_mapper[arch.arch]
shprint(sh.env, _env=build_env)
sh.perl(join(self.build_dir, "Configure"), target, _env=build_env)
if target.endswith('-cross'):
with open('Makefile', 'r') as makefile:
filedata = makefile.read()
filedata = filedata.replace('$(CROSS_TOP)/SDKs/$(CROSS_SDK)',
arch.sysroot)
with open('Makefile', 'w') as makefile:
makefile.write(filedata)
shprint(sh.make, "clean")
shprint(sh.make, self.ctx.concurrent_make, "build_libs")
def _run_vep(self, in_file):
out_file = append_stem(in_file, "vep")
if file_exists(out_file):
return out_file
with file_transaction(out_file) as tmp_out_file:
sh.perl(self.vep,
"-i",
in_file,
"-o",
tmp_out_file,
species=self.species,
_convert_underscore=False,
**self.options)
return out_file
def normalize_transcription(transcriptions, wsj_root: Path):
""" Passes the dirty transcription dict to a Kaldi Perl script for cleanup.
We use the original Perl file, to make sure, that the cleanup is done
exactly as it is done by Kaldi.
:param transcriptions: Dirty transcription dictionary
:param wsj_root: Path to WSJ database
:return result: Clean transcription dictionary
"""
assert len(transcriptions) > 0, 'No transcriptions to clean up.'
with tempfile.TemporaryDirectory() as temporary_directory:
temporary_directory = Path(temporary_directory).absolute()
with open(temporary_directory / 'dirty.txt', 'w') as f:
for key, value in transcriptions.items():
f.write('{} {}\n'.format(key, value))
result = sh.perl(
sh.cat(str(temporary_directory / 'dirty.txt')),
kaldi_wsj_tools / 'normalize_transcript.pl',
'<NOISE>'
)
result = [line.split(maxsplit=1) for line in result.strip().split('\n')]
result = {k: v for k, v in result}
return result
def main(base_directory, simulation_type):
flag = None
if (simulation_type == "forward"):
flag = "-f"
elif (simulation_type == "source"):
flag = "-a"
elif (simulation_type == "structure"):
flag = "-b"
elif (simulation_type == "forward_save"):
flag = "-F"
else:
raise Exception("no such simulation type")
all_simulation_directories = sorted(glob(join(base_directory, "*")))
current_path = str(sh.pwd())[:-1] # pylint: disable=not-callable
for each_simulation_directory in all_simulation_directories:
sh.cd(each_simulation_directory)
sh.perl("change_simulation_type.pl", flag)
sh.cd(current_path)
def ancestral_reconstr(self, outgroup = None):
print "Make trait File"
if not os.path.exists(self.anc_rec_path):
os.makedirs(self.anc_rec_path)
json2nexus(self.processed_clusters, self.anc_rec_path + "Trait_genome.fasta", self.anc_rec_path + "Traits.nex" )
with open(self.anc_rec_path + "Trait_genome.fasta") as handle:
seqs=[s for s in SeqIO.parse(handle,"fasta") ]
for s in seqs:
s.seq.alphabet = Bio.Alphabet.DNAAlphabet()
with open(self.anc_rec_path + "Matrix.nex","w") as handle:
SeqIO.write(seqs,handle,"nexus")
os.system("sed -i 's/format datatype=dna missing=? gap=- interleave;/format interleave datatype=standard gap=- symbols=\"0123456789ABCDEF\";/' " + self.anc_rec_path + "Matrix.nex")
print "Make tree file"
new = dendropy.Tree.get(path=self.scg_tree, schema="newick")
# new.write(path = self.anc_rec_path + "Tree.nex", schema="nexus")
tree_string = re.sub("[0-9]{2,}","", re.sub(r':[0123456789.]+','', str(new)))
taxons = {x : i for i,x in enumerate([t.taxon.label for t in new.nodes() if t.is_leaf()])}
for k in taxons.keys():
tree_string=re.sub(k+"([),])",str(taxons[k]+1)+r"\1", tree_string)
taxons_t = sorted(taxons.items(), key=operator.itemgetter(1))
t_table = "\n".join([("\t\t" + str(p[1]+1) + "\t" + p[0].replace(" ","_") +",") for p in taxons_t])
paup_tree = """#NEXUS
Set AllowPunct=Yes;
BEGIN TREES;
\tTRANSLATE
%s\b
;
TREE Bioperl_1 = [&U] %s ;
END;
""" % (t_table , tree_string)
paup_script = """
#NEXUS
BEGIN PAUP;
log file=%sPaup.log replace=yes start=yes;
exe %sMatrix.nex;
exe %sAssumptions.nex;
exe %sTree.nex;
ctype GeneCopy:All;
pset Opt=AccTran;
taxset out= %d;
outgroup %s;
set root=outgroup outroot=monophyl;
root root=outgroup;
describetrees 1/Xout=both ApoList=Yes ChgList=yes; [brLens=yes;]
savetrees from=1 to=1 format=nexus root=yes file=%sOutTree.rtree;
q;
END;
""" % (self.anc_rec_path, self.anc_rec_path, self.anc_rec_path, self.anc_rec_path, len(seqs), outgroup if outgroup else "out", self.anc_rec_path)
assumptions="""
#NEXUS
BEGIN ASSUMPTIONS;
USERTYPE GeneCopy (STEPMATRIX) = 16
0 1 2 3 4 5 6 7 8 9 A B C D E F
[0] . 10.0 11.0 11.2 11.4 11.6 11.8 12.0 12.2 12.4 12.6 12.8 13.0 13.2 13.4 13.6
[1] 5.0 . 1.0 1.2 1.4 1.6 1.8 2.0 2.2 2.4 2.6 2.8 3.0 3.2 3.4 3.6
[2] 5.2 0.2 . 0.2 0.4 0.6 0.8 1.0 1.2 1.4 1.6 1.8 2.0 2.2 2.4 2.6
[3] 5.4 0.4 0.2 . 0.2 0.4 0.6 0.8 1.0 1.2 1.4 1.6 1.8 2.0 2.2 2.4
[4] 5.6 0.6 0.4 0.2 . 0.2 0.4 0.6 0.8 1.0 1.2 1.4 1.6 1.8 2.0 2.2
[5] 5.8 0.8 0.6 0.4 0.2 . 0.2 0.4 0.6 0.8 1.0 1.2 1.4 1.6 1.8 2.0
[6] 6.0 1.0 0.8 0.6 0.4 0.2 . 0.2 0.4 0.6 0.8 1.0 1.2 1.4 1.6 1.8
[7] 6.2 1.2 1.0 0.8 0.6 0.4 0.2 . 0.2 0.4 0.6 0.8 1.0 1.2 1.4 1.6
[8] 6.4 1.4 1.2 1.0 0.8 0.6 0.4 0.2 . 0.2 0.4 0.6 0.8 1.0 1.2 1.4
[9] 6.6 1.6 1.4 1.2 1.0 0.8 0.6 0.4 0.2 . 0.2 0.4 0.6 0.8 1.0 1.2
[A] 6.8 1.8 1.6 1.4 1.2 1.0 0.8 0.6 0.4 0.2 . 0.2 0.4 0.6 0.8 1.0
[B] 7.0 2.0 1.8 1.6 1.4 1.2 1.0 0.8 0.6 0.4 0.2 . 0.2 0.4 0.6 0.8
[C] 7.2 2.2 2.0 1.8 1.6 1.4 1.2 1.0 0.8 0.6 0.4 0.2 . 0.2 0.4 0.6
[D] 7.4 2.4 2.2 2.0 1.8 1.6 1.4 1.2 1.0 0.8 0.6 0.4 0.2 . 0.2 0.4
[E] 7.6 2.6 2.4 2.2 2.0 1.8 1.6 1.4 1.2 1.0 0.8 0.6 0.4 0.2 . 0.2
[F] 7.8 2.8 2.6 2.4 2.2 2.0 1.8 1.6 1.4 1.2 1.0 0.8 0.6 0.4 0.2 .
;
charset All =1-.;
END;
"""
print "Write nexus files"
with open(self.anc_rec_path + "Script.nex", "w") as handle:
handle.writelines(paup_script)
with open(self.anc_rec_path + "Assumptions.nex", "w") as handle:
handle.writelines(assumptions)
with open(self.anc_rec_path + "Tree.nex", "w") as handle:
handle.writelines(paup_tree)
print "Run and parse paup"
os.system("paup4a146_centos64 -f -n " + self.anc_rec_path + "Script.nex > " + self.anc_rec_path + "OutPut.nex")
sh.perl("/pica/h1/moritz/repos/Pyscratches/20150610_orthomcl_tools/orthmcl_tools/parsePaupLog.pl", self.anc_rec_path + "OutPut.nex")
os.remove(self.anc_rec_path + "OutPut.nex.nodes.johan")
print "Merge data with clusters"
merged_json = mergeJSONwPAUP(self.processed_clusters, self.anc_rec_path + "OutPut.nex.changes")
with open(self.anc_rec_path + "clusters.json","w") as handle:
json.dump(merged_json, handle, indent=4, sort_keys=True)
return merged_json
#!/home/jinho93/miniconda3/bin/python
from xml.etree.ElementTree import iterparse
import sh
import numpy as np
import sys
out = str(sh.perl('/home/jinho93/bin/checkforce', '-v'))
arr = [r.split() for r in out.split('\n')[:-4]]
selective = []
for event, elem in iterparse('vasprun.xml', events=('end', )):
if 'name' in elem.attrib.keys() and elem.attrib['name'] == 'selective':
for i in elem:
selective.append(
[True if r is 'T' else False for r in str(i.text).split()])
break
for i, j in zip(selective, arr):
if j:
for k in range(3):
if not i[k]:
j[3 + k] = 'conv'
print(' '.join(j))
