def _setup(self, config, temp): with open(os.path.join(temp, "contigs.table"), "w") as handle: handle.write("ID\tSize\tNs\tHits\n") # Workaround for pysam < 0.9 returning list, >= 0.9 returning str for line in "".join(pysam.idxstats(self._input_file)).split("\n"): line = line.strip() if not line: continue name, size, hits, _ = line.split("\t") name = self._mapping.get(name, name) if name not in self._contigs: # Excluding contigs is allowed continue row = { "ID": name, "Size": self._contigs[name]["Size"], "Ns": self._contigs[name]["Ns"], "Hits": hits, } handle.write("{ID}\t{Size}\t{Ns}\t{Hits}\n".format(**row)) CommandNode._setup(self, config, temp)
def _setup(self, config, temp): CommandNode._setup(self, config, temp) # The temp folder may contain old files: # Remove old pipes to prevent failure at _teardown for pipe_fname in glob.glob(os.path.join(temp, "pipe*")): fileutils.try_remove(pipe_fname) # ExaML refuses to overwrite old info files fileutils.try_remove(os.path.join(temp, "ExaML_info.Pypeline")) # Resume from last checkpoint, if one such was generated checkpoints = glob.glob(os.path.join(temp, "ExaML_binaryCheckpoint.Pypeline_*")) if not checkpoints: return cache = FileStatusCache() if not cache.are_files_outdated(self.input_files, checkpoints): checkpoints.sort(key=lambda fname: int(fname.rsplit("_", 1)[-1])) # FIXME: Less hacky solution to modifying AtomicCmds needed self._command._command.append("-R") self._command._command.append(checkpoints[-1]) else: for fpath in checkpoints: fileutils.try_remove(fpath)
def _setup(self, config, temp): CommandNode._setup(self, config, temp) input_files = [ self._input_file, fileutils.swap_ext(self._input_file, ".bim"), fileutils.swap_ext(self._input_file, ".fam"), ] for filename in input_files: basename = os.path.basename(filename) os.symlink(os.path.abspath(filename), os.path.join(temp, basename)) if self._supervised: fam_filename = fileutils.swap_ext(self._input_file, ".fam") pop_filename = fileutils.swap_ext(fam_filename, ".pop") pop_filename = fileutils.reroot_path(temp, pop_filename) key = "Group(%i)" % (self._k_groups,) with open(fam_filename) as fam_handle: with open(pop_filename, "w") as pop_handle: for line in fam_handle: sample, _ = line.split(None, 1) group = self._samples.get(sample, {}).get(key, "-") pop_handle.write("%s\n" % (group,))
def _setup(self, config, temp): CommandNode._setup(self, config, temp) for fname in ("3pGtoA_freq.txt", "5pCtoT_freq.txt", "dnacomp.txt", "misincorporation.txt"): relpath = os.path.join(self._directory, fname) abspath = os.path.abspath(relpath) os.symlink(abspath, os.path.join(temp, fname))
def _setup(self, config, temp): with open(os.path.join(temp, "contigs.table"), "w") as handle: handle.write("ID\tSize\tNs\tHits\n") # Workaround for pysam < 0.9 returning list, >= 0.9 returning str for line in "".join(pysam.idxstats(self._input_file)).split('\n'): line = line.strip() if not line: continue name, size, hits, _ = line.split('\t') name = contig_name_to_plink_name(name) if name is None or not (name.isdigit() or name == 'X'): continue if int(size) != self._contigs[name]['Size']: raise NodeError("TODO: size mismatch") row = { 'ID': name, 'Size': self._contigs[name]['Size'], 'Ns': self._contigs[name]['Ns'], 'Hits': hits, } handle.write('{ID}\t{Size}\t{Ns}\t{Hits}\n'.format(**row)) CommandNode._setup(self, config, temp)
def _setup(self, config, temp): """See CommandNode._setup.""" infile = os.path.abspath(self._infile) outfile = reroot_path(temp, self._infile) os.symlink(infile, outfile) CommandNode._setup(self, config, temp)
def _setup(self, config, temp): CommandNode._setup(self, config, temp) # The temp folder may contain old files: # Remove old pipes to prevent failure at _teardown for pipe_fname in glob.glob(os.path.join(temp, "pipe*")): fileutils.try_remove(pipe_fname) # ExaML refuses to overwrite old info files fileutils.try_remove(os.path.join(temp, "ExaML_info.Pypeline")) # Resume from last checkpoint, if one such was generated checkpoints = glob.glob( os.path.join(temp, "ExaML_binaryCheckpoint.Pypeline_*")) if not checkpoints: return cache = FileStatusCache() if not cache.are_files_outdated(self.input_files, checkpoints): checkpoints.sort(key=lambda fname: int(fname.rsplit("_", 1)[-1])) # FIXME: Less hacky solution to modifying AtomicCmds needed self._command._command.append("-R") self._command._command.append(checkpoints[-1]) else: for fpath in checkpoints: fileutils.try_remove(fpath)
def _setup(self, config, temp): CommandNode._setup(self, config, temp) for fname in ("3pGtoA_freq.txt", "5pCtoT_freq.txt", "dnacomp.txt", "misincorporation.txt"): relpath = os.path.join(self._directory, fname) abspath = os.path.abspath(relpath) os.symlink(abspath, os.path.join(temp, fname))
def _setup(self, config, temp): CommandNode._setup(self, config, temp) input_files = [ self._input_file, fileutils.swap_ext(self._input_file, ".bim"), fileutils.swap_ext(self._input_file, ".fam"), ] for filename in input_files: basename = os.path.basename(filename) os.symlink(os.path.abspath(filename), os.path.join(temp, basename)) if self._supervised: fam_filename = fileutils.swap_ext(self._input_file, ".fam") pop_filename = fileutils.swap_ext(fam_filename, ".pop") pop_filename = fileutils.reroot_path(temp, pop_filename) key = "Group(%i)" % (self._k_groups, ) with open(fam_filename) as fam_handle: with open(pop_filename, "w") as pop_handle: for line in fam_handle: sample, _ = line.split(None, 1) group = self._samples.get(sample, {}).get(key, "-") pop_handle.write("%s\n" % (group, ))
def _setup(self, config, temp): """See CommandNode._setup.""" infile = os.path.abspath(self._infile) outfile = reroot_path(temp, self._infile) os.symlink(infile, outfile) CommandNode._setup(self, config, temp)
def _setup(self, config, temp): with open(os.path.join(temp, "contigs.table"), "w") as handle: handle.write("ID\tSize\tNs\tHits\n") # Workaround for pysam < 0.9 returning list, >= 0.9 returning str for line in "".join(pysam.idxstats(self._input_file)).split('\n'): line = line.strip() if not line: continue name, size, hits, _ = line.split('\t') name = contig_name_to_plink_name(name) if name is None or not (name.isdigit() or name == 'X'): continue elif name not in self._contigs: # Excluding contigs is allowed continue if int(size) != self._contigs[name]['Size']: raise NodeError( "Size mismatch between database and BAM; " "expected size %i, found %i for contig %r" % (int(size), self._contigs[name]['Size'], name)) row = { 'ID': name, 'Size': self._contigs[name]['Size'], 'Ns': self._contigs[name]['Ns'], 'Hits': hits, } handle.write('{ID}\t{Size}\t{Ns}\t{Hits}\n'.format(**row)) CommandNode._setup(self, config, temp)
def _setup(self, config, temp): for key in ("IN_ALIGNMENT", "IN_PARTITION"): source = os.path.abspath(self._kwargs[key]) destination = os.path.join(temp, self._kwargs["TEMP_" + key]) os.symlink(source, destination) CommandNode._setup(self, config, temp)
def _setup(self, config, temp): CommandNode._setup(self, config, temp) # Required to avoid the creation of files outside the temp folder for filename in self._symlinks: source = os.path.abspath(filename) destination = os.path.join(temp, os.path.basename(filename)) os.symlink(source, destination)
def _setup(self, config, temp): CommandNode._setup(self, config, temp) with open(self._tfam) as in_handle: samples = [line.split(None, 1)[0] for line in in_handle] with open(os.path.join(temp, "samples.clust"), "w") as handle: for sample in samples: handle.write("{0} {0} {0}\n".format(sample))
def _setup(self, config, temp): CommandNode._setup(self, config, temp) # Required to avoid the creation of files outside the temp folder for filename in self._symlinks: source = os.path.abspath(filename) destination = os.path.join(temp, os.path.basename(filename)) os.symlink(source, destination)
def _setup(self, config, temp): CommandNode._setup(self, config, temp) with open(self._tfam) as in_handle: samples = [line.split(None, 1)[0] for line in in_handle] with open(os.path.join(temp, "samples.clust"), "w") as handle: for sample in samples: handle.write("{0} {0} {0}\n".format(sample))
def _setup(self, config, temp): os.symlink( os.path.abspath(self._input_alignment), os.path.join(temp, "RAxML_alignment"), ) os.symlink( os.path.abspath(self._input_partitions), os.path.join(temp, "RAxML_partitions"), ) CommandNode._setup(self, config, temp)
def _setup(self, config, temp): if self._k_file is not None: stats = read_summary(self._k_file) n_sites = float(stats[self._k_field]) k = max(1, int(math.ceil(self._snp_distance / (self._genome_size / n_sites)))) self._param_k = k self._command._command.extend(("-k", str(k))) CommandNode._setup(self, config, temp)
def _setup(self, config, temp): if self._k_file is not None: stats = read_summary(self._k_file) n_sites = float(stats[self._k_field]) k = max(1, int(math.ceil(self._snp_distance / (self._genome_size / n_sites)))) self._param_k = k self._command._command.extend(("-k", str(k))) CommandNode._setup(self, config, temp)
def _setup(self, config, temp_root): CommandNode._setup(self, config, temp_root) dst_fname = os.path.join(temp_root, self._bam_input.pipe) if len(self._bam_input.files) > 1: os.mkfifo(dst_fname) else: src_fname, = self._bam_input.files os.symlink(os.path.join(os.getcwd(), src_fname), dst_fname) if self._bam_input.indexed: src_fname = os.path.join(os.getcwd(), swap_ext(src_fname, ".bai")) os.symlink(src_fname, dst_fname + ".bai")
def _setup(self, config, temp): CommandNode._setup(self, config, temp) pipe_fname = os.path.join(temp, self.PIPE_FILE) if len(self._input_bams) > 1: os.mkfifo(pipe_fname) else: source_fname = os.path.abspath(self._input_bams[0]) os.symlink(source_fname, pipe_fname) if self._index_format: os.symlink(swap_ext(source_fname, self._index_format), swap_ext(pipe_fname, self._index_format))
def _setup(self, config, temp): CommandNode._setup(self, config, temp) pipe_fname = os.path.join(temp, self.PIPE_FILE) if len(self._input_bams) > 1: os.mkfifo(pipe_fname) else: source_fname = os.path.abspath(self._input_bams[0]) os.symlink(source_fname, pipe_fname) if self._index_format: os.symlink(swap_ext(source_fname, self._index_format), swap_ext(pipe_fname, self._index_format))
def _setup(self, config, temp): with open(self._bootstraps) as handle: bootstraps = [Newick.from_string(line.strip()) for line in handle] with open(self._treefile) as handle: tree = Newick.from_string(handle.read().strip()) tree = tree.reroot_on_midpoint() tree = tree.add_support(bootstraps, "{Percentage:.0f}") with open(os.path.join(temp, "rerooted.newick"), "w") as handle: handle.write("{}\n".format(tree)) CommandNode._setup(self, config, temp)
def _setup(self, config, temp): with open(self._bootstraps) as handle: bootstraps = [Newick.from_string(line.strip()) for line in handle] with open(self._treefile) as handle: tree = Newick.from_string(handle.read().strip()) tree = tree.reroot_on_midpoint() tree = tree.add_support(bootstraps, "{Percentage:.0f}") with open(os.path.join(temp, "rerooted.newick"), "w") as handle: handle.write("{}\n".format(tree)) CommandNode._setup(self, config, temp)
def _setup(self, config, temp): samples = {} with open(self._samples) as handle: header = handle.readline().strip().split('\t') for line in handle: row = dict(zip(header, line.strip().split('\t'))) samples[row["Name"]] = row with open(os.path.join(temp, "samples.txt"), "w") as handle: handle.write("{}\n".format("\t".join(header))) for name in self._order: row = samples[name] handle.write("{}\n".format("\t".join(row[key] for key in header))) CommandNode._setup(self, config, temp)
def _setup(self, config, temp): samples = {} with open(self._samples) as handle: header = handle.readline().strip().split('\t') for line in handle: row = dict(zip(header, line.strip().split('\t'))) samples[row["Name"]] = row with open(os.path.join(temp, "samples.txt"), "w") as handle: handle.write("{}\n".format("\t".join(header))) for name in self._order: row = samples[name] handle.write("{}\n".format("\t".join(row[key] for key in header))) CommandNode._setup(self, config, temp)
def _setup(self, config, temp): CommandNode._setup(self, config, temp) with open(os.path.join(temp, "parameters.txt"), "w") as handle: handle.write(""" genotypename: {input_prefix}.bed snpname: {input_prefix}.bim indivname: {input_prefix}.fam evecoutname: {output_prefix}.evec evaloutname: {output_prefix}.eval deletsnpoutname: {output_prefix}.deleted_snps altnormstyle: NO numoutevec: 5 familynames: YES numoutlieriter: 1 numchrom: {nchroms} """.format(input_prefix=os.path.abspath(self._input_prefix), output_prefix=os.path.basename(self._output_prefix), nchroms=self._nchroms))
def _setup(self, config, temp): CommandNode._setup(self, config, temp) with open(os.path.join(temp, "parameters.txt"), "w") as handle: handle.write(""" genotypename: {input_prefix}.bed snpname: {input_prefix}.bim indivname: {input_prefix}.fam evecoutname: {output_prefix}.evec evaloutname: {output_prefix}.eval deletsnpoutname: {output_prefix}.deleted_snps altnormstyle: NO numoutevec: 5 familynames: YES numoutlieriter: 1 numchrom: {nchroms} numthreads: 1 """.format(input_prefix=os.path.abspath(self._input_prefix), output_prefix=os.path.basename(self._output_prefix), nchroms=self._nchroms))
def test_command_node__run(): cfg_mock = Mock(temp_root=_DUMMY_TEMP_ROOT) mock = _build_cmd_mock() node_mock = CommandNode(mock) node_mock._create_temp_dir = mock._test_node_._create_temp_dir node_mock._create_temp_dir.return_value = _DUMMY_TEMP node_mock._setup = mock._test_node_._setup node_mock._teardown = mock._test_node_._teardown node_mock._remove_temp_dir = mock._test_node_._remove_temp_dir node_mock.run(cfg_mock) assert mock.mock_calls == [ call._test_node_._create_temp_dir(cfg_mock), call._test_node_._setup(cfg_mock, _DUMMY_TEMP), call.run(_DUMMY_TEMP), call.join(), call._test_node_._teardown(cfg_mock, _DUMMY_TEMP), call._test_node_._remove_temp_dir(_DUMMY_TEMP), ]
def _setup(self, config, temp): if self._multi_file_input: os.mkfifo(os.path.join(os.path.join(temp, "uncompressed_input"))) CommandNode._setup(self, config, temp)
def _setup(self, config, temp): CommandNode._setup(self, config, temp) os.mkfifo(os.path.join(temp, self._basename))
def test_commandnode_setup__files_missing(kwargs): cmd_mock = _build_cmd_mock(**kwargs) node = CommandNode(cmd_mock) with pytest.raises(NodeError): node._setup(None, None)
def test_commandnode_setup__files_exist(kwargs): cmd_mock = _build_cmd_mock(**kwargs) node = CommandNode(cmd_mock) node._setup(None, None)
def _setup(self, config, temp): os.symlink(os.path.abspath(self._input_alignment), os.path.join(temp, "RAxML_alignment")) os.symlink(os.path.abspath(self._input_partitions), os.path.join(temp, "RAxML_partitions")) CommandNode._setup(self, config, temp)
def _do_test_commandnode_setup(kwargs): cmd_mock = _build_cmd_mock(**kwargs) node = CommandNode(cmd_mock) node._setup(None, None)
def _setup(self, config, temp): CommandNode._setup(self, config, temp) for fname in ("Stats_out_MCMC_correct_prob.csv",): relpath = os.path.join(self._directory, fname) abspath = os.path.abspath(relpath) os.symlink(abspath, os.path.join(temp, fname))