def test_requirement__same_obj_if_name_differ():
obj1 = versions.Requirement("echo", "", versions.GE(1))
assert obj1.name == "echo"
obj2 = versions.Requirement("echo", "", versions.GE(1), name="foo")
assert obj2.name == "foo"
assert obj1 is obj2
obj3 = versions.Requirement("echo", "", versions.GE(1), name="bar")
assert obj3.name == "bar"
assert obj2 is obj3
obj4 = versions.Requirement("echo", "", versions.GE(1))
assert obj3.name == "bar"
assert obj3 is obj4
def test_requirement__same_obj_if_name_differ():
obj1 = versions.Requirement("echo", "", versions.GE(1))
assert_equal(obj1.name, "echo")
obj2 = versions.Requirement("echo", "", versions.GE(1), name="foo")
assert_equal(obj2.name, "foo")
assert_is(obj1, obj2)
obj3 = versions.Requirement("echo", "", versions.GE(1), name="bar")
assert_equal(obj3.name, "bar")
assert_is(obj2, obj3)
obj4 = versions.Requirement("echo", "", versions.GE(1))
assert_equal(obj3.name, "bar")
assert_is(obj3, obj4)
def picard_command(config, command):
"""Returns basic AtomicJavaCmdBuilder for Picard tools commands."""
jar_path = os.path.join(config.jar_root, _PICARD_JAR)
if jar_path not in _PICARD_VERSION_CACHE:
params = AtomicJavaCmdBuilder(jar_path,
temp_root=config.temp_root,
jre_options=config.jre_options)
# Arbitrary command, since just '--version' does not work
params.set_option("MarkDuplicates")
params.set_option("--version")
requirement = versions.Requirement(
call=params.finalized_call,
name="Picard tools",
search=r"\b(\d+)\.(\d+)\.\d+",
checks=versions.GE(1, 137),
)
_PICARD_VERSION_CACHE[jar_path] = requirement
version = _PICARD_VERSION_CACHE[jar_path]
params = AtomicJavaCmdBuilder(
jar_path,
temp_root=config.temp_root,
jre_options=config.jre_options,
CHECK_JAR=version,
set_cwd=True,
)
params.set_option(command)
return params
def requirement(module, checks=versions.Any(), cache={}):
key = (module, checks)
result = cache.get(key)
if result is None:
filename = rscript("common", "requires.r")
result = versions.Requirement(
call=("Rscript", filename, module),
search="d0fd3ea6: (\d+)\.(\d+)(?:\.(\d+))?",
checks=checks,
name="R module: {}".format(module))
cache[(module, checks)] = result
return result
def _get_gatk_version_check(config):
"""Returns a version-check object for the "GenomeAnalysisTK.jar" located at
config.jar_root; for now, this check only serves to verify that the JAR can
be executed, which may not be the case if the JRE is outdated.
"""
jar_file = os.path.join(config.jar_root, "GenomeAnalysisTK.jar")
if jar_file not in _GATK_VERSION:
params = AtomicJavaCmdBuilder(jar_file,
temp_root=config.temp_root,
jre_options=config.jre_options)
params.add_value("--version")
# Any version is fine; for now just catch old JREs
requirement = versions.Requirement(call=params.finalized_call,
name="GenomeAnalysisTK",
search=r"^(\d+)\.(\d+)",
checks=versions.Any())
_GATK_VERSION[jar_file] = requirement
return _GATK_VERSION[jar_file]
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#
from paleomix.node import CommandNode, NodeError
from paleomix.atomiccmd.builder import (
AtomicCmdBuilder,
apply_options,
)
from paleomix.common.fileutils import reroot_path
from paleomix.common.formats.msa import MSA, MSAError
import paleomix.common.versions as versions
MAFFT_VERSION = versions.Requirement(call=("mafft", "--version"),
search=r"v(\d+)\.(\d+)",
checks=versions.GE(7, 0))
# Presets mainly taken from
# http://mafft.cbrc.jp/alignment/software/algorithms/algorithms.html
_PRESETS = {
"mafft": ["mafft"],
"auto": ["mafft", "--auto"],
"fft-ns-1": ["mafft-fftns", "--retree", 1],
"fft-ns-2": ["mafft-fftns"],
"fft-ns-i": ["mafft-fftnsi"],
"nw-ns-i": ["mafft-nwnsi"],
"l-ins-i": ["mafft-linsi"],
"e-ins-i": ["mafft-einsi"],
"g-ins-i": ["mafft-ginsi"],
}
# SOFTWARE.
#
import os
from paleomix.node import CommandNode
from paleomix.atomiccmd.builder import (
AtomicCmdBuilder,
apply_options,
)
import paleomix.common.fileutils as fileutils
import paleomix.common.versions as versions
_VERSION_CHECK = versions.Requirement(
call=("AdapterRemoval", "--version"),
search=r"ver. (\d+)\.(\d+)\.(\d+)",
checks=versions.GE(2, 1, 5),
)
class SE_AdapterRemovalNode(CommandNode):
def __init__(self,
input_file,
output_prefix,
threads=1,
options={},
dependencies=()):
# See below for parameters in common between SE/PE
cmd = _get_common_parameters(threads=threads, options=options)
# Prefix for output files, ensure that all end up in temp folder
describe_files
from paleomix.common.utilities import \
safe_coerce_to_tuple
from paleomix.node import \
NodeError, \
CommandNode
from paleomix.nodes.picard import \
MultiBAMInputNode
from paleomix.atomiccmd.builder import \
AtomicCmdBuilder, \
use_customizable_cli_parameters, \
create_customizable_cli_parameters
MAPDAMAGE_VERSION = versions.Requirement(call=("mapDamage", "--version"),
search=r"(\d+)\.(\d+).(\d+)",
checks=versions.GE(2, 0, 1))
RSCRIPT_VERSION = versions.Requirement(call=("Rscript", "--version"),
search=r"(\d+)\.(\d+).(\d+)",
checks=versions.GE(2, 15, 1),
priority=10)
class MapDamagePlotNode(MultiBAMInputNode):
@create_customizable_cli_parameters
def customize(self,
config,
reference,
input_files,
output_directory,
AtomicCmdBuilder, \
use_customizable_cli_parameters, \
create_customizable_cli_parameters
from paleomix.atomiccmd.sets import \
ParallelCmds
from paleomix.nodes.bwa import \
_get_node_description, \
_process_output, \
_get_max_threads
from paleomix.common.utilities import \
safe_coerce_to_tuple
import paleomix.common.versions as versions
BOWTIE2_VERSION = versions.Requirement(call=("bowtie2", "--version"),
search=r"version (\d+)\.(\d+)\.(\d+)",
checks=versions.GE(2, 1, 0))
class Bowtie2IndexNode(CommandNode):
def __init__(self, input_file, prefix=None, dependencies=()):
prefix = prefix if prefix else input_file
builder = _bowtie2_template(("bowtie2-build"),
prefix,
iotype="OUT",
IN_FILE=input_file,
TEMP_OUT_PREFIX=os.path.basename(prefix),
CHECK_VERSION=BOWTIE2_VERSION)
builder.add_value("%(IN_FILE)s")
# Destination prefix, in temp folder
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
import collections
import paleomix.yaml
import paleomix.common.versions as versions
# Format number for database file; is incremented when the format is changed.
# The 'revision' field specifies updates to the table that do not change the
# format of the database (see below).
_SUPPORTED_DB_FORMAT = 1
RSCRIPT_VERSION = versions.Requirement(
call=("Rscript", "--version"),
search=r"version (\d+)\.(\d+)\.(\d+)",
checks=versions.GE(3, 0, 0),
priority=10,
)
class DBFileError(RuntimeError):
pass
def get_sample_names(handle):
samples = []
for readgroup in handle.header.get("RG", ()):
if "SM" in readgroup:
samples.append(readgroup["SM"])
return frozenset(samples)
# SOFTWARE.
#
import os
import re
import random
import paleomix.common.fileutils as fileutils
import paleomix.common.versions as versions
from paleomix.node import CommandNode
from paleomix.atomiccmd.builder import AtomicCmdBuilder
RAXML_VERSION = versions.Requirement(
call=("raxmlHPC", "-version"),
search=r"version (\d+)\.(\d+)\.(\d+)",
checks=versions.GE(7, 3, 2),
)
RAXML_PTHREADS_VERSION = versions.Requirement(
call=("raxmlHPC-PTHREADS", "-version"),
search=r"version (\d+)\.(\d+)\.(\d+)",
checks=versions.GE(7, 3, 2),
)
class RAxMLRapidBSNode(CommandNode):
def __init__(
self,
input_alignment,
output_template,
input_partition=None,
AtomicCmd
from paleomix.node import \
CommandNode, \
Node, \
NodeError
from paleomix.common.bedtools import \
read_bed_file
from paleomix.common.fileutils import \
move_file, \
reroot_path
BEDTOOLS_VERSION \
= versions.Requirement(call=("bedtools", "--version"),
search=r"bedtools v?(\d+)\.(\d+)\.(\d+)",
checks=versions.GE(2, 15, 0))
class SlopBedNode(CommandNode):
def __init__(self,
infile,
outfile,
genome,
from_start=0,
from_end=0,
strand_relative=False,
dependencies=()):
if type(from_start) != type(from_end):
raise ValueError("Parameters 'from_start' and 'from_end' should "
"be of same type!")
def test_requirement__new_obj_if_call_differ():
obj1 = versions.Requirement("echo", "", versions.LT(1))
obj2 = versions.Requirement("true", "", versions.LT(1))
assert_is_not(obj1, obj2)
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#
import os
from paleomix.node import CommandNode
from paleomix.atomiccmd.builder import AtomicCmdBuilder
from paleomix.atomiccmd.command import AtomicCmd
from paleomix.common.fileutils import reroot_path
import paleomix.common.versions as versions
_VERSION_REGEX = r"Version: (\d+)\.(\d+)(?:\.(\d+))?"
SAMTOOLS_VERSION = versions.Requirement(call=("samtools", ),
search=_VERSION_REGEX,
checks=versions.GE(1, 3, 0))
BCFTOOLS_VERSION = versions.Requirement(call=("bcftools", ),
search=_VERSION_REGEX,
checks=versions.GE(1, 3, 0))
TABIX_VERSION = versions.Requirement(call=("tabix", ),
search=_VERSION_REGEX,
checks=versions.GE(1, 3, 0))
class TabixIndexNode(CommandNode):
"""Tabix indexes a BGZip compressed VCF or pileup file.
The class currently supports the following presets:
import paleomix.common.versions as versions
import paleomix.tools.factory as factory
from paleomix.atomiccmd.builder import (
AtomicCmdBuilder,
apply_options,
)
from paleomix.atomiccmd.command import AtomicCmd
from paleomix.atomiccmd.sets import ParallelCmds
from paleomix.common.fileutils import describe_paired_files
from paleomix.node import CommandNode, NodeError
from paleomix.nodes.samtools import SAMTOOLS_VERSION
BWA_VERSION = versions.Requirement(call=("bwa", ),
search=r"Version: (\d+)\.(\d+)\.(\d+)",
checks=versions.GE(0, 7, 9))
class BWAIndexNode(CommandNode):
def __init__(self, input_file, prefix=None, dependencies=()):
prefix = prefix if prefix else input_file
builder = _new_bwa_command(
("bwa", "index", "%(IN_FILE)s", "-p", "%(TEMP_OUT_PREFIX)s"),
prefix,
iotype="OUT",
IN_FILE=input_file,
TEMP_OUT_PREFIX=os.path.basename(prefix),
)
description = "<BWA Index '%s' -> '%s.*'>" % (input_file, prefix)
def test_requirement_highest_priority_retained():
obj1 = versions.Requirement("echo", "", versions.LT(1), priority=5)
assert obj1.priority == 5
obj2 = versions.Requirement("echo", "", versions.LT(1), priority=0)
assert obj1 is obj2
assert obj2.priority == 5
def test_requirement_highest_priority_retained():
obj1 = versions.Requirement("echo", "", versions.LT(1), priority=5)
assert_equal(obj1.priority, 5)
obj2 = versions.Requirement("echo", "", versions.LT(1), priority=0)
assert_is(obj1, obj2)
assert_equal(obj2.priority, 5)
def test_requirement__new_obj_if_checks_differ():
obj1 = versions.Requirement("echo", "", versions.GE(1))
obj2 = versions.Requirement("echo", "", versions.LT(1))
assert obj1 is not obj2
def test_requirement__new_obj_if_search_differ():
obj1 = versions.Requirement("echo", r"(\d+)", versions.LT(1))
obj2 = versions.Requirement("echo", "", versions.LT(1))
assert obj1 is not obj2
def test_requirement__obj_is_cached_for_same_values():
obj1 = versions.Requirement("echo", "", versions.LT(1))
obj2 = versions.Requirement("echo", "", versions.LT(1))
assert obj1 is obj2
import paleomix.common.fileutils as fileutils
import paleomix.common.versions as versions
from paleomix.node import CommandNode
from paleomix.atomiccmd.builder import \
AtomicCmdBuilder, \
AtomicMPICmdBuilder, \
use_customizable_cli_parameters, \
create_customizable_cli_parameters
from paleomix.nodegraph import \
FileStatusCache
EXAML_VERSION = versions.Requirement(call=("examl", "-version"),
search=r"version (\d+)\.(\d+)\.(\d+)",
checks=versions.GE(3, 0, 0))
PARSER_VERSION = versions.Requirement(call=("parse-examl", "-h"),
search=r"version (\d+)\.(\d+)\.(\d+)",
checks=versions.GE(3, 0, 0))
class ExaMLParserNode(CommandNode):
@create_customizable_cli_parameters
def customize(cls,
input_alignment,
input_partition,
output_file,
dependencies=()):
"""
use_customizable_cli_parameters, \
create_customizable_cli_parameters
from paleomix.atomiccmd.sets import ParallelCmds
from paleomix.nodes.samtools import SAMTOOLS_VERSION
from paleomix.common.fileutils import \
describe_paired_files, \
missing_files
import paleomix.common.versions as versions
import paleomix.tools.factory as factory
BWA_VERSION = versions.Requirement(call=("bwa",),
search=r"Version: (\d+)\.(\d+)\.(\d+)",
checks=versions.Or(versions.EQ(0, 5, 9),
versions.EQ(0, 5, 10),
versions.EQ(0, 6, 2),
versions.GE(0, 7, 9)))
BWA_VERSION_07x = versions.Requirement(call=("bwa",),
search=r"Version: (\d+)\.(\d+)\.(\d+)",
checks=versions.GE(0, 7, 9))
class BWAIndexNode(CommandNode):
def __init__(self, input_file, prefix=None, dependencies=()):
prefix = prefix if prefix else input_file
builder = _get_bwa_template(("bwa", "index"), prefix, iotype="OUT",
IN_FILE=input_file,
TEMP_OUT_PREFIX=os.path.basename(prefix),
CHECK_BWA=BWA_VERSION)
import paleomix.common.rtools as rtools
import paleomix.common.versions as versions
import paleomix.tools.factory as factory
from paleomix.atomiccmd.builder import AtomicCmdBuilder
from paleomix.atomiccmd.command import AtomicCmd
from paleomix.atomiccmd.sets import SequentialCmds
from paleomix.node import CommandNode, Node, NodeError
from paleomix.tools.zonkey.common import \
RSCRIPT_VERSION, \
contig_name_to_plink_name, \
read_summary
ADMIXTURE_VERSION = versions.Requirement(call=("admixture", "--version"),
search="(\d+)\.(\d+)",
checks=versions.GE(1, 3))
PLINK_VERSION = versions.Requirement(call=("plink", "--noweb", "--help",
"--out", "/tmp/plink"),
search="v(\d+)\.(\d+)",
checks=versions.GE(1, 7))
SMARTPCA_VERSION = versions.Requirement(call=("smartpca", ),
search="version: (\d+)",
checks=versions.GE(13050))
TREEMIX_VERSION = versions.Requirement(call=("treemix", ),
search="TreeMix v. (\d+)\.(\d+)",
checks=versions.GE(1, 12))
import os
from paleomix.node import CommandNode
from paleomix.atomiccmd.command import AtomicCmd
from paleomix.atomiccmd.sets import SequentialCmds
from paleomix.common.fileutils import reroot_path, swap_ext
import paleomix.common.versions as versions
_VERSION_REGEX = r"Version: (\d+)\.(\d+)(?:\.(\d+))?"
# v0.2.0 was the pre-release version of v1.0, and lacks required features
_COMMON_CHECK = versions.Or(versions.EQ(0, 1, 19), versions.GE(1, 0, 0))
SAMTOOLS_VERSION = versions.Requirement(call=("samtools", ),
search=_VERSION_REGEX,
checks=_COMMON_CHECK)
SAMTOOLS_VERSION_1x = versions.Requirement(call=("samtools", ),
search=_VERSION_REGEX,
checks=versions.GE(1, 0, 0))
SAMTOOLS_VERSION_0119 = versions.Requirement(call=("samtools", ),
search=_VERSION_REGEX,
checks=versions.EQ(0, 1, 19))
BCFTOOLS_VERSION_0119 \
= versions.Requirement(call=("bcftools",),
search=_VERSION_REGEX,
checks=versions.EQ(0, 1, 19))
