def test_data_resolver_create_from_contents():
with tempdir() as d:
resolver = DataResolver(
{"foo": {
"path": "dir1/dir2/foo.txt",
"contents": "foo"
}}, UserConfiguration(None, cache_dir=d))
parent = d / "dir1" / "dir2"
foo = resolver.resolve("foo")
assert foo.path == parent / "foo.txt"
with open(foo.path, "rt") as inp:
assert inp.read() == "foo"
with tempdir() as d:
resolver = DataResolver(
{"foo": {
"name": "foo.txt",
"contents": "foo"
}}, UserConfiguration(None, cache_dir=d))
foo = resolver.resolve("foo")
assert foo.path == d / "foo.txt"
with open(foo.path, "rt") as inp:
assert inp.read() == "foo"
with tempdir() as d:
resolver = DataResolver({"foo": {
"contents": "foo"
}}, UserConfiguration(None, cache_dir=d))
foo = resolver.resolve("foo")
assert foo.path.parent == d
assert foo.path.exists()
with open(foo.path, "rt") as inp:
assert inp.read() == "foo"
def test_data_resolver_create_from_url():
with tempdir() as d:
resolver = DataResolver(
{"foo": {
"url": GOOD_URL,
"path": "dir1/dir2/sample.vcf"
}}, UserConfiguration(None, cache_dir=d))
foo = resolver.resolve("foo")
assert foo.path == d / "dir1" / "dir2" / "sample.vcf"
with open(foo.path, "rt") as inp:
assert inp.read() == "foo"
with tempdir() as d:
resolver = DataResolver(
{"foo": {
"url": GOOD_URL,
"name": "sample.vcf"
}}, UserConfiguration(None, cache_dir=d))
foo = resolver.resolve("foo")
assert foo.path == d / "sample.vcf"
with open(foo.path, "rt") as inp:
assert inp.read() == "foo"
with tempdir() as d:
resolver = DataResolver({"foo": {
"url": GOOD_URL
}}, UserConfiguration(None, cache_dir=d))
foo = resolver.resolve("foo")
assert foo.path == d / "test_file"
with open(foo.path, "rt") as inp:
assert inp.read() == "foo"
def test_user_config_no_defaults():
with tempdir(True):
config = UserConfiguration()
assert config.remove_cache_dir is True
assert config.cache_dir.exists()
config.cleanup()
assert not config.cache_dir.exists()
def test_url_localizer():
good_url = GOOD_URL
bad_url = "foo"
with tempdir() as d:
foo = d / "foo"
UrlLocalizer(good_url,
UserConfiguration(None, cache_dir=d),
digests={
"md5": "acbd18db4cc2f85cedef654fccc4a4d8"
}).localize(foo)
with open(foo, "rt") as inp:
assert inp.read() == "foo"
# test that the file is overwritten
with open(foo, "wt") as out:
out.write("bork")
UrlLocalizer(good_url, UserConfiguration(None,
cache_dir=d)).localize(foo)
with open(foo, "rt") as inp:
assert inp.read() == "foo"
with pytest.raises(RuntimeError):
with tempdir() as d:
foo = d / "foo"
UrlLocalizer(good_url,
UserConfiguration(None, cache_dir=d),
digests={
"md5": "XXX"
}).localize(foo)
with pytest.raises(RuntimeError):
UrlLocalizer(bad_url, UserConfiguration(None,
cache_dir=d)).localize(foo)
def test_user_config_from_file():
with tempdir() as d, setenv({
"HTTPS_PROXY": "http://foo.com/https",
"FOO_HEADER": "bar"
}):
cache_dir = d / "cache"
execution_dir = d / "execution"
config_dict = {
"cache_dir":
str(cache_dir),
"execution_dir":
str(execution_dir),
"proxies": {
"http": {
"value": "http://foo.com/http"
},
"https": {
"env": "HTTPS_PROXY"
}
},
"http_headers": [{
"pattern": "http://foo.com/.*",
"name": "foo",
"env": "FOO_HEADER"
}],
"executors": {
"foo": {
"bar": 1
}
}
}
config_file = d / "config.json"
with open(config_file, "wt") as out:
json.dump(config_dict, out)
config = UserConfiguration(config_file)
assert config.cache_dir == cache_dir
assert config.default_execution_dir == execution_dir
assert config.proxies == {
"http": "http://foo.com/http",
"https": "http://foo.com/https"
}
assert config.default_http_headers == [{
"pattern":
re.compile("http://foo.com/.*"),
"name":
"foo",
"env":
"FOO_HEADER"
}]
assert config.get_executor_defaults("foo") == {"bar": 1}
def test_data_resolver_local_path():
with tempdir() as d:
path = d / "foo.txt"
with open(path, "wt") as out:
out.write("foo")
resolver = DataResolver({"foo": {
"path": "foo.txt"
}}, UserConfiguration(None, cache_dir=d))
assert resolver.resolve("foo").path == path
with setenv({"MYPATH": str(d)}):
resolver = DataResolver({"foo": {
"path": "${MYPATH}/foo.txt"
}}, UserConfiguration(None, cache_dir=d))
assert resolver.resolve("foo").path == path
def create_executor(executor_name: str, import_dirs: Sequence[Path],
user_config: UserConfiguration):
executor_class = EXECUTORS.get(executor_name)
if not executor_class:
raise RuntimeError(f"{executor_name} executor plugin is not installed")
return executor_class(import_dirs,
**user_config.get_executor_defaults(executor_name))
def test_data_file_class():
dd = DataResolver(data_descriptors={"foo": {
"class": "bar",
"value": 1
}},
user_config=UserConfiguration())
assert dd.resolve("foo") == 1
def test_url_localizer_set_proxies():
localizer = UrlLocalizer(
"http://foo.com",
UserConfiguration(proxies={"https": "https://foo.com/proxy"}))
proxies = localizer.proxies
assert len(proxies) == 1
assert "https" in proxies
assert proxies["https"] == "https://foo.com/proxy"
def test_data_resolver_env():
with tempdir() as d:
path = d / "foo.txt"
with open(path, "wt") as out:
out.write("foo")
with setenv({"FOO": str(path)}):
resolver = DataResolver({"foo": {
"env": "FOO"
}}, UserConfiguration(None, cache_dir=d))
assert resolver.resolve("foo").path == path
bar = d / "bar.txt"
resolver = DataResolver({"foo": {
"env": "FOO",
"path": bar
}}, UserConfiguration(None, cache_dir=d))
assert resolver.resolve("foo").path == bar
def test_http_header_set_in_workflow_data():
"""
Test that workflow data file can define the HTTP Headers. This is
important because the URLs referenced can be from different hosts and
require different headers, so setting them at this level allows that
fine-grained control.
"""
with tempdir() as d:
config = UserConfiguration(cache_dir=d)
assert not config.default_http_headers
resolver = DataResolver(
{
"foo": {
"url": GOOD_URL,
"path": "sample.vcf",
"http_headers": {
"Auth-Header-Token": "TOKEN"
}
}
}, config)
foo = resolver.resolve("foo")
assert foo.path == d / "sample.vcf"
with open(foo.path, "rt") as inp:
assert inp.read() == "foo"
with setenv({"TOKEN": "this_is_the_token"}), tempdir() as d:
config = UserConfiguration(cache_dir=d)
assert not config.default_http_headers
resolver = DataResolver(
{
"foo": {
"url": GOOD_URL,
"path": "sample.vcf",
"http_headers": {
"Auth-Header-Token": "TOKEN"
}
}
}, config)
foo = resolver.resolve("foo")
assert foo.path == d / "sample.vcf"
assert isinstance(foo.localizer, UrlLocalizer)
assert cast(UrlLocalizer, foo.localizer).http_headers == {
"Auth-Header-Token": "this_is_the_token"
}
with open(foo.path, "rt") as inp:
assert inp.read() == "foo"
def test_data_file_json_contents():
with tempdir() as d:
foo = d / "foo.json"
df = create_data_file(user_config=UserConfiguration(),
path=foo,
contents={
"a": 1,
"b": "foo"
})
with open(df.path, "rt") as inp:
assert json.load(inp) == {"a": 1, "b": "foo"}
def test_data_manager():
dm = DataManager(data_resolver=DataResolver(
{
"foo": {
"class": "x",
"value": 1
},
"bar": {
"class": "x",
"value": 2
}
}, UserConfiguration()),
datadirs=None)
assert [1, 2] == dm.get_list("foo", "bar")
assert {"foo": 1, "bork": 2} == dm.get_dict("foo", bork="bar")
def test_data_resolver():
with tempdir() as d:
test_data = {"foo": {"name": "foo.txt"}, "bar": 1}
foo_txt = d / "data" / "foo.txt"
foo_txt.parent.mkdir()
with open(foo_txt, "wt") as out:
out.write("bar")
mod = Mock()
mod.__name__ = ""
fun = Mock()
fun.__name__ = "test_foo"
dd = DataDirs(d, mod, fun)
resolver = DataResolver(test_data, UserConfiguration(None,
cache_dir=d))
with pytest.raises(FileNotFoundError):
resolver.resolve("bork", dd)
assert resolver.resolve("foo", dd).path == foo_txt
assert resolver.resolve("bar", dd) == 1
def test_data_resolver_create_from_datadir():
with tempdir() as d, tempdir() as d1:
mod = Mock()
mod.__name__ = "foo.bar"
cls = Mock()
cls.__name__ = "baz"
fun = Mock()
fun.__name__ = "blorf"
mod_cls_fun = d / "foo" / "bar" / "baz" / "blorf"
mod_cls_fun.mkdir(parents=True)
data_mod_cls_fun = d / "data" / "foo" / "bar" / "baz" / "blorf"
data_mod_cls_fun.mkdir(parents=True)
dd = DataDirs(d / "foo", mod, fun, cls)
resolver = DataResolver(
{
"boink": {
"name": "boink.txt",
},
"bobble": {
"name": "bobble.txt"
},
"burp": {
"name": "burp.txt",
"path": "burp.txt"
}
}, UserConfiguration(None, cache_dir=d1))
boink = d / "foo" / "bar" / "boink.txt"
with open(boink, "wt") as out:
out.write("boink")
assert boink == resolver.resolve("boink", dd).path
with pytest.raises(FileNotFoundError):
resolver.resolve("bobble", dd)
burp = d / "foo" / "bar" / "burp.txt"
with open(burp, "wt") as out:
out.write("burp")
burp_resolved = resolver.resolve("burp", dd).path
assert burp_resolved == d1 / "burp.txt"
assert burp_resolved.is_symlink()
with pytest.raises(FileNotFoundError):
resolver.resolve("bobble")
def test_data_file_dict_type():
with tempdir() as d:
foo = d / "foo.txt.gz"
with gzip.open(foo, "wt") as out:
out.write("foo\nbar")
df = create_data_file(user_config=UserConfiguration(),
path=foo,
type={
"name": "default",
"allowed_diff_lines": 1
})
bar = d / "bar.txt.gz"
with gzip.open(bar, "wt") as out:
out.write("foo\nbaz")
df.assert_contents_equal(bar)
df.assert_contents_equal(str(bar))
df.assert_contents_equal(DefaultDataFile(bar))
def test_url_localizer_add_headers():
with setenv({"FOO": "bar"}):
localizer = UrlLocalizer(
"http://foo.com/bork",
UserConfiguration(
http_headers=[{
"name": "beep",
"pattern": re.compile(r"http://foo.com/.*"),
"env": "FOO",
"value": "baz"
}, {
"name": "boop",
"pattern": re.compile(r"http://foo.*/bork"),
"env": "BAR",
"value": "blorf"
}]), {"boop": {
"value": "blammo"
}})
headers = localizer.http_headers
assert len(headers) == 2
assert set(headers.keys()) == {"beep", "boop"}
assert headers["beep"] == "bar"
assert headers["boop"] == "blammo"