def test_check_exists_follows_redirects() -> None:
fetcher = DefaultFetcher({}, Session())
# This tests that the redirect from http to https doesn't cause a
# false positive, this URL doesn't exist and it should return
# False because if it follows the redirect it'll get a 404. Tests
# for previous bug where allow_redirects wasn't set and as a
# result it would return True even though the result was 3xx.
assert not fetcher.check_exists("http://commonwl.org/does/not/exist")
def __init__(
self,
fetcher=None, # type: Optional[Fetcher]
namespaces=None, # type: Optional[Dict[str, str]]
schemas=None, # type: Optional[Dict[str, str]]
fileuri=None, # type: Optional[str]
copyfrom=None, # type: Optional[LoadingOptions]
original_doc=None, # type: Optional[Any]
): # type: (...) -> None
self.idx = {} # type: Dict[str, Dict[str, Any]]
self.fileuri = fileuri # type: Optional[str]
self.namespaces = namespaces
self.schemas = schemas
self.original_doc = original_doc
if copyfrom is not None:
self.idx = copyfrom.idx
if fetcher is None:
fetcher = copyfrom.fetcher
if fileuri is None:
self.fileuri = copyfrom.fileuri
if namespaces is None:
self.namespaces = copyfrom.namespaces
if schemas is None:
self.schemas = copyfrom.schemas
if fetcher is None:
import requests
from cachecontrol.caches import FileCache
from cachecontrol.wrapper import CacheControl
if "HOME" in os.environ:
session = CacheControl(
requests.Session(),
cache=FileCache(
os.path.join(os.environ["HOME"], ".cache", "salad")),
)
elif "TMPDIR" in os.environ:
session = CacheControl(
requests.Session(),
cache=FileCache(
os.path.join(os.environ["TMPDIR"], ".cache", "salad")),
)
else:
session = CacheControl(requests.Session(),
cache=FileCache("/tmp", ".cache",
"salad"))
self.fetcher = DefaultFetcher({}, session) # type: Fetcher
else:
self.fetcher = fetcher
self.vocab = _vocab
self.rvocab = _rvocab
if namespaces is not None:
self.vocab = self.vocab.copy()
self.rvocab = self.rvocab.copy()
for k, v in namespaces.items():
self.vocab[k] = v
self.rvocab[v] = k
def __init__(
self,
fetcher: Optional[Fetcher] = None,
namespaces: Optional[Dict[str, str]] = None,
schemas: Optional[Dict[str, str]] = None,
fileuri: Optional[str] = None,
copyfrom: Optional["LoadingOptions"] = None,
original_doc: Optional[Any] = None,
) -> None:
"""Create a LoadingOptions object."""
self.idx: Dict[str, Dict[str, Any]] = {}
self.fileuri: Optional[str] = fileuri
self.namespaces = namespaces
self.schemas = schemas
self.original_doc = original_doc
if copyfrom is not None:
self.idx = copyfrom.idx
if fetcher is None:
fetcher = copyfrom.fetcher
if fileuri is None:
self.fileuri = copyfrom.fileuri
if namespaces is None:
self.namespaces = copyfrom.namespaces
if schemas is None:
self.schemas = copyfrom.schemas
if fetcher is None:
import requests
from cachecontrol.caches import FileCache
from cachecontrol.wrapper import CacheControl
root = pathlib.Path(os.environ.get("HOME", tempfile.gettempdir()))
session = CacheControl(
requests.Session(),
cache=FileCache(root / ".cache" / "salad"),
)
self.fetcher: Fetcher = DefaultFetcher({}, session)
else:
self.fetcher = fetcher
self.vocab = _vocab
self.rvocab = _rvocab
if namespaces is not None:
self.vocab = self.vocab.copy()
self.rvocab = self.rvocab.copy()
for k, v in namespaces.items():
self.vocab[k] = v
self.rvocab[v] = k
def __init__(
self,
fetcher=None, # type: Optional[Fetcher]
namespaces=None, # type: Optional[Dict[str, str]]
schemas=None, # type: Optional[Dict[str, str]]
fileuri=None, # type: Optional[str]
copyfrom=None, # type: Optional[LoadingOptions]
original_doc=None, # type: Optional[Any]
): # type: (...) -> None
self.idx = {} # type: Dict[str, Dict[str, Any]]
self.fileuri = fileuri # type: Optional[str]
self.namespaces = namespaces
self.schemas = schemas
self.original_doc = original_doc
if copyfrom is not None:
self.idx = copyfrom.idx
if fetcher is None:
fetcher = copyfrom.fetcher
if fileuri is None:
self.fileuri = copyfrom.fileuri
if namespaces is None:
self.namespaces = copyfrom.namespaces
if schemas is None:
self.schemas = copyfrom.schemas
if fetcher is None:
import requests
from cachecontrol.caches import FileCache
from cachecontrol.wrapper import CacheControl
root = pathlib.Path(os.environ.get("HOME", tempfile.gettempdir()))
session = CacheControl(
requests.Session(),
cache=FileCache(root / ".cache" / "salad"),
)
self.fetcher: Fetcher = DefaultFetcher({}, session)
else:
self.fetcher = fetcher
self.vocab = _vocab
self.rvocab = _rvocab
if namespaces is not None:
self.vocab = self.vocab.copy()
self.rvocab = self.rvocab.copy()
for k, v in namespaces.items():
self.vocab[k] = v
self.rvocab[v] = k
def test_DefaultFetcher_urljoin_win32(tmp_dir_fixture: str) -> None:
# Ensure HOME is set.
os.environ["HOME"] = tmp_dir_fixture
fetcher = DefaultFetcher({}, None)
# Relative path, same folder
url = fetcher.urljoin("file:///C:/Users/fred/foo.cwl", "soup.cwl")
assert url == "file:///C:/Users/fred/soup.cwl"
# Relative path, sub folder
url = fetcher.urljoin("file:///C:/Users/fred/foo.cwl", "foo/soup.cwl")
assert url == "file:///C:/Users/fred/foo/soup.cwl"
# relative climb-up path
url = fetcher.urljoin("file:///C:/Users/fred/foo.cwl", "../alice/soup.cwl")
assert url == "file:///C:/Users/alice/soup.cwl"
# Path with drive: should not be treated as relative to directory
# Note: \ would already have been converted to / by resolve_ref()
url = fetcher.urljoin("file:///C:/Users/fred/foo.cwl", "c:/bar/soup.cwl")
assert url == "file:///c:/bar/soup.cwl"
# /C:/ (regular URI absolute path)
url = fetcher.urljoin("file:///C:/Users/fred/foo.cwl", "/c:/bar/soup.cwl")
assert url == "file:///c:/bar/soup.cwl"
# Relative, change drive
url = fetcher.urljoin("file:///C:/Users/fred/foo.cwl", "D:/baz/soup.cwl")
assert url == "file:///d:/baz/soup.cwl"
# Relative from root of base's D: drive
url = fetcher.urljoin("file:///d:/baz/soup.cwl", "/foo/soup.cwl")
assert url == "file:///d:/foo/soup.cwl"
# resolving absolute non-drive URIs still works
url = fetcher.urljoin("file:///C:/Users/fred/foo.cwl",
"http://example.com/bar/soup.cwl")
assert url == "http://example.com/bar/soup.cwl"
# and of course relative paths from http://
url = fetcher.urljoin("http://example.com/fred/foo.cwl", "soup.cwl")
assert url == "http://example.com/fred/soup.cwl"
# Stay on http:// and same host
url = fetcher.urljoin("http://example.com/fred/foo.cwl", "/bar/soup.cwl")
assert url == "http://example.com/bar/soup.cwl"
# Security concern - can't resolve file: from http:
with pytest.raises(ValidationException):
url = fetcher.urljoin("http://example.com/fred/foo.cwl",
"file:///c:/bar/soup.cwl")
# Drive-relative -- should NOT return "absolute" URI c:/bar/soup.cwl"
# as that is a potential remote exploit
with pytest.raises(ValidationException):
url = fetcher.urljoin("http://example.com/fred/foo.cwl",
"c:/bar/soup.cwl")
def test_DefaultFetcher_urljoin_linux(tmp_dir_fixture: str) -> None:
# Ensure HOME is set.
os.environ["HOME"] = tmp_dir_fixture
actual_platform = sys.platform
try:
# Pretend it's Linux (e.g. not win32)
sys.platform = "linux2"
fetcher = DefaultFetcher({}, None)
url = fetcher.urljoin("file:///home/fred/foo.cwl", "soup.cwl")
assert url == "file:///home/fred/soup.cwl"
url = fetcher.urljoin("file:///home/fred/foo.cwl", "../alice/soup.cwl")
assert url == "file:///home/alice/soup.cwl"
# relative from root
url = fetcher.urljoin("file:///home/fred/foo.cwl", "/baz/soup.cwl")
assert url == "file:///baz/soup.cwl"
url = fetcher.urljoin("file:///home/fred/foo.cwl",
"http://example.com/bar/soup.cwl")
assert url == "http://example.com/bar/soup.cwl"
url = fetcher.urljoin("http://example.com/fred/foo.cwl", "soup.cwl")
assert url == "http://example.com/fred/soup.cwl"
# Root-relative -- here relative to http host, not file:///
url = fetcher.urljoin("http://example.com/fred/foo.cwl",
"/bar/soup.cwl")
assert url == "http://example.com/bar/soup.cwl"
# Security concern - can't resolve file: from http:
with pytest.raises(ValidationException):
url = fetcher.urljoin("http://example.com/fred/foo.cwl",
"file:///bar/soup.cwl")
# But this one is not "dangerous" on Linux
fetcher.urljoin("http://example.com/fred/foo.cwl", "c:/bar/soup.cwl")
finally:
sys.platform = actual_platform
