def test_cache_checks_url_validity(broker, req):
res, meta = broker.execute("mock_retrieve", req).wait()
assert cache_status(meta) == "miss"
meta = broker.execute("mock_retrieve", req).metadata
assert cache_status(meta) == "hit"
first_result_url = res[0]["result"]["location"]
path = urlparse(first_result_url).path
os.unlink(path)
assert requests.head(first_result_url).status_code == 404
res, meta = broker.execute("mock_retrieve", req).wait()
assert cache_status(meta) == "miss"
first_result_url = res[0]["result"]["location"]
path = urlparse(first_result_url).path
with open(path, "wb") as f:
f.write(b"make file size not match cached value")
assert requests.head(first_result_url).status_code == 200
assert os.stat(path).st_size != res[0]["result"]["contentLength"]
meta = broker.execute("mock_retrieve", req).metadata
assert cache_status(meta) == "miss"
meta = broker.execute("mock_retrieve", req).metadata
assert cache_status(meta) == "hit"
def download(result, path):
url = result["location"]
p = urlparse(url)
if p.scheme not in {"http", "https"}:
raise Exception("{}: Unsupported URL scheme '{}'".format(
url, p.scheme))
if PY2:
path = str(path)
fd, tmp_fname = tempfile.mkstemp(dir=os.path.dirname(path))
try:
with closing(requests.get(url, stream=True, timeout=20)) as res:
res.raise_for_status()
for chunk in res.iter_content(XFER_BLOCK_SIZE):
os.write(fd, chunk)
except Exception as exc:
try:
os.unlink(tmp_fname)
except Exception:
pass
raise_from(exc, exc)
finally:
os.close(fd)
# TODO: Verify number of bytes downloaded matches `result["contentLength"]`
os.rename(tmp_fname, path)
def instance(url=None, group=None, name=None):
"""Factory for config clients.
Returns a config client reading values from the URL taken from the
environment variable `SERVICELIB_CONFIG_URL`. If that environment variable
is not set, the hard-coded value `http://localhost:9999/settings/` will be
used.
"""
if url is None:
url = os.environ.get("SERVICELIB_CONFIG_URL",
"http://localhost:9999/settings/")
try:
return _instances[url]
except KeyError:
scheme = urlparse(url).scheme
if scheme in {"http", "https"}:
ret = HTTPConfigClient(url, group=group, name=name)
elif scheme == "file":
ret = FileConfigClient(url, group=group, name=name)
else:
raise ValueError("Unsupported URL scheme in {}".format(url))
_instances[url] = ret
return ret
def as_local_file(self, result):
ret = Path(urlparse(result["location"]).path)
for d in self.result_dirs:
try:
d = Path(d)
ret.relative_to(d)
st_size = ret.stat().st_size
if st_size == result["contentLength"]:
return ret
self.log.debug(
"as_local_file(%s): size %s does not match contentLength",
result,
st_size,
)
except Exception as exc:
self.log.info("as_local_file(%s): Not in %s: %s", result, d,
exc)
def service_url(name, local_only=False):
# TODO: Cache results.
c = _redis_pool.connection()
k = redis_key(name)
url = None
if local_only:
for parsed, unparsed in [(urlparse(u), u) for u in c.smembers(k)]:
if parsed.netloc.split(":")[0] == HOSTNAME:
url = unparsed
break
else:
url = c.srandmember(k)
if url is None:
raise Exception(
"No URL for service {} (local-only: {})".format(name, local_only)
)
return url
def services_by_netloc():
ret = {}
for service, urls in services_by_name().items():
for url in urls:
ret.setdefault(urlparse(url).netloc, set()).add(service)
return ret
def __init__(self, url, *args, **kwargs):
super(FileConfigClient, self).__init__(url, *args, **kwargs)
self._fname = urlparse(self.url).path