def download_network_cached(cache_url, dir_url, software_url, software_root,
key, path, logger, signature_certificate_list,
download_from_binary_cache_url_blacklist=None):
"""Downloads from a network cache provider
return True if download succeeded.
"""
if not LIBNETWORKCACHE_ENABLED:
return False
if not(cache_url and dir_url and software_url and software_root):
return False
for url in download_from_binary_cache_url_blacklist:
if software_url.startswith(url):
return False
try:
nc = NetworkcacheClient(cache_url, dir_url,
signature_certificate_list=signature_certificate_list or None)
except TypeError:
logger.warning('Incompatible version of networkcache, not using it.')
return False
logger.info('Downloading %s binary from network cache.' % software_url)
try:
file_descriptor = None
json_entry_list = nc.select_generic(key)
for entry in json_entry_list:
json_information, _ = entry
try:
tags = json.loads(json_information)
if tags.get('machine') != platform.machine():
continue
if not os_matches(ast.literal_eval(tags.get('os')),
distribution_tuple()):
continue
if tags.get('software_url') != software_url:
continue
if tags.get('software_root') != software_root:
continue
sha512 = tags.get('sha512')
file_descriptor = nc.download(sha512)
break
except Exception:
continue
if file_descriptor is not None:
f = open(path, 'w+b')
try:
shutil.copyfileobj(file_descriptor, f)
finally:
f.close()
file_descriptor.close()
return True
except (IOError, DirectoryNotFound) as e:
logger.info('Failed to download from network cache %s: %s' % \
(software_url, str(e)))
return False
def upload_network_cached(software_root, software_url, cached_key,
cache_url, dir_url, path, logger, signature_private_key_file,
shacache_ca_file, shacache_cert_file, shacache_key_file,
shadir_ca_file, shadir_cert_file, shadir_key_file):
"""Upload file to a network cache server"""
if not LIBNETWORKCACHE_ENABLED:
return False
if not (software_root and software_url and cached_key \
and cache_url and dir_url):
return False
logger.info('Uploading %s binary into network cache.' % software_url)
# YXU: "file" and "urlmd5" should be removed when server side is ready
kw = dict(
file="file",
urlmd5="urlmd5",
software_url=software_url,
software_root=software_root,
machine=platform.machine(),
os=str(distribution_tuple())
)
f = open(path, 'r')
# convert '' into None in order to call nc nicely
if not signature_private_key_file:
signature_private_key_file = None
if not shacache_ca_file:
shacache_ca_file = None
if not shacache_cert_file:
shacache_cert_file = None
if not shacache_key_file:
shacache_key_file = None
if not shadir_ca_file:
shadir_ca_file = None
if not shadir_cert_file:
shadir_cert_file = None
if not shadir_key_file:
shadir_key_file = None
try:
nc = NetworkcacheClient(cache_url, dir_url,
signature_private_key_file=signature_private_key_file,
shacache_ca_file=shacache_ca_file,
shacache_cert_file=shacache_cert_file,
shacache_key_file=shacache_key_file,
shadir_ca_file=shadir_ca_file,
shadir_cert_file=shadir_cert_file,
shadir_key_file=shadir_key_file)
except TypeError:
logger.warning('Incompatible version of networkcache, not using it.')
return False
try:
return nc.upload_generic(f, cached_key, **kw)
except (IOError, UploadError), e:
logger.info('Failed to upload file. %s' % (str(e)))
return False
def do_lookup(logger, cache_dir, software_url):
if looks_like_md5(software_url):
md5 = software_url
else:
md5 = hashlib.md5(software_url).hexdigest()
try:
url = "%s/%s" % (cache_dir, md5)
logger.debug("Connecting to %s", url)
req = requests.get(url, timeout=5)
except (requests.Timeout, requests.ConnectionError):
logger.critical("Cannot connect to cache server at %s", url)
sys.exit(10)
if not req.ok:
if req.status_code == 404:
logger.critical("Object not in cache: %s", software_url)
else:
logger.critical("Error while looking object %s: %s", software_url, req.reason)
sys.exit(10)
entries = req.json()
if not entries:
logger.info("Object found in cache, but has no binary entries.")
return
ostable = sorted(ast.literal_eval(json.loads(entry[0])["os"]) for entry in entries)
pt = prettytable.PrettyTable(["distribution", "version", "id", "compatible?"])
linux_distribution = distribution_tuple()
for os in ostable:
compatible = "yes" if networkcache.os_matches(os, linux_distribution) else "no"
pt.add_row([os[0], os[1], os[2], compatible])
meta = json.loads(entries[0][0])
logger.info("Software URL: %s", meta["software_url"])
logger.info("MD5: %s", md5)
for line in pt.get_string(border=True, padding_width=0, vrules=prettytable.NONE).split("\n"):
logger.info(line)
