def download_network_cached(cache_url, dir_url, software_url, software_root,
key, path, logger, signature_certificate_list,
binary_cache_url_blacklist=None):
"""Downloads from a network cache provider
return True if download succeeded.
"""
if not LIBNETWORKCACHE_ENABLED:
return False
if not(cache_url and dir_url and software_url and software_root):
return False
for url in binary_cache_url_blacklist:
if software_url.startswith(url):
return False
# In order to call nc nicely.
if len(signature_certificate_list) == 0:
signature_certificate_list = None
try:
nc = NetworkcacheClient(cache_url, dir_url,
signature_certificate_list=signature_certificate_list)
except TypeError:
logger.warning('Incompatible version of networkcache, not using it.')
return False
logger.info('Downloading %s binary from network cache.' % software_url)
try:
file_descriptor = None
json_entry_list = nc.select_generic(key)
for entry in json_entry_list:
json_information, _ = entry
try:
tags = json.loads(json_information)
if tags.get('machine') != platform.machine():
continue
if tags.get('os') != str(platform.linux_distribution()):
continue
if tags.get('software_url') != software_url:
continue
if tags.get('software_root') != software_root:
continue
sha512 = tags.get('sha512')
file_descriptor = nc.download(sha512)
break
except Exception:
continue
if file_descriptor is not None:
f = open(path, 'w+b')
try:
shutil.copyfileobj(file_descriptor, f)
finally:
f.close()
file_descriptor.close()
return True
except (IOError, DirectoryNotFound), e:
logger.info('Failed to download from network cache %s: %s' % \
(software_url, str(e)))
def upload_network_cached(software_root, software_url, cached_key,
cache_url, dir_url, path, logger, signature_private_key_file,
shacache_ca_file, shacache_cert_file, shacache_key_file,
shadir_ca_file, shadir_cert_file, shadir_key_file):
"""Upload file to a network cache server"""
if not LIBNETWORKCACHE_ENABLED:
return False
if not (software_root and software_url and cached_key \
and cache_url and dir_url):
return False
logger.info('Uploading %s binary into network cache.' % software_url)
# YXU: "file" and "urlmd5" should be removed when server side is ready
kw = dict(
file="file",
urlmd5="urlmd5",
software_url=software_url,
software_root=software_root,
machine=platform.machine(),
os=str(distribution_tuple())
)
f = open(path, 'r')
# convert '' into None in order to call nc nicely
if not signature_private_key_file:
signature_private_key_file = None
if not shacache_ca_file:
shacache_ca_file = None
if not shacache_cert_file:
shacache_cert_file = None
if not shacache_key_file:
shacache_key_file = None
if not shadir_ca_file:
shadir_ca_file = None
if not shadir_cert_file:
shadir_cert_file = None
if not shadir_key_file:
shadir_key_file = None
try:
nc = NetworkcacheClient(cache_url, dir_url,
signature_private_key_file=signature_private_key_file,
shacache_ca_file=shacache_ca_file,
shacache_cert_file=shacache_cert_file,
shacache_key_file=shacache_key_file,
shadir_ca_file=shadir_ca_file,
shadir_cert_file=shadir_cert_file,
shadir_key_file=shadir_key_file)
except TypeError:
logger.warning('Incompatible version of networkcache, not using it.')
return False
try:
return nc.upload_generic(f, cached_key, **kw)
except (IOError, UploadError), e:
logger.info('Failed to upload file. %s' % (str(e)))
return False
def upload_network_cached(dir_url, cache_url, external_url, path, logger,
signature_private_key_file, shacache_ca_file,
shacache_cert_file, shacache_key_file,
shadir_ca_file, shadir_cert_file, shadir_key_file):
"""Upload file to a network cache server"""
# XXX use helper and FACTOR code
if not LIBNETWORKCACHE_ENABLED:
return False
if not (dir_url and cache_url):
return False
logger.info('Uploading %s into network cache.', external_url)
file_name = get_filename_from_url(external_url)
directory_key = get_directory_key(external_url)
kw = dict(file_name=file_name,
urlmd5=hashlib.md5(external_url.encode()).hexdigest())
# convert '' into None in order to call nc nicely
if not signature_private_key_file:
signature_private_key_file = None
if not shacache_ca_file:
shacache_ca_file = None
if not shacache_cert_file:
shacache_cert_file = None
if not shacache_key_file:
shacache_key_file = None
if not shadir_ca_file:
shadir_ca_file = None
if not shadir_cert_file:
shadir_cert_file = None
if not shadir_key_file:
shadir_key_file = None
try:
nc = NetworkcacheClient(
cache_url,
dir_url,
signature_private_key_file=signature_private_key_file,
shacache_ca_file=shacache_ca_file,
shacache_cert_file=shacache_cert_file,
shacache_key_file=shacache_key_file,
shadir_ca_file=shadir_ca_file,
shadir_cert_file=shadir_cert_file,
shadir_key_file=shadir_key_file)
except TypeError:
logger.warning('Incompatible version of networkcache, not using it.')
return False
try:
with open(path, 'rb') as f:
return nc.upload(f, directory_key, **kw)
except (IOError, UploadError) as e:
logger.info('Fail to upload file. %s', e)
return False
def upload_network_cached(dir_url, cache_url, external_url, path, logger,
signature_private_key_file, shacache_ca_file, shacache_cert_file,
shacache_key_file, shadir_ca_file, shadir_cert_file, shadir_key_file):
"""Upload file to a network cache server"""
# XXX use helper and FACTOR code
if not LIBNETWORKCACHE_ENABLED:
return False
if not (dir_url and cache_url):
return False
logger.info('Uploading %s into network cache.', external_url)
file_name = get_filename_from_url(external_url)
directory_key = get_directory_key(external_url)
kw = dict(file_name=file_name,
urlmd5=hashlib.md5(external_url.encode()).hexdigest())
# convert '' into None in order to call nc nicely
if not signature_private_key_file:
signature_private_key_file = None
if not shacache_ca_file:
shacache_ca_file = None
if not shacache_cert_file:
shacache_cert_file = None
if not shacache_key_file:
shacache_key_file = None
if not shadir_ca_file:
shadir_ca_file = None
if not shadir_cert_file:
shadir_cert_file = None
if not shadir_key_file:
shadir_key_file = None
try:
nc = NetworkcacheClient(cache_url, dir_url,
signature_private_key_file=signature_private_key_file,
shacache_ca_file=shacache_ca_file,
shacache_cert_file=shacache_cert_file,
shacache_key_file=shacache_key_file,
shadir_ca_file=shadir_ca_file,
shadir_cert_file=shadir_cert_file,
shadir_key_file=shadir_key_file)
except TypeError:
logger.warning('Incompatible version of networkcache, not using it.')
return False
try:
with open(path, 'rb') as f:
return nc.upload(f, directory_key, **kw)
except (IOError, UploadError) as e:
logger.info('Fail to upload file. %s', e)
return False
def upload_index_network_cached(dir_url, cache_url, external_url, base,
requirement, content, logger,
signature_private_key_file, shacache_ca_file,
shacache_cert_file, shacache_key_file,
shadir_ca_file, shadir_cert_file,
shadir_key_file):
# XXX use helper and FACTOR code
"""Upload content of a web page to a network cache server"""
if not LIBNETWORKCACHE_ENABLED:
return False
if not (dir_url and cache_url):
return False
logger.info('Uploading %s content into network cache.', external_url)
directory_key = get_index_directory_key(external_url, requirement)
kw = dict(file="file",
base=base,
urlmd5=hashlib.md5(external_url).hexdigest(),
requirement=requirement)
import tempfile
f = tempfile.TemporaryFile()
f.write(content)
# convert '' into None in order to call nc nicely
if not signature_private_key_file:
signature_private_key_file = None
if not shacache_ca_file:
shacache_ca_file = None
if not shacache_cert_file:
shacache_cert_file = None
if not shacache_key_file:
shacache_key_file = None
if not shadir_ca_file:
shadir_ca_file = None
if not shadir_cert_file:
shadir_cert_file = None
if not shadir_key_file:
shadir_key_file = None
try:
nc = NetworkcacheClient(
cache_url,
dir_url,
signature_private_key_file=signature_private_key_file,
shacache_ca_file=shacache_ca_file,
shacache_cert_file=shacache_cert_file,
shacache_key_file=shacache_key_file,
shadir_ca_file=shadir_ca_file,
shadir_cert_file=shadir_cert_file,
shadir_key_file=shadir_key_file)
except TypeError:
logger.warning('Incompatible version of networkcache, not using it.')
return False
try:
return nc.upload_generic(f, directory_key, **kw)
except (IOError, UploadError) as e:
logger.info('Fail to upload file. %s', e)
return False
finally:
f.close()
return True
def upload_index_network_cached(dir_url, cache_url, external_url, base, requirement, content, logger,
signature_private_key_file, shacache_ca_file, shacache_cert_file,
shacache_key_file, shadir_ca_file, shadir_cert_file, shadir_key_file):
# XXX use helper and FACTOR code
"""Upload content of a web page to a network cache server"""
if not LIBNETWORKCACHE_ENABLED:
return False
if not (dir_url and cache_url):
return False
logger.info('Uploading %s content into network cache.', external_url)
directory_key = get_index_directory_key(external_url, requirement)
kw = dict(file="file",
base=base,
urlmd5=hashlib.md5(external_url).hexdigest(),
requirement=requirement)
import tempfile
f = tempfile.TemporaryFile()
f.write(content)
# convert '' into None in order to call nc nicely
if not signature_private_key_file:
signature_private_key_file = None
if not shacache_ca_file:
shacache_ca_file = None
if not shacache_cert_file:
shacache_cert_file = None
if not shacache_key_file:
shacache_key_file = None
if not shadir_ca_file:
shadir_ca_file = None
if not shadir_cert_file:
shadir_cert_file = None
if not shadir_key_file:
shadir_key_file = None
try:
nc = NetworkcacheClient(cache_url, dir_url,
signature_private_key_file=signature_private_key_file,
shacache_ca_file=shacache_ca_file,
shacache_cert_file=shacache_cert_file,
shacache_key_file=shacache_key_file,
shadir_ca_file=shadir_ca_file,
shadir_cert_file=shadir_cert_file,
shadir_key_file=shadir_key_file)
except TypeError:
logger.warning('Incompatible version of networkcache, not using it.')
return False
try:
return nc.upload_generic(f, directory_key, **kw)
except (IOError, UploadError) as e:
logger.info('Fail to upload file. %s', e)
return False
finally:
f.close()
return True
