def download_network_cached(cache_url, dir_url, software_url, software_root, key, path, logger, signature_certificate_list, download_from_binary_cache_url_blacklist=None): """Downloads from a network cache provider return True if download succeeded. """ if not LIBNETWORKCACHE_ENABLED: return False if not(cache_url and dir_url and software_url and software_root): return False for url in download_from_binary_cache_url_blacklist: if software_url.startswith(url): return False try: nc = NetworkcacheClient(cache_url, dir_url, signature_certificate_list=signature_certificate_list or None) except TypeError: logger.warning('Incompatible version of networkcache, not using it.') return False logger.info('Downloading %s binary from network cache.' % software_url) try: file_descriptor = None json_entry_list = nc.select_generic(key) for entry in json_entry_list: json_information, _ = entry try: tags = json.loads(json_information) if tags.get('machine') != platform.machine(): continue if not os_matches(ast.literal_eval(tags.get('os')), patched_linux_distribution()): continue if tags.get('software_url') != software_url: continue if tags.get('software_root') != software_root: continue sha512 = tags.get('sha512') file_descriptor = nc.download(sha512) break except Exception: continue if file_descriptor is not None: f = open(path, 'w+b') try: shutil.copyfileobj(file_descriptor, f) finally: f.close() file_descriptor.close() return True except (IOError, DirectoryNotFound), e: logger.info('Failed to download from network cache %s: %s' % \ (software_url, str(e)))
def upload_network_cached(software_root, software_url, cached_key, cache_url, dir_url, path, logger, signature_private_key_file, shacache_cert_file, shacache_key_file, shadir_cert_file, shadir_key_file): """Upload file to a network cache server""" if not LIBNETWORKCACHE_ENABLED: return False if not (software_root and software_url and cached_key \ and cache_url and dir_url): return False logger.info('Uploading %s binary into network cache.' % software_url) # YXU: "file" and "urlmd5" should be removed when server side is ready kw = dict( file="file", urlmd5="urlmd5", software_url=software_url, software_root=software_root, machine=platform.machine(), os=str(patched_linux_distribution()) ) f = open(path, 'r') # convert '' into None in order to call nc nicely if not signature_private_key_file: signature_private_key_file = None if not shacache_cert_file: shacache_cert_file = None if not shacache_key_file: shacache_key_file = None if not shadir_cert_file: shadir_cert_file = None if not shadir_key_file: shadir_key_file = None try: nc = NetworkcacheClient(cache_url, dir_url, signature_private_key_file=signature_private_key_file, shacache_cert_file=shacache_cert_file, shacache_key_file=shacache_key_file, shadir_cert_file=shadir_cert_file, shadir_key_file=shadir_key_file) except TypeError: logger.warning('Incompatible version of networkcache, not using it.') return False try: return nc.upload_generic(f, cached_key, **kw) except (IOError, UploadError), e: logger.info('Failed to upload file. %s' % (str(e))) return False
def do_lookup(logger, cache_dir, software_url): if looks_like_md5(software_url): md5 = software_url else: md5 = hashlib.md5(software_url).hexdigest() try: url = '%s/%s' % (cache_dir, md5) logger.debug('Connecting to %s', url) req = requests.get(url, timeout=5) except (requests.Timeout, requests.ConnectionError): logger.critical('Cannot connect to cache server at %s', url) sys.exit(10) if not req.ok: if req.status_code == 404: logger.critical('Object not in cache: %s', software_url) else: logger.critical('Error while looking object %s: %s', software_url, req.reason) sys.exit(10) entries = req.json() if not entries: logger.info('Object found in cache, but has no binary entries.') return ostable = sorted(ast.literal_eval(json.loads(entry[0])['os']) for entry in entries) pt = prettytable.PrettyTable(['distribution', 'version', 'id', 'compatible?']) linux_distribution = patched_linux_distribution() for os in ostable: compatible = 'yes' if networkcache.os_matches(os, linux_distribution) else 'no' pt.add_row([os[0], os[1], os[2], compatible]) meta = json.loads(entries[0][0]) logger.info('Software URL: %s', meta['software_url']) logger.info('MD5: %s', md5) for line in pt.get_string(border=True, padding_width=0, vrules=prettytable.NONE).split('\n'): logger.info(line)