def getlist(self, parser, url): """ Uses the supplied parser to get a list of urls. Takes a parser object, url, and filering options. """ self.output.write('getlist(): fetching ' + url + '\n', 2) self.output.print_info('Downloading a list of mirrors...\n') # setup the ssl-fetch ouptut map connector_output = { 'info':self.output.write, 'debug': self.output.write, 'error': self.output.print_err, 'kwargs-info': {'level': 2}, 'kwargs-debug': {'level':2}, 'kwargs-error': {'level':0}, } fetcher = Connector(connector_output, self.proxies, USERAGENT) success, mirrorlist, timestamp = fetcher.fetch_content(url) parser.parse(mirrorlist) if (not mirrorlist) or len(parser.tuples()) == 0: self.output.print_err('Could not get mirror list. ' 'Check your internet connection.') self.output.write(' Got %d mirrors.\n' % len(parser.tuples())) return parser.tuples()
def fetch_seeds(self, seeds): '''Fetch new seed files''' # TODO: add support for separated fetching # setup the ssl-fetch ouptut map connector_output = { 'info': self.logger.info, 'error': self.logger.error, 'kwargs-info': {}, 'kwargs-error': {}, } http_check = re.compile(r'^(http|https)://') urls = [] messages = [] devseeds = self.config.get_key('developers.seeds') relseeds = self.config.get_key('release.seeds') if not http_check.match(devseeds) and not http_check.match(relseeds): urls.extend([self.config['seedurls']['developers.seeds'], self.config['seedurls']['release.seeds']]) else: urls.extend([devseeds, relseeds]) fetcher = Connector(connector_output, None, "Gentoo Keys") for url in urls: seed = url.rsplit('/', 1)[1] timestamp_prefix = seed[:3] timestamp_path = self.config['%s-timestamp' % timestamp_prefix] filename = self.config['%s-seedfile' % timestamp_prefix] file_exists = os.path.exists(filename) success, seeds, timestamp = fetcher.fetch_content(url, timestamp_path) if not timestamp and file_exists: messages.append("%s is already up to date." % seed) elif success: self.logger.debug("SeedHandler: fetch_seed; got results.") filename = filename + '.new' with open(filename, 'w') as seedfile: seedfile.write(seeds) filename = self.config['%s-seedfile' % timestamp_prefix] old = filename + '.old' try: self.logger.info("Backing up existing file...") if os.path.exists(old): self.logger.debug( "SeedHandler: fetch_seeds; Removing 'old' seed file: %s" % old) os.unlink(old) if os.path.exists(filename): self.logger.debug( "SeedHandler: fetch_seeds; Renaming current seed file to: " "%s" % old) os.rename(filename, old) self.logger.debug("SeedHandler: fetch_seeds; Renaming '.new' seed file to %s" % filename) os.rename(filename + '.new', filename) with open(timestamp_path, 'w+') as timestampfile: timestampfile.write(str(timestamp) + '\n') messages.append("Successfully fetched %s." % seed) except IOError: raise else: messages.append("Failed to fetch %s." % seed) return messages
class Fetch(object): def __init__(self, logger): self.logger = logger connector_output = { 'info': self.logger.info, 'debug': self.logger.debug, 'error': self.logger.error, 'exception': self.logger.exception, # we want any warnings to be printed to the terminal # so assign it to logging.error 'warning': self.logger.error, 'kwargs-info': {}, 'kwargs-debug': {}, 'kwargs-error': {}, 'kwargs-exception': {}, 'kwargs-warning': {}, } self.fetcher = Connector(connector_output, None, "Gentoo Keys") self.sig_path = None def fetch_url(self, url, filepath, signature=True, timestamp=None, timestamp_path=None, climit=60): if not timestamp_path: timestamp_path = filepath + ".timestamp" messages = [] self.logger.debug( _unicode("FETCH: fetching %s signed file ") % filepath) self.logger.debug( _unicode("FETCH: timestamp path: %s") % timestamp_path) success, signedfile, timestamp = self.fetcher.fetch_file( url, filepath, timestamp_path, climit=climit, timestamp=timestamp) if timestamp is '': self.logger.debug("Fetch.fetch_url; file not downloaded") return (False, messages) elif not success: messages.append(_unicode("File %s cannot be retrieved.") % filepath) elif '.' + url.rsplit('.', 1)[1] not in EXTENSIONS: self.logger.debug("File %s successfully retrieved.", filepath) if signature: success_fetch = False for ext in EXTENSIONS: sig_path = filepath + ext signature = url + ext self.logger.debug( _unicode("FETCH: fetching %s signature ") % signature) success_fetch, sig, timestamp = self.fetcher.fetch_file(signature, sig_path) if success_fetch: self.sig_path = sig_path break else: signature = None return (success, messages) def verify_cycle(self, tpath, climit=60): return self.fetcher.verify_cycle(tpath, climit=60)
def _fetch(self, base, archive_url, dest_dir): ''' Fetches overlay source archive. @params base: string of directory base for installed overlays. @params archive_url: string of URL where archive is located. @params dest_dir: string of destination of extracted archive. @rtype tuple (str of package location, bool to clean_archive) ''' ext = self.get_extension() if 'file://' not in archive_url: # set up ssl-fetch output map connector_output = { 'info': self.output.debug, 'error': self.output.error, 'kwargs-info': { 'level': 5 }, 'kwargs-debug': { 'level': 2 }, 'kwargs-error': { 'level': None }, } fetcher = Connector(connector_output, self.proxies, USERAGENT) success, archive, timestamp = fetcher.fetch_content(archive_url) pkg = path([base, self.parent.name + ext]) try: with fileopen(pkg, 'w+b') as out_file: out_file.write(archive) except Exception as error: raise Exception('Failed to store archive package in '\ '%(pkg)s\nError was: %(error)s'\ % ({'pkg': pkg, 'error': error})) else: self.clean_archive = False pkg = archive_url.replace('file://', '') return pkg
def _extract(self, base, tar_url, dest_dir): ext = '.tar.noidea' clean_tar = self.config['clean_tar'] for i in [('tar.%s' % e) for e in ('bz2', 'gz', 'lzma', 'xz', 'Z')] \ + ['tgz', 'tbz', 'taz', 'tlz', 'txz']: candidate_ext = '.%s' % i if self.src.endswith(candidate_ext): ext = candidate_ext break if 'file://' not in tar_url: # setup the ssl-fetch output map connector_output = { 'info': self.output.debug, 'error': self.output.error, 'kwargs-info': {'level': 2}, 'kwargs-error':{'level': None}, } fetcher = Connector(connector_output, self.proxies, USERAGENT) success, tar, timestamp = fetcher.fetch_content(tar_url) pkg = path([base, self.parent.name + ext]) try: with fileopen(pkg, 'w+b') as out_file: out_file.write(tar) except Exception as error: raise Exception('Failed to store tar package in ' + pkg + '\nError was:' + str(error)) else: clean_tar = False pkg = tar_url.replace('file://', '') # tar -v -x -f SOURCE -C TARGET args = ['-v', '-x', '-f', pkg, '-C', dest_dir] result = self.run_command(self.command(), args, cmd=self.type) if clean_tar: os.unlink(pkg) return result
def _fetch(self, base, archive_url, dest_dir): ''' Fetches overlay source archive. @params base: string of directory base for installed overlays. @params archive_url: string of URL where archive is located. @params dest_dir: string of destination of extracted archive. @rtype tuple (str of package location, bool to clean_archive) ''' ext = self.get_extension() if 'file://' not in archive_url: # set up ssl-fetch output map connector_output = { 'info': self.output.debug, 'error': self.output.error, 'kwargs-info': {'level': 5}, 'kwargs-debug': {'level': 2}, 'kwargs-error': {'level': None}, } fetcher = Connector(connector_output, self.proxies, USERAGENT) success, archive, timestamp = fetcher.fetch_content(archive_url) pkg = path([base, self.parent.name + ext]) try: with fileopen(pkg, 'w+b') as out_file: out_file.write(archive) except Exception as error: raise Exception('Failed to store archive package in '\ '%(pkg)s\nError was: %(error)s'\ % ({'pkg': pkg, 'error': error})) else: self.clean_archive = False pkg = archive_url.replace('file://', '') return pkg
def getlist(self, parser, url): """ Uses the supplied parser to get a list of urls. Takes a parser object, url, and filering options. """ self.output.write('getlist(): fetching ' + url + '\n', 2) self.output.print_info('Downloading a list of mirrors...\n') # setup the ssl-fetch ouptut map connector_output = { 'info': self.output.write, 'debug': self.output.write, 'error': self.output.print_err, 'kwargs-info': { 'level': 2 }, 'kwargs-debug': { 'level': 2 }, 'kwargs-error': { 'level': 0 }, } fetcher = Connector(connector_output, self.proxies, USERAGENT) success, mirrorlist, timestamp = fetcher.fetch_content(url, climit=60) parser.parse(mirrorlist) if (not mirrorlist) or len(parser.tuples()) == 0: self.output.print_err('Could not get mirror list. ' 'Check your internet connection.') self.output.write(' Got %d mirrors.\n' % len(parser.tuples())) return parser.tuples()
def __init__(self, logger): self.logger = logger connector_output = { 'info': self.logger.info, 'debug': self.logger.debug, 'error': self.logger.error, 'exception': self.logger.exception, # we want any warnings to be printed to the terminal # so assign it to logging.error 'warning': self.logger.error, 'kwargs-info': {}, 'kwargs-debug': {}, 'kwargs-error': {}, 'kwargs-exception': {}, 'kwargs-warning': {}, } self.fetcher = Connector(connector_output, None, "Gentoo Keys") self.sig_path = None
def cache(self): ''' Copy the remote overlay list to the local cache. >>> import tempfile >>> here = os.path.dirname(os.path.realpath(__file__)) >>> tmpdir = tempfile.mkdtemp(prefix="laymantmp_") >>> cache = os.path.join(tmpdir, 'cache') >>> myoptions = {'overlays' : ... ['file://' + here + '/tests/testfiles/global-overlays.xml'], ... 'cache' : cache, ... 'nocheck' : 'yes', ... 'proxy' : None} >>> from layman.config import OptionConfig >>> config = OptionConfig(myoptions) >>> config.set_option('quietness', 3) >>> a = RemoteDB(config) >>> a.cache() (True, True) >>> b = fileopen(a.filepath(config['overlays'])+'.xml') >>> b.readlines()[24] ' A collection of ebuilds from Gunnar Wrobel [[email protected]].\\n' >>> b.close() >>> os.unlink(a.filepath(config['overlays'])+'.xml') >>> a.overlays.keys() ['wrobel', 'wrobel-stable'] >>> import shutil >>> shutil.rmtree(tmpdir) ''' has_updates = False self._create_storage(self.config['storage']) # succeeded reset when a failure is detected succeeded = True url_lists = [self.urls, self.detached_urls, self.signed_urls] need_gpg = [False, True, True] # setup the ssl-fetch output map connector_output = { 'info': self.output.debug, 'error': self.output.error, 'kwargs-info': {'level': 2}, 'kwargs-error':{'level': None}, } fetcher = Connector(connector_output, self.proxies, USERAGENT) for index in range(0, 3): self.output.debug("RemoteDB.cache() index = %s" %str(index), 2) urls = url_lists[index] if need_gpg[index] and len(urls) and self.gpg is None: #initialize our gpg instance self.init_gpg() # main working loop for url in urls: sig = '' self.output.debug("RemoteDB.cache() url = %s is a tuple=%s" %(str(url), str(isinstance(url, tuple))), 2) filepath, mpath, tpath, sig = self._paths(url) if 'file://' in url: success, olist, timestamp = self._fetch_file( url, mpath, tpath) elif sig: success, olist, timestamp = fetcher.fetch_content( url[0], tpath) else: success, olist, timestamp = fetcher.fetch_content( url, tpath) if not success: #succeeded = False continue self.output.debug("RemoteDB.cache() len(olist) = %s" % str(len(olist)), 2) # GPG handling if need_gpg[index]: olist, verified = self.verify_gpg(url, sig, olist) if not verified: self.output.debug("RemoteDB.cache() gpg returned " "verified = %s" %str(verified), 2) succeeded = False filename = os.path.join(self.config['storage'], "Failed-to-verify-sig") self.write_cache(olist, filename) continue # Before we overwrite the old cache, check that the downloaded # file is intact and can be parsed if isinstance(url, tuple): olist = self._check_download(olist, url[0]) else: olist = self._check_download(olist, url) # Ok, now we can overwrite the old cache has_updates = max(has_updates, self.write_cache(olist, mpath, tpath, timestamp)) self.output.debug("RemoteDB.cache() self.urls: has_updates, " "succeeded %s, %s" % (str(has_updates), str(succeeded)), 4) return has_updates, succeeded
def cache(self): ''' Copy the remote overlay list to the local cache. @rtype tuple: reflects whether the cache has updates and whether or not the cache retrieval was successful. ''' has_updates = False self._create_storage(self.config['storage']) # succeeded reset when a failure is detected succeeded = True url_lists = [self.urls, self.detached_urls, self.signed_urls] need_gpg = [False, True, True] # setup the ssl-fetch output map connector_output = { 'info': self.output.info, 'debug': self.output.debug, 'error': self.output.error, 'exception': self.output.error, # we want any warnings to be printed to the terminal # so assign it to output.info with a lower noise level 'warning': self.output.info, 'kwargs-exception': { 'level': None }, 'kwargs-info': { 'level': 5 }, 'kwargs-debug': { 'level': 2 }, 'kwargs-error': { 'level': None }, 'kwargs-warning': { 'level': 2 }, } fetcher = Connector(connector_output, self.proxies, USERAGENT) for index in range(0, 3): self.output.debug("RemoteDB.cache() index = %s" % str(index), 2) urls = url_lists[index] if need_gpg[index] and len(urls) and self.gpg is None: #initialize our gpg instance self.init_gpg() # main working loop for url in urls: sig = '' self.output.debug( "RemoteDB.cache() url = %s is a tuple=%s" % (str(url), str(isinstance(url, tuple))), 2) filepath, mpath, tpath, sig = self._paths(url) if 'file://' in url: success, olist, timestamp = self._fetch_file( url, mpath, tpath) elif sig: success, olist, timestamp = fetcher.fetch_content( url[0], tpath, climit=60) else: success, olist, timestamp = fetcher.fetch_content( url, tpath, climit=60) if not success: #succeeded = False continue self.output.debug( "RemoteDB.cache() len(olist) = %s" % str(len(olist)), 2) # GPG handling if need_gpg[index]: olist, verified = self.verify_gpg(url, sig, olist) if not verified: self.output.debug( "RemoteDB.cache() gpg returned " "verified = %s" % str(verified), 2) succeeded = False filename = os.path.join(self.config['storage'], "Failed-to-verify-sig") self.write_cache(olist, filename) continue # Before we overwrite the old cache, check that the downloaded # file is intact and can be parsed if isinstance(url, tuple): olist = self._check_download(olist, url[0]) else: olist = self._check_download(olist, url) # Ok, now we can overwrite the old cache has_updates = max( has_updates, self.write_cache(olist, mpath, tpath, timestamp)) self.output.debug( "RemoteDB.cache() self.urls: has_updates, " "succeeded %s, %s" % (str(has_updates), str(succeeded)), 4) return has_updates, succeeded
def _verify(self, args, key, messages=None): if messages == None: messages = [] self.category = args.category filepath, signature = args.filename, args.signature timestamp_path = None isurl = success = verified = False if filepath.startswith('http'): isurl = True url = filepath filepath = args.destination # a bit hackish, but save it to current directory # with download file name if not filepath: filepath = url.split('/')[-1] self.logger.debug(_unicode( "ACTIONS: verify; destination filepath was " "not supplied, using current directory ./%s") % filepath) if args.timestamp: timestamp_path = filepath + ".timestamp" climit = 60 else: climit = 0 sig_path = None if isurl: from sslfetch.connections import Connector connector_output = { 'info': self.logger.info, 'debug': self.logger.debug, 'error': self.logger.error, 'exception': self.logger.exception, # we want any warnings to be printed to the terminal # so assign it to logging.error 'warning': self.logger.error, 'kwargs-info': {}, 'kwargs-debug': {}, 'kwargs-error': {}, 'kwargs-exception': {}, 'kwargs-warning': {}, } fetcher = Connector(connector_output, None, "Gentoo Keys") self.logger.debug( _unicode("ACTIONS: verify; fetching %s signed file ") % filepath) self.logger.debug( _unicode("ACTIONS: verify; timestamp path: %s") % timestamp_path) success, signedfile, timestamp = fetcher.fetch_file( url, filepath, timestamp_path, climit=climit) if not success: messages.append(_unicode("File %s cannot be retrieved.") % filepath) elif '.' + url.rsplit('.', 1)[1] not in EXTENSIONS: if not signature: success_fetch = False for ext in EXTENSIONS: sig_path = filepath + ext if isurl: signature = url + ext self.logger.debug( _unicode("ACTIONS: verify; fetching %s signature ") % signature) success_fetch, sig, timestamp = fetcher.fetch_file(signature, sig_path) if success_fetch: break else: signature = None else: filepath = os.path.abspath(filepath) self.logger.debug( _unicode("ACTIONS: verify; local file %s") % filepath) success = os.path.isfile(filepath) if (not signature and '.' + filepath.rsplit('.', 1)[-1] not in EXTENSIONS): success_fetch = False for ext in EXTENSIONS: sig_path = filepath + ext sig_path = os.path.abspath(sig_path) self.logger.debug( _unicode("ACTIONS: verify; checking %s signature ") % sig_path) success_sig = os.path.isfile(sig_path) if success_sig: break else: sig_path = None self.logger.info("Verifying file...") verified = False results = self.gpg.verify_file(key, sig_path, filepath) keyid = key.keyid[0] (valid, trust) = results.verified if valid: verified = True messages.extend( [_unicode("Verification succeeded.: %s") % (filepath), _unicode("Key info...............: %s <%s>, %s") % ( key.name, key.nick, keyid), _unicode(" category, nick.....: %s %s") % (args.category, args.nick)]) else: messages.extend( [_unicode("Verification failed....: %s") % (filepath), _unicode("Key info...............: %s <%s>, %s") % ( key.name, key.nick, keyid)]) found, args, new_msgs = self.keyhandler.autosearch_key(args, results) messages.extend(new_msgs) if found: return self.verify(args, messages) return (verified, messages)
def verify(self, args): '''File verification action''' connector_output = { 'info': self.logger.debug, 'error': self.logger.error, 'kwargs-info': {}, 'kwargs-error': {}, } if not args.filename: return (False, ['Please provide a signed file.']) if not args.category: args.category = self.config.get_key('verify_keyring') self.logger.debug("ACTIONS: verify; keyring category not specified, using default: %s" % args.category) handler = SeedHandler(self.logger, self.config) keys = handler.load_category(args.category) if not keys: return (False, ['No installed keys found, try installkey action.']) keyrings = self.config.get_key('keyring') catdir = os.path.join(keyrings, args.category) self.logger.debug("ACTIONS: verify; catdir = %s" % catdir) self.gpg = GkeysGPG(self.config, catdir) filepath, signature = args.filename, args.signature timestamp_path = None isurl = success = verified = False if filepath.startswith('http'): isurl = True url = filepath filepath = args.destination # a bit hackish, but save it to current directory # with download file name if not filepath: filepath = url.split('/')[-1] self.logger.debug("ACTIONS: verify; destination filepath was " "not supplied, using current directory ./%s" % filepath) if args.timestamp: timestamp_path = filepath + ".timestamp" if isurl: from sslfetch.connections import Connector connector_output = { 'info': self.logger.info, 'debug': self.logger.debug, 'error': self.logger.error, 'kwargs-info': {}, 'kwargs-debug': {}, 'kwargs-error': {}, } fetcher = Connector(connector_output, None, "Gentoo Keys") self.logger.debug("ACTIONS: verify; fetching %s signed file " % filepath) self.logger.debug("ACTIONS: verify; timestamp path: %s" % timestamp_path) success, signedfile, timestamp = fetcher.fetch_file(url, filepath, timestamp_path) else: filepath = os.path.abspath(filepath) self.logger.debug("ACTIONS: verify; local file %s" % filepath) success = os.path.isfile(filepath) if not success: messages = ["File %s cannot be retrieved." % filepath] else: if not signature: EXTENSIONS = ['.sig', '.asc', 'gpg','.gpgsig'] success_fetch = False for ext in EXTENSIONS: sig_path = filepath + ext if isurl: signature = url + ext self.logger.debug("ACTIONS: verify; fetching %s signature " % signature) success_fetch, sig, timestamp = fetcher.fetch_file(signature, sig_path) else: signature = filepath + ext signature = os.path.abspath(signature) self.logger.debug("ACTIONS: verify; checking %s signature " % signature) success_fetch = os.path.isfile(signature) if success_fetch: break else: sig_path = signature messages = [] self.logger.info("Verifying file...") verified = False key = keys.nick_search(args.nick) if not key: messages.append("Failed to find nick: %s in %s category" % (args.nick, args.category)) return (False, messages) results = self.gpg.verify_file(key, sig_path, filepath) keyid = key.keyid[0] (valid, trust) = results.verified if valid: verified = True messages = ["Verification succeeded.: %s" % (filepath), "Key info...............: %s <%s>, %s" % ( key.name, key.nick, keyid)] else: messages = ["Verification failed..... %s:" % (filepath), "Key info................: %s <%s>, %s" % ( key.name, key.nick, keyid)] return (verified, messages)
def _verify(self, args, key, messages=None): if messages == None: messages = [] self._set_category(args.category) filepath, signature = args.filename, args.signature timestamp_path = None isurl = success = verified = False if filepath.startswith('http'): isurl = True url = filepath filepath = args.destination # a bit hackish, but save it to current directory # with download file name if not filepath: filepath = url.split('/')[-1] self.logger.debug( _unicode("ACTIONS: verify; destination filepath was " "not supplied, using current directory ./%s") % filepath) if args.timestamp: timestamp_path = filepath + ".timestamp" climit = 60 else: climit = 0 sig_path = None if isurl: from sslfetch.connections import Connector connector_output = { 'info': self.logger.info, 'debug': self.logger.debug, 'error': self.logger.error, 'exception': self.logger.exception, # we want any warnings to be printed to the terminal # so assign it to logging.error 'warning': self.logger.error, 'kwargs-info': {}, 'kwargs-debug': {}, 'kwargs-error': {}, 'kwargs-exception': {}, 'kwargs-warning': {}, } fetcher = Connector(connector_output, None, "Gentoo Keys") self.logger.debug( _unicode("ACTIONS: verify; fetching %s signed file ") % filepath) self.logger.debug( _unicode("ACTIONS: verify; timestamp path: %s") % timestamp_path) success, signedfile, timestamp = fetcher.fetch_file(url, filepath, timestamp_path, climit=climit) if not success: messages.append( _unicode("File %s cannot be retrieved.") % filepath) elif '.' + url.rsplit('.', 1)[1] not in EXTENSIONS: if not signature: success_fetch = False for ext in EXTENSIONS: sig_path = filepath + ext if isurl: signature = url + ext self.logger.debug( _unicode( "ACTIONS: verify; fetching %s signature ") % signature) success_fetch, sig, timestamp = fetcher.fetch_file( signature, sig_path) if success_fetch: break else: signature = None elif signature is not None and os.path.exists(signature): sig_path = signature else: filepath = os.path.abspath(filepath) self.logger.debug( _unicode("ACTIONS: verify; local file %s") % filepath) success = os.path.isfile(filepath) if (not signature and '.' + filepath.rsplit('.', 1)[-1] not in EXTENSIONS): success_fetch = False for ext in EXTENSIONS: sig_path = filepath + ext sig_path = os.path.abspath(sig_path) self.logger.debug( _unicode("ACTIONS: verify; checking %s signature ") % sig_path) success_sig = os.path.isfile(sig_path) if success_sig: break else: sig_path = None self.logger.info("Verifying file...") verified = False results = self.gpg.verify_file(key, sig_path, filepath) keyid = key.keyid[0] (valid, trust) = results.verified if valid: verified = True messages.extend([ _unicode("Verification succeeded.: %s") % (filepath), _unicode("Key info...............: %s <%s>, %s") % (key.name, key.nick, keyid), _unicode(" category, nick.....: %s %s") % (args.category, args.nick) ]) else: messages.extend([ _unicode("Verification failed....: %s") % (filepath), _unicode("Key info...............: %s <%s>, %s") % (key.name, key.nick, keyid) ]) found, args, new_msgs = self.keyhandler.autosearch_key( args, results) messages.extend(new_msgs) if found: return self.verify(args, messages) return (verified, messages)
def cache(self): ''' Copy the remote overlay list to the local cache. @rtype tuple: reflects whether the cache has updates and whether or not the cache retrieval was successful. ''' has_updates = False self._create_storage(self.config['storage']) # succeeded reset when a failure is detected succeeded = True url_lists = [self.urls, self.detached_urls, self.signed_urls] need_gpg = [False, True, True] # setup the ssl-fetch output map connector_output = { 'info': self.output.debug, 'error': self.output.error, 'kwargs-info': {'level': 2}, 'kwargs-error':{'level': None}, } fetcher = Connector(connector_output, self.proxies, USERAGENT) for index in range(0, 3): self.output.debug("RemoteDB.cache() index = %s" %str(index), 2) urls = url_lists[index] if need_gpg[index] and len(urls) and self.gpg is None: #initialize our gpg instance self.init_gpg() # main working loop for url in urls: sig = '' self.output.debug("RemoteDB.cache() url = %s is a tuple=%s" %(str(url), str(isinstance(url, tuple))), 2) filepath, mpath, tpath, sig = self._paths(url) if 'file://' in url: success, olist, timestamp = self._fetch_file( url, mpath, tpath) elif sig: success, olist, timestamp = fetcher.fetch_content( url[0], tpath) else: success, olist, timestamp = fetcher.fetch_content( url, tpath) if not success: #succeeded = False continue self.output.debug("RemoteDB.cache() len(olist) = %s" % str(len(olist)), 2) # GPG handling if need_gpg[index]: olist, verified = self.verify_gpg(url, sig, olist) if not verified: self.output.debug("RemoteDB.cache() gpg returned " "verified = %s" %str(verified), 2) succeeded = False filename = os.path.join(self.config['storage'], "Failed-to-verify-sig") self.write_cache(olist, filename) continue # Before we overwrite the old cache, check that the downloaded # file is intact and can be parsed if isinstance(url, tuple): olist = self._check_download(olist, url[0]) else: olist = self._check_download(olist, url) # Ok, now we can overwrite the old cache has_updates = max(has_updates, self.write_cache(olist, mpath, tpath, timestamp)) self.output.debug("RemoteDB.cache() self.urls: has_updates, " "succeeded %s, %s" % (str(has_updates), str(succeeded)), 4) return has_updates, succeeded
def _verify(self, args, handler, key, messages=None): if messages == None: messages = [] filepath, signature = args.filename, args.signature timestamp_path = None isurl = success = verified = False if filepath.startswith('http'): isurl = True url = filepath filepath = args.destination # a bit hackish, but save it to current directory # with download file name if not filepath: filepath = url.split('/')[-1] self.logger.debug(_unicode( "ACTIONS: verify; destination filepath was " "not supplied, using current directory ./%s") % filepath) if args.timestamp: timestamp_path = filepath + ".timestamp" sig_path = None if isurl: from sslfetch.connections import Connector connector_output = { 'info': self.logger.info, 'debug': self.logger.debug, 'error': self.logger.error, 'kwargs-info': {}, 'kwargs-debug': {}, 'kwargs-error': {}, } fetcher = Connector(connector_output, None, "Gentoo Keys") self.logger.debug( _unicode("ACTIONS: verify; fetching %s signed file ") % filepath) self.logger.debug( _unicode("ACTIONS: verify; timestamp path: %s") % timestamp_path) success, signedfile, timestamp = fetcher.fetch_file( url, filepath, timestamp_path) if not success: messages.append(_unicode("File %s cannot be retrieved.") % filepath) elif '.' + url.rsplit('.', 1)[1] not in EXTENSIONS: if not signature: success_fetch = False for ext in EXTENSIONS: sig_path = filepath + ext if isurl: signature = url + ext self.logger.debug( _unicode("ACTIONS: verify; fetching %s signature ") % signature) success_fetch, sig, timestamp = fetcher.fetch_file(signature, sig_path) if success_fetch: break else: signature = None else: filepath = os.path.abspath(filepath) self.logger.debug( _unicode("ACTIONS: verify; local file %s") % filepath) success = os.path.isfile(filepath) if (not signature and '.' + filepath.rsplit('.', 1)[1] not in EXTENSIONS): success_fetch = False for ext in EXTENSIONS: sig_path = filepath + ext sig_path = os.path.abspath(sig_path) self.logger.debug( _unicode("ACTIONS: verify; checking %s signature ") % sig_path) success_sig = os.path.isfile(sig_path) if success_sig: break else: sig_path = None self.logger.info("Verifying file...") verified = False results = self.gpg.verify_file(key, sig_path, filepath) keyid = key.keyid[0] (valid, trust) = results.verified if valid: verified = True messages.extend( [_unicode("Verification succeeded.: %s") % (filepath), _unicode("Key info...............: %s <%s>, %s") % ( key.name, key.nick, keyid), _unicode(" category, nick.....: %s %s") % (args.category, args.nick)]) else: messages.extend( [_unicode("Verification failed....: %s") % (filepath), _unicode("Key info...............: %s <%s>, %s") % ( key.name, key.nick, keyid)]) has_no_pubkey, s_keyid = results.no_pubkey if has_no_pubkey: messages.append( _unicode("Auto-searching for key.: 0x%s") % s_keyid) # reset all but keyid and pass thru data args.keyid = s_keyid args.keydir = None args.fingerprint = None args.exact = False args.category = None args.nick = None args.name = None args.all = False keys = self.key_search(args, data_only=True) if keys: args.category = list(keys)[0] args.nick = keys[args.category][0].nick return self.verify(args, messages) messages.append(_unicode("Failed to find gpg key.: 0x%s") % s_keyid) return (verified, messages)