def getlist(self, parser, url):
"""
Uses the supplied parser to get a list of urls.
Takes a parser object, url, and filering options.
"""
self.output.write('getlist(): fetching ' + url + '\n', 2)
self.output.print_info('Downloading a list of mirrors...\n')
# setup the ssl-fetch ouptut map
connector_output = {
'info':self.output.write,
'debug': self.output.write,
'error': self.output.print_err,
'kwargs-info': {'level': 2},
'kwargs-debug': {'level':2},
'kwargs-error': {'level':0},
}
fetcher = Connector(connector_output, self.proxies, USERAGENT)
success, mirrorlist, timestamp = fetcher.fetch_content(url)
parser.parse(mirrorlist)
if (not mirrorlist) or len(parser.tuples()) == 0:
self.output.print_err('Could not get mirror list. '
'Check your internet connection.')
self.output.write(' Got %d mirrors.\n' % len(parser.tuples()))
return parser.tuples()
def fetch_seeds(self, seeds):
'''Fetch new seed files'''
# TODO: add support for separated fetching
# setup the ssl-fetch ouptut map
connector_output = {
'info': self.logger.info,
'error': self.logger.error,
'kwargs-info': {},
'kwargs-error': {},
}
http_check = re.compile(r'^(http|https)://')
urls = []
messages = []
devseeds = self.config.get_key('developers.seeds')
relseeds = self.config.get_key('release.seeds')
if not http_check.match(devseeds) and not http_check.match(relseeds):
urls.extend([self.config['seedurls']['developers.seeds'], self.config['seedurls']['release.seeds']])
else:
urls.extend([devseeds, relseeds])
fetcher = Connector(connector_output, None, "Gentoo Keys")
for url in urls:
seed = url.rsplit('/', 1)[1]
timestamp_prefix = seed[:3]
timestamp_path = self.config['%s-timestamp' % timestamp_prefix]
filename = self.config['%s-seedfile' % timestamp_prefix]
file_exists = os.path.exists(filename)
success, seeds, timestamp = fetcher.fetch_content(url, timestamp_path)
if not timestamp and file_exists:
messages.append("%s is already up to date." % seed)
elif success:
self.logger.debug("SeedHandler: fetch_seed; got results.")
filename = filename + '.new'
with open(filename, 'w') as seedfile:
seedfile.write(seeds)
filename = self.config['%s-seedfile' % timestamp_prefix]
old = filename + '.old'
try:
self.logger.info("Backing up existing file...")
if os.path.exists(old):
self.logger.debug(
"SeedHandler: fetch_seeds; Removing 'old' seed file: %s"
% old)
os.unlink(old)
if os.path.exists(filename):
self.logger.debug(
"SeedHandler: fetch_seeds; Renaming current seed file to: "
"%s" % old)
os.rename(filename, old)
self.logger.debug("SeedHandler: fetch_seeds; Renaming '.new' seed file to %s"
% filename)
os.rename(filename + '.new', filename)
with open(timestamp_path, 'w+') as timestampfile:
timestampfile.write(str(timestamp) + '\n')
messages.append("Successfully fetched %s." % seed)
except IOError:
raise
else:
messages.append("Failed to fetch %s." % seed)
return messages
def _fetch(self, base, archive_url, dest_dir):
'''
Fetches overlay source archive.
@params base: string of directory base for installed overlays.
@params archive_url: string of URL where archive is located.
@params dest_dir: string of destination of extracted archive.
@rtype tuple (str of package location, bool to clean_archive)
'''
ext = self.get_extension()
if 'file://' not in archive_url:
# set up ssl-fetch output map
connector_output = {
'info': self.output.debug,
'error': self.output.error,
'kwargs-info': {
'level': 5
},
'kwargs-debug': {
'level': 2
},
'kwargs-error': {
'level': None
},
}
fetcher = Connector(connector_output, self.proxies, USERAGENT)
success, archive, timestamp = fetcher.fetch_content(archive_url)
pkg = path([base, self.parent.name + ext])
try:
with fileopen(pkg, 'w+b') as out_file:
out_file.write(archive)
except Exception as error:
raise Exception('Failed to store archive package in '\
'%(pkg)s\nError was: %(error)s'\
% ({'pkg': pkg, 'error': error}))
else:
self.clean_archive = False
pkg = archive_url.replace('file://', '')
return pkg
def _extract(self, base, tar_url, dest_dir):
ext = '.tar.noidea'
clean_tar = self.config['clean_tar']
for i in [('tar.%s' % e) for e in ('bz2', 'gz', 'lzma', 'xz', 'Z')] \
+ ['tgz', 'tbz', 'taz', 'tlz', 'txz']:
candidate_ext = '.%s' % i
if self.src.endswith(candidate_ext):
ext = candidate_ext
break
if 'file://' not in tar_url:
# setup the ssl-fetch output map
connector_output = {
'info': self.output.debug,
'error': self.output.error,
'kwargs-info': {'level': 2},
'kwargs-error':{'level': None},
}
fetcher = Connector(connector_output, self.proxies, USERAGENT)
success, tar, timestamp = fetcher.fetch_content(tar_url)
pkg = path([base, self.parent.name + ext])
try:
with fileopen(pkg, 'w+b') as out_file:
out_file.write(tar)
except Exception as error:
raise Exception('Failed to store tar package in '
+ pkg + '\nError was:' + str(error))
else:
clean_tar = False
pkg = tar_url.replace('file://', '')
# tar -v -x -f SOURCE -C TARGET
args = ['-v', '-x', '-f', pkg, '-C', dest_dir]
result = self.run_command(self.command(), args, cmd=self.type)
if clean_tar:
os.unlink(pkg)
return result
def _fetch(self, base, archive_url, dest_dir):
'''
Fetches overlay source archive.
@params base: string of directory base for installed overlays.
@params archive_url: string of URL where archive is located.
@params dest_dir: string of destination of extracted archive.
@rtype tuple (str of package location, bool to clean_archive)
'''
ext = self.get_extension()
if 'file://' not in archive_url:
# set up ssl-fetch output map
connector_output = {
'info': self.output.debug,
'error': self.output.error,
'kwargs-info': {'level': 5},
'kwargs-debug': {'level': 2},
'kwargs-error': {'level': None},
}
fetcher = Connector(connector_output, self.proxies, USERAGENT)
success, archive, timestamp = fetcher.fetch_content(archive_url)
pkg = path([base, self.parent.name + ext])
try:
with fileopen(pkg, 'w+b') as out_file:
out_file.write(archive)
except Exception as error:
raise Exception('Failed to store archive package in '\
'%(pkg)s\nError was: %(error)s'\
% ({'pkg': pkg, 'error': error}))
else:
self.clean_archive = False
pkg = archive_url.replace('file://', '')
return pkg
def getlist(self, parser, url):
"""
Uses the supplied parser to get a list of urls.
Takes a parser object, url, and filering options.
"""
self.output.write('getlist(): fetching ' + url + '\n', 2)
self.output.print_info('Downloading a list of mirrors...\n')
# setup the ssl-fetch ouptut map
connector_output = {
'info': self.output.write,
'debug': self.output.write,
'error': self.output.print_err,
'kwargs-info': {
'level': 2
},
'kwargs-debug': {
'level': 2
},
'kwargs-error': {
'level': 0
},
}
fetcher = Connector(connector_output, self.proxies, USERAGENT)
success, mirrorlist, timestamp = fetcher.fetch_content(url, climit=60)
parser.parse(mirrorlist)
if (not mirrorlist) or len(parser.tuples()) == 0:
self.output.print_err('Could not get mirror list. '
'Check your internet connection.')
self.output.write(' Got %d mirrors.\n' % len(parser.tuples()))
return parser.tuples()
def cache(self):
'''
Copy the remote overlay list to the local cache.
>>> import tempfile
>>> here = os.path.dirname(os.path.realpath(__file__))
>>> tmpdir = tempfile.mkdtemp(prefix="laymantmp_")
>>> cache = os.path.join(tmpdir, 'cache')
>>> myoptions = {'overlays' :
... ['file://' + here + '/tests/testfiles/global-overlays.xml'],
... 'cache' : cache,
... 'nocheck' : 'yes',
... 'proxy' : None}
>>> from layman.config import OptionConfig
>>> config = OptionConfig(myoptions)
>>> config.set_option('quietness', 3)
>>> a = RemoteDB(config)
>>> a.cache()
(True, True)
>>> b = fileopen(a.filepath(config['overlays'])+'.xml')
>>> b.readlines()[24]
' A collection of ebuilds from Gunnar Wrobel [[email protected]].\\n'
>>> b.close()
>>> os.unlink(a.filepath(config['overlays'])+'.xml')
>>> a.overlays.keys()
['wrobel', 'wrobel-stable']
>>> import shutil
>>> shutil.rmtree(tmpdir)
'''
has_updates = False
self._create_storage(self.config['storage'])
# succeeded reset when a failure is detected
succeeded = True
url_lists = [self.urls, self.detached_urls, self.signed_urls]
need_gpg = [False, True, True]
# setup the ssl-fetch output map
connector_output = {
'info': self.output.debug,
'error': self.output.error,
'kwargs-info': {'level': 2},
'kwargs-error':{'level': None},
}
fetcher = Connector(connector_output, self.proxies, USERAGENT)
for index in range(0, 3):
self.output.debug("RemoteDB.cache() index = %s" %str(index), 2)
urls = url_lists[index]
if need_gpg[index] and len(urls) and self.gpg is None:
#initialize our gpg instance
self.init_gpg()
# main working loop
for url in urls:
sig = ''
self.output.debug("RemoteDB.cache() url = %s is a tuple=%s"
%(str(url), str(isinstance(url, tuple))), 2)
filepath, mpath, tpath, sig = self._paths(url)
if 'file://' in url:
success, olist, timestamp = self._fetch_file(
url, mpath, tpath)
elif sig:
success, olist, timestamp = fetcher.fetch_content(
url[0], tpath)
else:
success, olist, timestamp = fetcher.fetch_content(
url, tpath)
if not success:
#succeeded = False
continue
self.output.debug("RemoteDB.cache() len(olist) = %s"
% str(len(olist)), 2)
# GPG handling
if need_gpg[index]:
olist, verified = self.verify_gpg(url, sig, olist)
if not verified:
self.output.debug("RemoteDB.cache() gpg returned "
"verified = %s" %str(verified), 2)
succeeded = False
filename = os.path.join(self.config['storage'],
"Failed-to-verify-sig")
self.write_cache(olist, filename)
continue
# Before we overwrite the old cache, check that the downloaded
# file is intact and can be parsed
if isinstance(url, tuple):
olist = self._check_download(olist, url[0])
else:
olist = self._check_download(olist, url)
# Ok, now we can overwrite the old cache
has_updates = max(has_updates,
self.write_cache(olist, mpath, tpath, timestamp))
self.output.debug("RemoteDB.cache() self.urls: has_updates, "
"succeeded %s, %s" % (str(has_updates), str(succeeded)), 4)
return has_updates, succeeded
def cache(self):
'''
Copy the remote overlay list to the local cache.
@rtype tuple: reflects whether the cache has updates and whether or not
the cache retrieval was successful.
'''
has_updates = False
self._create_storage(self.config['storage'])
# succeeded reset when a failure is detected
succeeded = True
url_lists = [self.urls, self.detached_urls, self.signed_urls]
need_gpg = [False, True, True]
# setup the ssl-fetch output map
connector_output = {
'info': self.output.info,
'debug': self.output.debug,
'error': self.output.error,
'exception': self.output.error,
# we want any warnings to be printed to the terminal
# so assign it to output.info with a lower noise level
'warning': self.output.info,
'kwargs-exception': {
'level': None
},
'kwargs-info': {
'level': 5
},
'kwargs-debug': {
'level': 2
},
'kwargs-error': {
'level': None
},
'kwargs-warning': {
'level': 2
},
}
fetcher = Connector(connector_output, self.proxies, USERAGENT)
for index in range(0, 3):
self.output.debug("RemoteDB.cache() index = %s" % str(index), 2)
urls = url_lists[index]
if need_gpg[index] and len(urls) and self.gpg is None:
#initialize our gpg instance
self.init_gpg()
# main working loop
for url in urls:
sig = ''
self.output.debug(
"RemoteDB.cache() url = %s is a tuple=%s" %
(str(url), str(isinstance(url, tuple))), 2)
filepath, mpath, tpath, sig = self._paths(url)
if 'file://' in url:
success, olist, timestamp = self._fetch_file(
url, mpath, tpath)
elif sig:
success, olist, timestamp = fetcher.fetch_content(
url[0], tpath, climit=60)
else:
success, olist, timestamp = fetcher.fetch_content(
url, tpath, climit=60)
if not success:
#succeeded = False
continue
self.output.debug(
"RemoteDB.cache() len(olist) = %s" % str(len(olist)), 2)
# GPG handling
if need_gpg[index]:
olist, verified = self.verify_gpg(url, sig, olist)
if not verified:
self.output.debug(
"RemoteDB.cache() gpg returned "
"verified = %s" % str(verified), 2)
succeeded = False
filename = os.path.join(self.config['storage'],
"Failed-to-verify-sig")
self.write_cache(olist, filename)
continue
# Before we overwrite the old cache, check that the downloaded
# file is intact and can be parsed
if isinstance(url, tuple):
olist = self._check_download(olist, url[0])
else:
olist = self._check_download(olist, url)
# Ok, now we can overwrite the old cache
has_updates = max(
has_updates,
self.write_cache(olist, mpath, tpath, timestamp))
self.output.debug(
"RemoteDB.cache() self.urls: has_updates, "
"succeeded %s, %s" % (str(has_updates), str(succeeded)), 4)
return has_updates, succeeded
def cache(self):
'''
Copy the remote overlay list to the local cache.
@rtype tuple: reflects whether the cache has updates and whether or not
the cache retrieval was successful.
'''
has_updates = False
self._create_storage(self.config['storage'])
# succeeded reset when a failure is detected
succeeded = True
url_lists = [self.urls, self.detached_urls, self.signed_urls]
need_gpg = [False, True, True]
# setup the ssl-fetch output map
connector_output = {
'info': self.output.debug,
'error': self.output.error,
'kwargs-info': {'level': 2},
'kwargs-error':{'level': None},
}
fetcher = Connector(connector_output, self.proxies, USERAGENT)
for index in range(0, 3):
self.output.debug("RemoteDB.cache() index = %s" %str(index), 2)
urls = url_lists[index]
if need_gpg[index] and len(urls) and self.gpg is None:
#initialize our gpg instance
self.init_gpg()
# main working loop
for url in urls:
sig = ''
self.output.debug("RemoteDB.cache() url = %s is a tuple=%s"
%(str(url), str(isinstance(url, tuple))), 2)
filepath, mpath, tpath, sig = self._paths(url)
if 'file://' in url:
success, olist, timestamp = self._fetch_file(
url, mpath, tpath)
elif sig:
success, olist, timestamp = fetcher.fetch_content(
url[0], tpath)
else:
success, olist, timestamp = fetcher.fetch_content(
url, tpath)
if not success:
#succeeded = False
continue
self.output.debug("RemoteDB.cache() len(olist) = %s"
% str(len(olist)), 2)
# GPG handling
if need_gpg[index]:
olist, verified = self.verify_gpg(url, sig, olist)
if not verified:
self.output.debug("RemoteDB.cache() gpg returned "
"verified = %s" %str(verified), 2)
succeeded = False
filename = os.path.join(self.config['storage'],
"Failed-to-verify-sig")
self.write_cache(olist, filename)
continue
# Before we overwrite the old cache, check that the downloaded
# file is intact and can be parsed
if isinstance(url, tuple):
olist = self._check_download(olist, url[0])
else:
olist = self._check_download(olist, url)
# Ok, now we can overwrite the old cache
has_updates = max(has_updates,
self.write_cache(olist, mpath, tpath, timestamp))
self.output.debug("RemoteDB.cache() self.urls: has_updates, "
"succeeded %s, %s" % (str(has_updates), str(succeeded)), 4)
return has_updates, succeeded
