def run(args):
from jinja2 import Environment, PackageLoader, FileSystemLoader
mirror2name = {}
mirror2url = {}
code2relname = dict([(r, cfg.get('release names', r))
for r in cfg.options('release names')
if not r == 'data'])
if cfg.has_section('mirror names'):
mirror2name = dict([(m, codecs.decode(cfg.get('mirror names', m), 'utf-8'))
for m in cfg.options('mirrors')])
if cfg.has_section('mirrors'):
mirror2url = dict([(m, cfg.get('mirrors', m))
for m in cfg.options('mirrors')])
if not args.template is None:
templ_dir = os.path.dirname(args.template)
templ_basename = os.path.basename(args.template)
jinja_env = Environment(loader=FileSystemLoader(templ_dir))
srclist_template = jinja_env.get_template(templ_basename)
else:
jinja_env = Environment(loader=PackageLoader('bigmess'))
srclist_template = jinja_env.get_template('sources_lists.rst')
print codecs.encode(
srclist_template.render(code2name=code2relname,
mirror2name=mirror2name,
mirror2url=mirror2url),
'utf-8')
def _find_release_origin_archive(cfg, release):
# available
origins = []
for origin in cfg.options('release bases'):
archive = cfg.get('release bases', origin)
if not archive:
continue
url = '%s/dists/%s/Release' % (archive, release)
try:
urip = urllib2.urlopen(url)
info = urip.info()
origins.append(archive)
except urllib2.HTTPError:
lgr.debug("No '%s'" % url)
except urllib2.URLError:
lgr.debug("Can't connect to'%s'" % url)
if len(origins) == 0:
lgr.info("Found no origin for %r. Assuming it originates here."
% release)
return None
elif len(origins) > 1:
lgr.warning("More than a single origin archive was found for %r: %s. "
"!Disambiguate (TODO)!" % (release, origins))
return None
return origins[0]
def run(args):
for release in cfg.options('release files'):
if release == 'data':
# no seperate list for the data archive
continue
for mirror in cfg.options('mirrors'):
for areas, tag in (('main contrib non-free', 'full'),
('main', 'libre')):
listname = '%s.%s.%s' % (release, mirror, tag)
lf = open(opj(args.dest_dir, listname), 'w')
for rel in ('data', release):
aptcfg = '%s %s %s\n' % (cfg.get('mirrors', mirror),
rel,
areas)
lf.write('deb %s' % aptcfg)
lf.write('#deb-src %s' % aptcfg)
lf.close()
def run(args):
import codecs, time, urllib2
from jinja2 import Environment, PackageLoader, FileSystemLoader
jinja_env = Environment(loader=PackageLoader('bigmess'))
template = jinja_env.get_template('mirrors_status.rst')
stampfile = cfg.get('mirrors monitor', 'stampfile', 'TIMESTAMP')
warn_threshold = cfg.getfloat('mirrors monitor', 'warn-threshold') * 3600
lgr.debug("using stampfile %(stampfile)s", locals())
mirrors_info = {}
for mirror in cfg.options('mirrors'):
mirror_url = cfg.get('mirrors', mirror)
mirror_name = cfg.get('mirror names', mirror)
age = None
age_str = None
status = "**N/A**"
try:
url = '%(mirror_url)s/%(stampfile)s' % locals()
u = urllib2.urlopen(url)
stamp = u.read()
age = (time.time() - int(stamp)) # age in hours
age_str = _literal_seconds(age)
if age > warn_threshold:
lgr.warning("Mirror %(mirror)s is %(age_str)s old", locals())
status = "**OLD**"
else:
status = "OK"
except urllib2.URLError, e:
lgr.error("Cannot fetch '%s': %s" % (url, e))
# Here ideally we should revert to use previously known state
except (TypeError, ValueError):
# int conversion has failed -- there is smth else in that file
lgr.error("Cannot assess the age. Retrieved stamp was %r" % stamp)
status = "**BRK**"
def run(args):
lgr.debug("using file cache at '%s'" % args.filecache)
# get all metadata files from the repo
meta_baseurl = cfg.get('metadata', 'source extracts baseurl',
default=None)
meta_filenames = cfg.get('metadata', 'source extracts filenames',
default='').split()
#
# Releases archives
#
releases = cfg.options('release files')
# for preventing unnecessary queries
lookupcache = {}
# ensure the cache is there
if not os.path.exists(args.filecache):
os.makedirs(args.filecache)
for release in releases:
rurl = cfg.get('release files', release)
# first get 'Release' files
dst_path = _url2filename(args.filecache, rurl)
if not _download_file(rurl, dst_path, args.force_update):
continue
baseurl = '/'.join(rurl.split('/')[:-1])
comps, archs = _proc_release_file(dst_path, baseurl)
# Fetch information on binary packages
for comp in comps:
for arch in archs:
# also get 'Packages.gz' for each component and architecture
purl = '/'.join((baseurl, comp,
'binary-%s' % arch, 'Packages.gz'))
dst_path = _url2filename(args.filecache, purl)
if not _download_file(purl, dst_path, args.force_update):
continue
# also get 'Sources.gz' for each component
surl = '/'.join((baseurl, comp, 'source', 'Sources.gz'))
dst_path = _url2filename(args.filecache, surl)
if not _download_file(surl, dst_path, args.force_update):
continue
# TODO go through the source file and try getting 'debian/upstream'
# from the referenced repo
for spkg in deb822.Sources.iter_paragraphs(gzip.open(dst_path)):
# TODO pull stuff directly form VCS
#vcsurl = spkg.get('Vcs-Browser', None)
#if vcsurl is None:
# lgr.warning("no VCS URL for '%s'" % spkg['Package'])
# continue
#print vcsurl
#http://github.com/yarikoptic/vowpal_wabbit
#->
#http://raw.github.com/yarikoptic/vowpal_wabbit/debian/debian/compat
src_name = spkg['Package']
if not len(meta_filenames) or meta_baseurl is None:
continue
lgr.debug("query metadata for source package '%s'" % src_name)
for mfn in meta_filenames:
mfurl = '/'.join((meta_baseurl, src_name, mfn))
dst_path = _url2filename(args.filecache, mfurl)
if dst_path in lookupcache:
continue
_download_file(mfurl, dst_path, args.force_update,
ignore_missing=True)
lookupcache[dst_path] = None
# Also fetch corresponding Release from the base distribution
# Figure out the base distribution based on the release description
rname = cfg.get('release names', release)
if not rname:
continue
# Look-up release bases for the release among available bases
oarchive = _find_release_origin_archive(cfg, release)
if not oarchive:
continue
obaseurl = '%s/%s' % (oarchive, '/'.join(rurl.split('/')[-3:-1]))
orurl = '%s/Release' % obaseurl
# first get 'Release' files
dst_path = _url2filename(args.filecache, orurl)
if not _download_file(orurl, dst_path, args.force_update):
continue
comps, _ = _proc_release_file(dst_path, obaseurl)
for comp in comps:
# Fetch information on source packages -- we are not interested
# to provide a thorough coverage -- just the version
osurl = '/'.join((obaseurl, comp, 'source', 'Sources.gz'))
dst_path = _url2filename(args.filecache, osurl)
if not _download_file(osurl, dst_path, args.force_update):
continue
#
# Tasks
#
tasks = cfg.options('task files')
for task in tasks:
rurl = cfg.get('task files', task)
dst_path = opj(args.filecache, 'task_%s' % task)
if not _download_file(rurl, dst_path, args.force_update):
continue
def run(args):
lgr.debug("using file cache at '%s'" % args.filecache)
# get all metadata files from the repo
meta_baseurl = cfg.get('metadata', 'source extracts baseurl',
default=None)
meta_filenames = cfg.get('metadata', 'source extracts filenames',
default='').split()
rurls = cfg.get('release files', 'urls', default='').split()
if args.init_db is None:
db = {'src': {}, 'bin': {}, 'task': {}}
else:
db = load_db(args.init_db)
srcdb = db['src']
bindb = db['bin']
taskdb = db['task']
releases = cfg.options('release files')
for release in releases:
rurl = cfg.get('release files', release)
# first 'Release' files
relf_path = _url2filename(args.filecache, rurl)
baseurl = '/'.join(rurl.split('/')[:-1])
codename, comps, archs = _proc_release_file(relf_path, baseurl)
for comp in comps:
# also get 'Sources.gz' for each component
surl = '/'.join((baseurl, comp, 'source', 'Sources.gz'))
srcf_path = _url2filename(args.filecache, surl)
for spkg in deb822.Sources.iter_paragraphs(gzip.open(srcf_path)):
src_name = spkg['Package']
sdb = srcdb.get(src_name, {})
src_version = spkg['Version']
if apt_pkg.version_compare(src_version,
sdb.get('latest_version', '')) > 0:
# this is a more recent version, so let's update all info
sdb['latest_version'] = src_version
for field in ('Homepage', 'Vcs-Browser', 'Maintainer',
'Uploaders'):
sdb[field.lower().replace('-', '_')] = spkg.get(field, '')
# record all binary packages
bins = [s.strip() for s in spkg.get('Binary', '').split(',')]
sdb['binary'] = bins
for b in bins:
if not b in bindb:
bindb[b] = {'in_release': {codename: {src_version: []}},
'src_name': src_name,
'latest_version': src_version}
else:
bindb[b]['in_release'][codename] = {src_version: []}
if apt_pkg.version_compare(
src_version, bindb[b].get('latest_version', '')) > 0:
bindb[b]['src_name'] = src_name
bindb[b]['latest_version'] = src_version
if 'upstream' in meta_filenames and not meta_baseurl is None:
import yaml
mfn = 'upstream'
mfurl = '/'.join((meta_baseurl, src_name, mfn))
mfpath = _url2filename(args.filecache, mfurl)
if os.path.exists(mfpath):
lgr.debug("import metadata for source package '%s'"
% src_name)
try:
upstream = yaml.safe_load(open(mfpath))
except yaml.scanner.ScannerError, e:
lgr.warning("Malformed upstream YAML data for '%s'"
% src_name)
lgr.debug("Caught exception was: %s" % (e,))
# uniformize structure
if 'Reference' in upstream and not isinstance(upstream['Reference'], list):
upstream['Reference'] = [upstream['Reference']]
sdb['upstream'] = upstream
sdb['component'] = comp
for mf in meta_filenames:
if os.path.exists(_url2filename(args.filecache,
'/'.join((meta_baseurl,
src_name,
mf)))):
sdb['havemeta_%s' % mf.replace('.', '_').replace('-', '_')] = True
srcdb[src_name] = sdb
for arch in archs:
# next 'Packages.gz' for each component and architecture
purl = '/'.join((baseurl, comp, 'binary-%s' % arch, 'Packages.gz'))
pkgf_path = _url2filename(args.filecache, purl)
for bpkg in deb822.Packages.iter_paragraphs(gzip.open(pkgf_path)):
bin_name = bpkg['Package']
bin_version = bpkg['Version']
try:
bin_srcname = bpkg['Source']
except KeyError:
# if a package has no source name, let's hope it is the
# same as the binary name
bin_srcname = bin_name # unused bin_srcname ???
if not bin_name in bindb:
lgr.warning("No corresponding source package for "
"binary package '%s' in [%s, %s, %s]"
% (bin_name, codename, comp, arch))
continue
try:
bindb[bin_name]['in_release'][codename][bin_version].append(arch)
except KeyError:
if not codename in bindb[bin_name]['in_release']:
# package not listed in this release?
bindb[bin_name]['in_release'][codename] = {bin_version: [arch]}
elif not bin_version in bindb[bin_name]['in_release'][codename]:
# package version not listed in this release?
bindb[bin_name]['in_release'][codename][bin_version] = [arch]
else:
raise
if apt_pkg.version_compare(
bin_version,
bindb[bin_name]['latest_version']) >= 0:
# most recent -> store description
descr = bpkg['Description'].split('\n')
bindb[bin_name]['short_description'] = descr[0].strip()
bindb[bin_name]['long_description'] = descr[1:]
brelf_path = _url2filename(args.filecache, brurl)
codename, comps, archs = _proc_release_file(brelf_path, bbaseurl)
for comp in comps:
# also get 'Sources.gz' for each component
surl = '/'.join((bbaseurl, comp, 'source', 'Sources.gz'))
srcf_path = _url2filename(args.filecache, surl)
for spkg in deb822.Sources.iter_paragraphs(gzip.open(srcf_path)):
sdb = srcdb.get(spkg['Package'], None)
if not sdb:
continue
src_version = spkg['Version']
if not 'in_base_release' in sdb:
sdb['in_base_release'] = {}
sdb['in_base_release'][codename] = src_version
tasks = cfg.options('task files')
for task in tasks:
srcf_path = opj(args.filecache, 'task_%s' % task)
for st in deb822.Packages.iter_paragraphs(open(srcf_path)):
if st.has_key('Task'):
taskdb[task] = st['Task']
continue
elif st.has_key('Depends'):
pkg = st['Depends']
elif st.has_key('Recommends'):
pkg = st['Recommends']
elif st.has_key('Suggests'):
pkg = st['Suggests']
else:
lgr.warning("Ignoring unkown stanza in taskfile: %s" % st)
continue
