def fetch_relic_if_not_exists(req, channel, index, relic_name, upstream):
indexobj = fetch_index_from_names(channel, index)
if not indexobj:
return
relics = DBSession.query(Relic) \
.filter_by(index_id=indexobj.uid, name=relic_name)
# doesn't exist locally yet
if relics.count() <= 0:
# get valid paths, if there are valid paths to be had
pathcheck = validate_reliquary_location(req,
channel,
index,
relic_name=relic_name) # noqa
if type(pathcheck) == Response:
return
reliquary, relic_folder, relic_path = pathcheck
# create the channel/index if it doesn't exist
if not os.path.exists(relic_folder):
os.makedirs(relic_folder)
# fetch
resp = requests.get(upstream)
# save
with open(relic_path, 'wb') as fout:
fout.write(resp.content)
# index
with transaction.manager:
DBSession.add(
Relic(dirty=False,
index_id=indexobj.uid,
name=relic_name,
mtime=str(os.path.getmtime(relic_path)),
size=os.path.getsize(relic_path)))
def get_debian_release_data(index_id, force=False):
indexobj = DBSession.query(Index).filter_by(uid=index_id).one()
index = indexobj.name
channel = DBSession.query(Channel).filter_by(
uid=indexobj.channel_id).one().name
key = "{}-{}-release"
if not force:
try:
cacheddata = DBSession.query(FileCache).filter_by(
key=key).one_or_none()
if cacheddata:
return cacheddata.value
except:
logger.error(
"apparently there's more than one release cached file for /{}/dist/{}/Release"
.format(channel, index))
return None
else:
with transaction.manager:
DBSession.query(FileCache).filter_by(key=key).delete()
lines = []
# Suite -- indicates one of 'oldstable', 'stable', 'testing', 'unstable',
# 'experimental' with optional suffixes such as '-updates'
# for now, fixed to 'stable'
# (required)
lines.append("Suite: stable")
# Codename -- describe codename of the release
# for now, fixed to 'reliquary'
# (required)
lines.append("Codename: reliquary")
# Origin -- describe origin of the release
# for now, fixed to 'reliquary'
# (optional)
lines.append("Origin: reliquary")
# Architectures -- what are all the different architectures for packages
# being managed?
# (required)
unique_arches = get_unique_architectures_set(index_id)
arches = []
for a in unique_arches:
arches.append(a)
lines.append("Architectures: {}".format(" ".join(arches)))
# Components -- this is a fixed and static value for now
# (required)
lines.append("Components: main")
# Date -- the time the release file was created in UTC
# (required)
utcnow = "{:%a, %b %d %Y %H:%M:%S +0000}".format(
datetime.datetime.utcnow())
lines.append("Date: {}".format(utcnow))
# MD5Sum/SHA1/SHA256/SHA512 -- lists of indices and the hash and size for them
# each line contains white space separated values:
# 1. checksum in the corresponding format
# 2. size of the file
# 3. filename relative to the directory of the Release file
arches = get_unique_architectures_set(index_id)
md5sums = []
sha1s = []
sha256s = []
for arch in arches:
package = generate_debian_package_index(channel, index, arch)
packagegz = generate_debian_package_index(channel,
index,
arch,
compression='gz')
packagebz2 = generate_debian_package_index(channel,
index,
arch,
compression='bz2')
release = generate_debian_arch_release_index(arch)
# uncompressed
if not package:
logger.error(
"Failed to get package details for debian/{}/dist/{}/[In]Release ({})"
.format(channel, index, arch))
else:
md5sums.append(" {} {} {}".format(
package[3],
str(package[2]).rjust(15),
"main/binary-{}/Packages".format(arch)))
sha1s.append(" {} {} {}".format(
package[4],
str(package[2]).rjust(15),
"main/binary-{}/Packages".format(arch)))
sha256s.append(" {} {} {}".format(
package[5],
str(package[2]).rjust(15),
"main/binary-{}/Packages".format(arch)))
# gz
if not packagegz:
logger.error(
"Failed to get package.gz details for debian/{}/dist/{}/[In]Release ({})"
.format(channel, index, arch))
else:
md5sums.append(" {} {} {}".format(
packagegz[3],
str(packagegz[2]).rjust(15),
"main/binary-{}/Packages.gz".format(arch)))
sha1s.append(" {} {} {}".format(
packagegz[4],
str(packagegz[2]).rjust(15),
"main/binary-{}/Packages.gz".format(arch)))
sha256s.append(" {} {} {}".format(
packagegz[5],
str(packagegz[2]).rjust(15),
"main/binary-{}/Packages.gz".format(arch)))
# bz2
if not packagebz2:
logger.error(
"Failed to get package.bz2 details for debian/{}/dist/{}/[In]Release ({})"
.format(channel, index, arch))
else:
md5sums.append(" {} {} {}".format(
packagebz2[3],
str(packagebz2[2]).rjust(15),
"main/binary-{}/Packages.bz2".format(arch)))
sha1s.append(" {} {} {}".format(
packagebz2[4],
str(packagebz2[2]).rjust(15),
"main/binary-{}/Packages.bz2".format(arch)))
sha256s.append(" {} {} {}".format(
packagebz2[5],
str(packagebz2[2]).rjust(15),
"main/binary-{}/Packages.bz2".format(arch)))
# release
if not release:
logger.error(
"Failed to get release details for debian/{}/dist/{}/[In]Release ({})"
.format(channel, index, arch))
else:
md5sums.append(" {} {} {}".format(
release[3],
str(release[2]).rjust(15),
"main/binary-{}/Release".format(arch)))
sha1s.append(" {} {} {}".format(
release[4],
str(release[2]).rjust(15),
"main/binary-{}/Release".format(arch)))
sha256s.append(" {} {} {}".format(
release[5],
str(release[2]).rjust(15),
"main/binary-{}/Release".format(arch)))
lines.append("MD5Sum:")
lines.append("\n".join(md5sums))
lines.append("SHA1:")
lines.append("\n".join(sha1s))
lines.append("SHA256:")
lines.append("\n".join(sha256s))
# Acquire-By-Hash -- an alternative method for clients, this is just an
# indicator for whether or not the server supports this
# for now, reliquary does not support this
lines.append("Acquire-By-Hash: no")
data = "\n".join(lines)
bytedata = data.encode()
with transaction.manager:
DBSession.add(
FileCache(
key=key,
value=bytedata,
mtime=datetime.datetime.utcnow(),
size=len(data),
md5sum=hashlib.md5(bytedata).hexdigest(),
sha1=hashlib.sha1(bytedata).hexdigest(),
sha256=hashlib.sha256(bytedata).hexdigest(),
))
return data
def generate_debian_package_index(channel,
index,
arch,
compression=None,
force=False):
basekey = '{}-{}-{}-'.format(channel, index, arch)
key = basekey
key += compression if compression else 'none'
if not force:
try:
cacheddata = DBSession.query(FileCache).filter_by(
key=key).one_or_none()
if cacheddata:
return (cacheddata.value, cacheddata.mtime, cacheddata.size,
cacheddata.md5sum, cacheddata.sha1, cacheddata.sha256)
except:
logger.error('multiple values with cache key "{}"'.format(key))
return None
else:
with transaction.manager:
DBSession.query(FileCache).filter_by(key=key).delete()
# since the request is for a compressed file, check to see if we have a cached
# uncompressed file first, and base our compressed one off of that...
# mainly so the cached files remain in sync in terms of content
if compression:
uncompressedkey = basekey + 'none'
try:
cacheddata = DBSession.query(FileCache).filter_by(
key=uncompressedkey).one_or_none()
if cacheddata:
bytedata = cacheddata.value
if compression == "gz":
finaldata = gzip.compress(bytedata)
elif compression == "bz2":
finaldata = bz2.compress(bytedata)
else:
finaldata = bytedata
mtime = datetime.datetime.utcnow()
size = len(finaldata)
md5sum = hashlib.md5(finaldata).hexdigest()
sha1 = hashlib.sha1(finaldata).hexdigest()
sha256 = hashlib.sha256(finaldata).hexdigest()
newcacheddata = FileCache(key=key,
value=finaldata,
mtime=mtime,
size=size,
md5sum=md5sum,
sha1=sha1,
sha256=sha256)
with transaction.manager:
DBSession.add(newcacheddata)
return (finaldata, mtime, size, md5sum, sha1, sha256)
except:
logger.error(
'multiple values with cache key "{}"'.format(uncompressedkey))
return None
# generate and cache Package file
lines = []
archobjs = DBSession.query(Relic, DebInfo) \
.filter(Relic.uid == DebInfo.relic_id) \
.filter(DebInfo.architecture.ilike('%{0}%'.format(arch)))
for relic, debinfo in archobjs:
# we're possibly pulling only partial matches, so this just confirms the
# selection choice
arches = [a.lower().strip() for a in debinfo.architecture.split()]
if arch not in arches:
continue
lines.append("Package: {}".format(debinfo.package))
if debinfo.source:
lines.append("Source: {}".format(debinfo.source))
lines.append("Version: {}".format(debinfo.version))
if debinfo.section:
lines.append("Section: {}".format(debinfo.section))
if debinfo.section:
lines.append("Priority: {}".format(debinfo.priority))
lines.append("Architecture: {}".format(debinfo.architecture))
if debinfo.essential:
lines.append("Essential: {}".format(debinfo.essential))
if debinfo.depends:
lines.append("Depends: {}".format(debinfo.depends))
if debinfo.recommends:
lines.append("Recommends: {}".format(debinfo.recommends))
if debinfo.suggests:
lines.append("Suggests: {}".format(debinfo.suggests))
if debinfo.enhances:
lines.append("Enhances: {}".format(debinfo.enhances))
if debinfo.pre_depends:
lines.append("Pre-Depends: {}".format(debinfo.pre_depends))
if debinfo.installed_size:
lines.append("Installed-Size: {}".format(debinfo.installed_size))
lines.append("Maintainer: {}".format(debinfo.maintainer))
lines.append("Description: {}".format(debinfo.description))
if debinfo.homepage:
lines.append("Homepage: {}".format(debinfo.homepage))
if debinfo.built_using:
lines.append("Built-Using: {}".format(debinfo.built_using))
lines.append("Filename: {}".format(debinfo.filename))
lines.append("Size: {}".format(relic.size))
lines.append("MD5Sum: {}".format(debinfo.md5sum))
lines.append("SHA1: {}".format(debinfo.sha1))
lines.append("SHA256: {}".format(debinfo.sha256))
lines.append("SHA512: {}".format(debinfo.sha512))
lines.append("Description-md5: {}".format(debinfo.description_md5))
if debinfo.multi_arch:
lines.append("Multi-Arch: {}".format(debinfo.multi_arch))
lines.append("")
data = "\n".join(lines)
bytedata = data.encode()
if compression == "gz":
finaldata = gzip.compress(bytedata)
elif compression == "bz2":
finaldata = bz2.compress(bytedata)
else:
finaldata = bytedata
mtime = datetime.datetime.utcnow()
size = len(finaldata)
md5sum = hashlib.md5(finaldata).hexdigest()
sha1 = hashlib.sha1(finaldata).hexdigest()
sha256 = hashlib.sha256(finaldata).hexdigest()
newcacheddata = FileCache(key=key,
value=finaldata,
mtime=mtime,
size=size,
md5sum=md5sum,
sha1=sha1,
sha256=sha256)
with transaction.manager:
DBSession.add(newcacheddata)
return (finaldata, mtime, size, md5sum, sha1, sha256)
def index_deb_info(name, path, obj, indexname):
# relative to the repository root, which would be something
# like /api/v1/{channel}/
filename = "pool/{}/{}".format(indexname, name)
# md5, sha1, and sha256 of file
blocksize = 65536
md5sum_hasher = hashlib.md5()
sha1_hasher = hashlib.sha1()
sha256_hasher = hashlib.sha256()
sha512_hasher = hashlib.sha512()
with open(path, 'rb') as fin:
buf = fin.read(blocksize)
while len(buf) > 0:
md5sum_hasher.update(buf)
sha1_hasher.update(buf)
sha256_hasher.update(buf)
sha512_hasher.update(buf)
buf = fin.read(blocksize)
md5sum = md5sum_hasher.hexdigest()
sha1 = sha1_hasher.hexdigest()
sha256 = sha256_hasher.hexdigest()
sha512 = sha512_hasher.hexdigest()
# the rest is ripped out of the .deb file or generated based
# on the information there.
deb = debfile.DebFile(path)
control = deb.control.debcontrol()
# deb control is dict-like object where key lookups are case-insensitive
multi_arch = control.get("multi-arch", None)
package = control.get("package", None)
source = control.get("source", None)
version = control.get("version", None)
section = control.get("section", None)
priority = control.get("priority", None)
architecture = control.get("architecture", None)
essential = control.get("essential", None)
depends = control.get("depends", None)
recommends = control.get("recommends", None)
suggests = control.get("suggests", None)
enhances = control.get("enhances", None)
pre_depends = control.get("pre-depends", None)
installed_size = control.get("installed-size", None)
maintainer = control.get("maintainer", None)
description = control.get("description", None)
description_md5 = control.get("description-md5", None)
homepage = control.get("homepage", None)
built_using = control.get("built_using", None)
# if the description-md5 wasn't specified, comput it!
# the computed value starts at the second character after the colon in the
# control file (basically, allowing the 'Header: ' format of the text file)
# and includes a trailing newline character. The value must be lowercase
# hex md5.
if not description_md5:
if description[-1] == "\n":
description_md5 = hashlib.md5(description.encode()).hexdigest()
else:
description_md5 = hashlib.md5((description+"\n").encode()).hexdigest()
# missing required fields are a deal breaker for including this package
# in the index
msg = name+" skipped for deb info: '{}' not found in control"
if not package:
logger.error(msg.format('Package'))
return
if not version:
logger.error(msg.format('Version'))
return
if not architecture:
logger.error(msg.format('Architecture'))
return
if not maintainer:
logger.error(msg.format('Maintainer'))
return
if not description:
logger.error(msg.format('Description'))
return
kwargs = dict(
filename=filename,
md5sum=md5sum,
sha1=sha1,
sha256=sha256,
sha512=sha512,
multi_arch=multi_arch,
package=package,
source=source,
version=version,
section=section,
priority=priority,
architecture=architecture,
essential=essential,
depends=depends,
recommends=recommends,
suggests=suggests,
enhances=enhances,
pre_depends=pre_depends,
installed_size=installed_size,
maintainer=maintainer,
description=description,
description_md5=description_md5,
homepage=homepage,
built_using=built_using)
try:
debinfo_dbobj = DBSession.query(DebInfo) \
.filter_by(relic_id=obj.uid) \
.one_or_none()
with transaction.manager:
if debinfo_dbobj:
logger.info("Adding deb info for " + name)
DBSession.query(DebInfo) \
.filter_by(uid=debinfo_dbobj.uid) \
.update(kwargs)
else:
logger.info("Updating deb info for " + name)
kwargs['relic_id'] = obj.uid
DBSession.add(DebInfo(**kwargs))
except MultipleResultsFound:
logger.error("Apparently there's more than one debinfo object"
"associated with '"+obj.name+"'")
def reindex():
description = """\
Reindex reliquary storage.
"""
usage = "usage: %prog config_uri"
parser = optparse.OptionParser(
usage=usage,
description=textwrap.dedent(description)
)
options, args = parser.parse_args(sys.argv[1:])
if not len(args) > 0:
logger.error('at least the config uri is needed')
return 2
config_uri = args[0]
env = bootstrap(config_uri)
settings, closer = env['registry'].settings, env['closer']
try:
# mark everything as dirty so we can delete anything that
# is not clean by the end of the reindex
with transaction.manager:
DBSession.query(Channel).update({'dirty': True})
DBSession.query(Index).update({'dirty': True})
DBSession.query(Relic).update({'dirty': True})
# now, walk through the reliquary and index relics
reliquary = settings.get('reliquary.location', None)
# ### CHANNEL
for channel in os.listdir(reliquary):
# make sure the directory exists
channel_path = os.path.join(reliquary, channel)
if not os.path.isdir(channel_path):
continue
# make sure the db object exists
channel_dbobj = DBSession.query(Channel) \
.filter_by(name=channel) \
.first()
if not channel_dbobj:
channel_dbobj = Channel(dirty=False, name=channel)
with transaction.manager:
DBSession.add(channel_dbobj)
else:
# channel is has real path and db entry, so should not be dirty
with transaction.manager:
DBSession.query(Channel) \
.filter_by(uid=channel_dbobj.uid) \
.update({'dirty': False})
# ### INDEX
for index in os.listdir(channel_path):
# make sure the directory exists
index_path = os.path.join(channel_path, index)
if not os.path.isdir(index_path):
continue
# make sure the db object exists
index_dbobj = DBSession.query(Index) \
.filter_by(channel_id=channel_dbobj.uid, name=index) \
.first()
if not index_dbobj:
index_dbobj = Index(dirty=False,
name=index,
channel_id=channel_dbobj.uid)
with transaction.manager:
DBSession.add(index_dbobj)
else:
# index has real path and db entry, so should not be dirty
with transaction.manager:
DBSession.query(Index).filter_by(uid=index_dbobj.uid) \
.update({'dirty': False})
# ### RELIC
for relic in os.listdir(index_path):
relic_path = os.path.join(index_path, relic)
try:
relic_dbobj = DBSession.query(Relic) \
.filter_by(index_id=index_dbobj.uid, name=relic) \
.one_or_none()
relic_mtime = str(os.path.getmtime(relic_path))
relic_size = os.path.getsize(relic_path)
if relic_dbobj:
with transaction.manager:
DBSession.query(Relic) \
.filter_by(uid=relic_dbobj.uid) \
.update({'dirty': False,
'mtime': relic_mtime,
'size': relic_size})
else:
relic_dbobj = Relic(dirty=False,
index_id=index_dbobj.uid,
name=relic,
mtime=relic_mtime,
size=relic_size)
with transaction.manager:
DBSession.add(relic_dbobj)
# if the relic is a debian archive, there's additional
# info that should be pulled out to make generating
# a deb repo more efficient
if relic[-4:] == ".deb":
index_deb_info(relic, relic_path, relic_dbobj, index)
except MultipleResultsFound:
logger.error('index [{}/{}/{}] contains non-unique '
'/channel/index/relic_name'
.format(channel, index, relic))
# delete all relics, still dirty, from the index
with transaction.manager:
DBSession.query(Channel) \
.filter_by(dirty=True) \
.delete()
DBSession.query(Index) \
.filter_by(dirty=True) \
.delete()
DBSession.query(Relic) \
.filter_by(dirty=True) \
.delete(synchronize_session=False)
pregenerate_deb_indices()
except Exception as ex:
logger.critical("something went wrong")
logger.critical(ex)
finally:
closer()