def _add_deb(changes, filename):
d = DebFile(filename)
changes['Files'].append({
'name': filename.basename(),
'size': filename.getsize(),
'md5sum': filename.read_hash('md5').encode('hex'),
'section': d.debcontrol()['Section'],
'priority': d.debcontrol()['Priority'],
})
changes['Checksums-Sha1'].append({
'name':
filename.basename(),
'size':
filename.getsize(),
'sha1':
filename.read_hash('sha1').encode('hex')
})
changes['Checksums-Sha256'].append({
'name':
filename.basename(),
'size':
filename.getsize(),
'sha256':
filename.read_hash('sha256').encode('hex')
})
def add_package_to_cache(config, cache, codename, component, package):
distro = get_distro(config, codename)
if component not in distro["Components"]:
raise RuntimeError(
"Can not add package %s, component %s not supported by the repository"
% (package, component))
deb = DebFile(filename=package)
fields = deb.debcontrol()
arch = fields["Architecture"]
if arch not in distro["Architectures"]:
raise RuntimeError(
"Can not add package %s, architecture not supported by the repository"
% package)
fields["SHA256"] = hash_file(package, hashlib.sha256)
fields["SHA1"] = hash_file(package, hashlib.sha1)
fields["MD5sum"] = hash_file(package, hashlib.md5)
fields["Size"] = str(os.path.getsize(package))
fields["Filename"] = "pool/%s/%s" % (component, os.path.basename(package))
packages = get_package_versions(cache, codename, component, arch,
fields["Package"])
if fields["Version"] in packages:
raise RuntimeError(
"Package %s with version %s already exists in the repository" %
(fields["Package"], fields["version"]))
# convert dict like object to tuple to keep original sorting in json file
packages[fields["Version"]] = [(field, fields[field]) for field in fields]
print("-> Adding %s=%s to cache for %s/%s" %
(fields["Package"], fields["Version"], codename, arch))
return fields["Filename"]
def generate_sut_from_deb(path):
""" Generate a Firehose SUT from a .deb file. """
obj = DebFile(filename=path, mode='r')
control = obj.debcontrol()
version = control['Version']
version, local = parse_version(version)
name, arch = [control[x] for x in ['Package', 'Architecture']]
return DebianBinary(name, version, local, arch)
def scan_packages(filenames):
"""Parse names and versions from .deb files.
Returns a list of `Package`.
"""
packages = []
for filename in filenames:
deb = DebFile(filename)
control = deb.debcontrol()
packages.append(Package.from_control(deb.debcontrol()))
return packages
def from_deb(cls, deb_file_path):
"""Create a FetchedPackage from a binary package on disk."""
debcontrol = DebFile(deb_file_path).control.debcontrol()
name = debcontrol['Package']
version = debcontrol['Version']
filename = os.path.basename(deb_file_path)
size = os.path.getsize(deb_file_path)
md5sum = hashlib.md5(open(deb_file_path).read()).hexdigest()
sha256sum = hashlib.sha256(open(deb_file_path).read()).hexdigest()
architecture = debcontrol['Architecture']
depends = debcontrol.get('Depends')
pre_depends = debcontrol.get('Pre-Depends')
multi_arch = debcontrol.get('Multi-Arch')
conflicts = debcontrol.get('Conflicts')
recommends = debcontrol.get('Recommends')
provides = debcontrol.get('Provides')
replaces = debcontrol.get('Replaces')
breaks = debcontrol.get('Breaks')
pkg = cls(
name, version, filename, size, md5sum, sha256sum, architecture,
depends, pre_depends, multi_arch, conflicts, recommends, provides,
replaces, breaks)
pkg.content = open(deb_file_path)
pkg._file_path = deb_file_path
return pkg
def getBinFromDeb(fname):
"""
Get a binary package from a .deb file
"""
dfile = DebFile(fname)
cdict = dfile.debcontrol()
result = BinPackageDeb(
id=-1,
name=cdict['Package'],
control=dfile.control.get_content('control', 'utf-8'),
cdict=dfile.debcontrol(),
Version=cdict['Version'],
Architecture=cdict['Architecture'],
udeb=fname.endswith('.udeb'),
Description_md5=hashlib.md5(cdict['Description'].encode() +
b'\n').hexdigest(),
debfile=dfile,
origfile=fname)
result.__dict__.update(utils.Hasher.hash(fname)._asdict())
return result
def _add_deb(changes, filename):
d = DebFile(filename)
changes['Files'].append({
'name': filename.basename(),
'size': filename.getsize(),
'md5sum': filename.read_hash('md5').encode('hex'),
'section': d.debcontrol()['Section'],
'priority': d.debcontrol()['Priority'],
})
changes['Checksums-Sha1'].append({
'name': filename.basename(),
'size': filename.getsize(),
'sha1': filename.read_hash('sha1').encode('hex')
})
changes['Checksums-Sha256'].append({
'name': filename.basename(),
'size': filename.getsize(),
'sha256': filename.read_hash('sha256').encode('hex')
})
def from_deb(cls, deb_file_path):
"""Create a FetchedPackage from a binary package on disk."""
debcontrol = DebFile(deb_file_path).control.debcontrol()
name = debcontrol['Package']
version = debcontrol['Version']
filename = os.path.basename(deb_file_path)
size = os.path.getsize(deb_file_path)
md5sum = hashlib.md5(open(deb_file_path).read()).hexdigest()
architecture = debcontrol['Architecture']
depends = debcontrol.get('Depends')
pre_depends = debcontrol.get('Pre-Depends')
multi_arch = debcontrol.get('Multi-Arch')
conflicts = debcontrol.get('Conflicts')
recommends = debcontrol.get('Recommends')
provides = debcontrol.get('Provides')
replaces = debcontrol.get('Replaces')
breaks = debcontrol.get('Breaks')
pkg = cls(name, version, filename, size, md5sum, architecture, depends,
pre_depends, multi_arch, conflicts, recommends, provides,
replaces, breaks)
pkg.content = open(deb_file_path)
pkg._file_path = deb_file_path
return pkg
def migrate(*args, **kwargs):
"""
Add relationship fields (breaks/depends/etc)
"""
warnings_encountered = False
collection = connection.get_collection('units_deb')
for unit in collection.find({}):
unit_id = unit['_id']
path = unit['_storage_path']
if not os.path.exists(path):
warnings_encountered = True
msg = 'deb package file corresponding to db_unit with _id = {}\n'\
'was not found at _storage_path = {}.\n'\
'The unit was not migrated!'.format(unit_id, path)
_logger.warn(msg)
continue
try:
control_fields = DebFile(path).debcontrol()
except ArError as error:
warnings_encountered = True
msg = 'deb package file corresponding to db_unit with _id = {}\n'\
'with _storage_path = {}\n'\
'was not recognized as a valid deb file:\n'\
'{}\n'\
'The unit was not migrated!'.format(unit_id,
path,
str(error),)
_logger.warn(msg)
continue
update_dict = dict()
for field, deb_key in REL_FIELDS_MAP.iteritems():
if deb_key in control_fields:
update_dict[field] = DependencyParser.from_string(
control_fields[deb_key])
else:
update_dict[field] = []
collection.update_one(dict(_id=unit_id), {'$set': update_dict})
if warnings_encountered:
msg = 'Warnings were encountered during the db migration!\n'\
'Check the logs for more information, and consider deleting broken units.'
_logger.warn(msg)
def analyze(debfile, package="?", group="?", show_errors=False):
deb = DebFile(filename=debfile)
tgz = deb.data.tgz()
if not os.path.exists(debfile):
# print >> sys.stderr, "%s doesn't exists!" % debfile
pass
if not debfile.endswith(".deb"):
return
output = {}
output["package"] = package
output["group"] = group
output["build"] = os.path.basename(debfile)
output["files"] = []
output["daemon"] = False
flag = False
directory = False
for entry in tgz.getmembers():
size = entry.size
# check if package is a daemon
if "/etc/rc.d/init.d" in entry.name or "/lib/systemd" in entry.name:
output["daemon"] = True
# skip 0 byte files only
if size == 0 and not stat.S_ISDIR(entry.mode):
continue
# we are only interested in particular kind of directories
if stat.S_ISDIR(entry.mode):
if not ((entry.mode & stat.S_ISUID) or
(stat.S_ISGID & entry.mode)):
continue
else:
flag = True
directory = True
if not entry.mode & 0o111:
continue
# always report setuid files
if ((entry.mode & stat.S_ISUID) or (stat.S_ISGID & entry.mode)):
flag = True
# skip library files
filename = entry.name.lstrip(".")
if ("lib" in filename and ".so" in filename) or \
filename.endswith(".so"):
continue
try:
contents = tgz.extractfile(entry).read()
except Exception as exc:
print(exc)
# invoke checksec
returncode = -1
try:
fh = BytesIO(contents)
elf = Elf(fh)
out = process_file(elf)
returncode = 0
dataline = "%s,%s,%s,%s" % (package, os.path.basename(debfile),
filename, out)
except ELFError as exc:
continue
except IOError as exc:
continue
# print p.returncode, filename
if returncode == 0 or flag:
# populate fileinfo object
fileinfo = {}
fileinfo["name"] = filename
fileinfo["size"] = entry.size
fileinfo["mode"] = entry.mode
if directory:
fileinfo["directory"] = directory
output["files"].append(fileinfo)
if returncode == 0 and opformat == "csv":
print(dataline)
if returncode == 0 and opformat == "json":
try:
for kvp in out.rstrip().split(","):
key, value = kvp.split("=")
fileinfo[key] = value
except Exception:
pass
if opformat == "json":
print(
json.dumps(output,
sort_keys=True,
indent=4,
separators=(',', ': ')))
git_refs = git_repo.remotes.origin.refs
git_refs_name = list(map(lambda x: str(x).split('/')[-1], git_refs))
logging.debug(git_refs_name)
if gh_branch not in git_refs_name:
git_repo.git.checkout(b=gh_branch)
else:
git_repo.git.checkout(gh_branch)
# Generate metadata
logging.debug("cwd : {}".format(os.getcwd()))
logging.debug(os.listdir())
deb_file_handle = DebFile(filename=deb_file_path)
deb_file_control = deb_file_handle.debcontrol()
current_metadata = {
'format_version': 1,
'sw_version': deb_file_control['Version'],
'sw_architecture': deb_file_control['Architecture'],
'linux_version': deb_file_version
}
current_metadata_str = json.dumps(current_metadata)
logging.debug('Metadata {}'.format(current_metadata_str))
# Get metadata
all_commit = git_repo.iter_commits(gh_branch)
all_apt_action_commit = list(filter(lambda x: (x.message[:12] == '[apt-action]'), all_commit))
def migrate(*args, **kwargs):
"""
This migration should achieve the following:
* All existing entries in the units_deb collection should store parsed
relationship information in any relevant REL_FIELDS. (These fields should
never be used for plain Debian relationship strings).
* The control_file field is added to all entries in the units_deb collection.
This field should contain a dict, containing all control file fields
associated with the relevant .deb package. This way we can ensure the db
contains all control file information needed to publish a repo containing
the unit (including any plain Debian relationship strings).
"""
warnings_encountered = False
deb_collection = connection.get_collection('units_deb')
deb_count = deb_collection.count()
with utils.MigrationProgressLog('Deb Package', deb_count) as progress_log:
for deb_package in deb_collection.find({}).batch_size(100):
storage_path = deb_package['_storage_path']
package_id = deb_package['_id']
if not os.path.exists(storage_path):
warnings_encountered = True
msg = 'deb package file corresponding to db_unit with _id = {}\n'\
'was not found at _storage_path = {}.\n'\
'The unit was not migrated!'.format(package_id, storage_path)
_logger.warn(msg)
continue
try:
control_fields = DebFile(storage_path).debcontrol()
except ArError as error:
warnings_encountered = True
msg = 'deb package file corresponding to db_unit with _id = {}\n'\
'with _storage_path = {}\n'\
'was not recognized as a valid deb file:\n'\
'{}\n'\
'The unit was not migrated!'.format(package_id,
storage_path,
str(error),)
_logger.warn(msg)
continue
new_fields = dict()
remove_fields = dict()
new_fields.update(control_fields=control_fields)
# Add parsed relational fields and remove empty relational fields:
for field, deb_key in REL_FIELDS_MAP.iteritems():
if deb_key in control_fields:
new_fields[field] = DependencyParser.from_string(
control_fields[deb_key])
else:
remove_fields[field] = ''
# Also (re)add any fields that may have been previously ignored:
for field, deb_key in IGNORED_FIELDS_MAP.iteritems():
if deb_key in control_fields:
new_fields[field] = control_fields[deb_key]
deb_collection.update_one(
{'_id': package_id},
{'$set': new_fields},
)
deb_collection.update_one(
{'_id': package_id},
{'$unset': remove_fields},
)
progress_log.progress()
if warnings_encountered:
msg = 'Warnings were encountered during the db migration!\n'\
'Check the logs for more information, and consider deleting broken units.'
_logger.warn(msg)
