def _add_deb(changes, filename): d = DebFile(filename) changes['Files'].append({ 'name': filename.basename(), 'size': filename.getsize(), 'md5sum': filename.read_hash('md5').encode('hex'), 'section': d.debcontrol()['Section'], 'priority': d.debcontrol()['Priority'], }) changes['Checksums-Sha1'].append({ 'name': filename.basename(), 'size': filename.getsize(), 'sha1': filename.read_hash('sha1').encode('hex') }) changes['Checksums-Sha256'].append({ 'name': filename.basename(), 'size': filename.getsize(), 'sha256': filename.read_hash('sha256').encode('hex') })
def add_package_to_cache(config, cache, codename, component, package): distro = get_distro(config, codename) if component not in distro["Components"]: raise RuntimeError( "Can not add package %s, component %s not supported by the repository" % (package, component)) deb = DebFile(filename=package) fields = deb.debcontrol() arch = fields["Architecture"] if arch not in distro["Architectures"]: raise RuntimeError( "Can not add package %s, architecture not supported by the repository" % package) fields["SHA256"] = hash_file(package, hashlib.sha256) fields["SHA1"] = hash_file(package, hashlib.sha1) fields["MD5sum"] = hash_file(package, hashlib.md5) fields["Size"] = str(os.path.getsize(package)) fields["Filename"] = "pool/%s/%s" % (component, os.path.basename(package)) packages = get_package_versions(cache, codename, component, arch, fields["Package"]) if fields["Version"] in packages: raise RuntimeError( "Package %s with version %s already exists in the repository" % (fields["Package"], fields["version"])) # convert dict like object to tuple to keep original sorting in json file packages[fields["Version"]] = [(field, fields[field]) for field in fields] print("-> Adding %s=%s to cache for %s/%s" % (fields["Package"], fields["Version"], codename, arch)) return fields["Filename"]
def generate_sut_from_deb(path): """ Generate a Firehose SUT from a .deb file. """ obj = DebFile(filename=path, mode='r') control = obj.debcontrol() version = control['Version'] version, local = parse_version(version) name, arch = [control[x] for x in ['Package', 'Architecture']] return DebianBinary(name, version, local, arch)
def scan_packages(filenames): """Parse names and versions from .deb files. Returns a list of `Package`. """ packages = [] for filename in filenames: deb = DebFile(filename) control = deb.debcontrol() packages.append(Package.from_control(deb.debcontrol())) return packages
def from_deb(cls, deb_file_path): """Create a FetchedPackage from a binary package on disk.""" debcontrol = DebFile(deb_file_path).control.debcontrol() name = debcontrol['Package'] version = debcontrol['Version'] filename = os.path.basename(deb_file_path) size = os.path.getsize(deb_file_path) md5sum = hashlib.md5(open(deb_file_path).read()).hexdigest() sha256sum = hashlib.sha256(open(deb_file_path).read()).hexdigest() architecture = debcontrol['Architecture'] depends = debcontrol.get('Depends') pre_depends = debcontrol.get('Pre-Depends') multi_arch = debcontrol.get('Multi-Arch') conflicts = debcontrol.get('Conflicts') recommends = debcontrol.get('Recommends') provides = debcontrol.get('Provides') replaces = debcontrol.get('Replaces') breaks = debcontrol.get('Breaks') pkg = cls( name, version, filename, size, md5sum, sha256sum, architecture, depends, pre_depends, multi_arch, conflicts, recommends, provides, replaces, breaks) pkg.content = open(deb_file_path) pkg._file_path = deb_file_path return pkg
def getBinFromDeb(fname): """ Get a binary package from a .deb file """ dfile = DebFile(fname) cdict = dfile.debcontrol() result = BinPackageDeb( id=-1, name=cdict['Package'], control=dfile.control.get_content('control', 'utf-8'), cdict=dfile.debcontrol(), Version=cdict['Version'], Architecture=cdict['Architecture'], udeb=fname.endswith('.udeb'), Description_md5=hashlib.md5(cdict['Description'].encode() + b'\n').hexdigest(), debfile=dfile, origfile=fname) result.__dict__.update(utils.Hasher.hash(fname)._asdict()) return result
def from_deb(cls, deb_file_path): """Create a FetchedPackage from a binary package on disk.""" debcontrol = DebFile(deb_file_path).control.debcontrol() name = debcontrol['Package'] version = debcontrol['Version'] filename = os.path.basename(deb_file_path) size = os.path.getsize(deb_file_path) md5sum = hashlib.md5(open(deb_file_path).read()).hexdigest() architecture = debcontrol['Architecture'] depends = debcontrol.get('Depends') pre_depends = debcontrol.get('Pre-Depends') multi_arch = debcontrol.get('Multi-Arch') conflicts = debcontrol.get('Conflicts') recommends = debcontrol.get('Recommends') provides = debcontrol.get('Provides') replaces = debcontrol.get('Replaces') breaks = debcontrol.get('Breaks') pkg = cls(name, version, filename, size, md5sum, architecture, depends, pre_depends, multi_arch, conflicts, recommends, provides, replaces, breaks) pkg.content = open(deb_file_path) pkg._file_path = deb_file_path return pkg
def migrate(*args, **kwargs): """ Add relationship fields (breaks/depends/etc) """ warnings_encountered = False collection = connection.get_collection('units_deb') for unit in collection.find({}): unit_id = unit['_id'] path = unit['_storage_path'] if not os.path.exists(path): warnings_encountered = True msg = 'deb package file corresponding to db_unit with _id = {}\n'\ 'was not found at _storage_path = {}.\n'\ 'The unit was not migrated!'.format(unit_id, path) _logger.warn(msg) continue try: control_fields = DebFile(path).debcontrol() except ArError as error: warnings_encountered = True msg = 'deb package file corresponding to db_unit with _id = {}\n'\ 'with _storage_path = {}\n'\ 'was not recognized as a valid deb file:\n'\ '{}\n'\ 'The unit was not migrated!'.format(unit_id, path, str(error),) _logger.warn(msg) continue update_dict = dict() for field, deb_key in REL_FIELDS_MAP.iteritems(): if deb_key in control_fields: update_dict[field] = DependencyParser.from_string( control_fields[deb_key]) else: update_dict[field] = [] collection.update_one(dict(_id=unit_id), {'$set': update_dict}) if warnings_encountered: msg = 'Warnings were encountered during the db migration!\n'\ 'Check the logs for more information, and consider deleting broken units.' _logger.warn(msg)
def analyze(debfile, package="?", group="?", show_errors=False): deb = DebFile(filename=debfile) tgz = deb.data.tgz() if not os.path.exists(debfile): # print >> sys.stderr, "%s doesn't exists!" % debfile pass if not debfile.endswith(".deb"): return output = {} output["package"] = package output["group"] = group output["build"] = os.path.basename(debfile) output["files"] = [] output["daemon"] = False flag = False directory = False for entry in tgz.getmembers(): size = entry.size # check if package is a daemon if "/etc/rc.d/init.d" in entry.name or "/lib/systemd" in entry.name: output["daemon"] = True # skip 0 byte files only if size == 0 and not stat.S_ISDIR(entry.mode): continue # we are only interested in particular kind of directories if stat.S_ISDIR(entry.mode): if not ((entry.mode & stat.S_ISUID) or (stat.S_ISGID & entry.mode)): continue else: flag = True directory = True if not entry.mode & 0o111: continue # always report setuid files if ((entry.mode & stat.S_ISUID) or (stat.S_ISGID & entry.mode)): flag = True # skip library files filename = entry.name.lstrip(".") if ("lib" in filename and ".so" in filename) or \ filename.endswith(".so"): continue try: contents = tgz.extractfile(entry).read() except Exception as exc: print(exc) # invoke checksec returncode = -1 try: fh = BytesIO(contents) elf = Elf(fh) out = process_file(elf) returncode = 0 dataline = "%s,%s,%s,%s" % (package, os.path.basename(debfile), filename, out) except ELFError as exc: continue except IOError as exc: continue # print p.returncode, filename if returncode == 0 or flag: # populate fileinfo object fileinfo = {} fileinfo["name"] = filename fileinfo["size"] = entry.size fileinfo["mode"] = entry.mode if directory: fileinfo["directory"] = directory output["files"].append(fileinfo) if returncode == 0 and opformat == "csv": print(dataline) if returncode == 0 and opformat == "json": try: for kvp in out.rstrip().split(","): key, value = kvp.split("=") fileinfo[key] = value except Exception: pass if opformat == "json": print( json.dumps(output, sort_keys=True, indent=4, separators=(',', ': ')))
git_refs = git_repo.remotes.origin.refs git_refs_name = list(map(lambda x: str(x).split('/')[-1], git_refs)) logging.debug(git_refs_name) if gh_branch not in git_refs_name: git_repo.git.checkout(b=gh_branch) else: git_repo.git.checkout(gh_branch) # Generate metadata logging.debug("cwd : {}".format(os.getcwd())) logging.debug(os.listdir()) deb_file_handle = DebFile(filename=deb_file_path) deb_file_control = deb_file_handle.debcontrol() current_metadata = { 'format_version': 1, 'sw_version': deb_file_control['Version'], 'sw_architecture': deb_file_control['Architecture'], 'linux_version': deb_file_version } current_metadata_str = json.dumps(current_metadata) logging.debug('Metadata {}'.format(current_metadata_str)) # Get metadata all_commit = git_repo.iter_commits(gh_branch) all_apt_action_commit = list(filter(lambda x: (x.message[:12] == '[apt-action]'), all_commit))
def migrate(*args, **kwargs): """ This migration should achieve the following: * All existing entries in the units_deb collection should store parsed relationship information in any relevant REL_FIELDS. (These fields should never be used for plain Debian relationship strings). * The control_file field is added to all entries in the units_deb collection. This field should contain a dict, containing all control file fields associated with the relevant .deb package. This way we can ensure the db contains all control file information needed to publish a repo containing the unit (including any plain Debian relationship strings). """ warnings_encountered = False deb_collection = connection.get_collection('units_deb') deb_count = deb_collection.count() with utils.MigrationProgressLog('Deb Package', deb_count) as progress_log: for deb_package in deb_collection.find({}).batch_size(100): storage_path = deb_package['_storage_path'] package_id = deb_package['_id'] if not os.path.exists(storage_path): warnings_encountered = True msg = 'deb package file corresponding to db_unit with _id = {}\n'\ 'was not found at _storage_path = {}.\n'\ 'The unit was not migrated!'.format(package_id, storage_path) _logger.warn(msg) continue try: control_fields = DebFile(storage_path).debcontrol() except ArError as error: warnings_encountered = True msg = 'deb package file corresponding to db_unit with _id = {}\n'\ 'with _storage_path = {}\n'\ 'was not recognized as a valid deb file:\n'\ '{}\n'\ 'The unit was not migrated!'.format(package_id, storage_path, str(error),) _logger.warn(msg) continue new_fields = dict() remove_fields = dict() new_fields.update(control_fields=control_fields) # Add parsed relational fields and remove empty relational fields: for field, deb_key in REL_FIELDS_MAP.iteritems(): if deb_key in control_fields: new_fields[field] = DependencyParser.from_string( control_fields[deb_key]) else: remove_fields[field] = '' # Also (re)add any fields that may have been previously ignored: for field, deb_key in IGNORED_FIELDS_MAP.iteritems(): if deb_key in control_fields: new_fields[field] = control_fields[deb_key] deb_collection.update_one( {'_id': package_id}, {'$set': new_fields}, ) deb_collection.update_one( {'_id': package_id}, {'$unset': remove_fields}, ) progress_log.progress() if warnings_encountered: msg = 'Warnings were encountered during the db migration!\n'\ 'Check the logs for more information, and consider deleting broken units.' _logger.warn(msg)