def __get_packages(self): # get all files files = glob.glob( os.path.join(self.binary_path,"") + "*" + self.packageType ) for fname in files: # extract the package information pkg_info = DpkgInfo(fname) # if arch is defined and does not match package, move on to the next if self.arch is not None: if str(pkg_info.headers['Architecture']) != self.arch: continue # if --multiversion switch is passed, append to the list if self.multiversion==True: self.packageList.append(pkg_info) else: # finf if package is already in the list matchedItems = [(index,pkg) for (index,pkg) in enumerate(self.packageList) if self.packageList and pkg.headers['Package'] == pkg_info.headers['Package']] if len(matchedItems)==0: # add if not self.packageList.append(pkg_info) else: # compare versions and add if newer matchedIndex = matchedItems[0][0] matchedItem = matchedItems[0][1] dpkg = Dpkg(pkg_info.headers['Filename']) if dpkg.compare_version_with(matchedItem.headers['Version']) == 1: self.packageList[matchedIndex] = pkg_info
def CreateDEB(self, bundle_id, recorded_version): """ Creates a DEB from information stored in the "temp" folder. String bundle_id: The bundle id of the package to compress. """ # TODO: Find a Python-based method to safely delete all DS_Store files. call(["find", ".", "-name", ".DS_Store", "-delete"], cwd=self.root + "temp/" + bundle_id) # Remove .DS_Store. Kinda finicky. for file_name in os.listdir(self.root + "temp/" + bundle_id): if file_name.endswith(".deb"): # Check if the DEB is a newer version deb = Dpkg(self.root + "temp/" + bundle_id + "/" + file_name) if Dpkg.compare_versions(recorded_version, deb.version) == -1: # Update package stuff package_name = PackageLister.BundleIdToDirName( self, bundle_id) with open( self.root + "Packages/" + package_name + "/silica_data/index.json", "r") as content_file: update_json = json.load(content_file) update_json['version'] = deb.version changelog_entry = input( "The DEB provided for \"" + update_json['name'] + "\" has a new version available (" + recorded_version + " -> " + deb.version + "). What changed in this version?\n(Add multiple lines" + " by using a newline character [\\n] and use valid Markdown syntax.): " ) try: update_json['changelog'].append({ "version": deb.version, "changes": changelog_entry }) except Exception: update_json['changelog'] = { "version": deb.version, "changes": changelog_entry } return_str = json.dumps(update_json) print("Updating package index.json...") PackageLister.CreateFile( self, "Packages/" + package_name + "/silica_data/index.json", return_str) pass DpkgPy.extract( self, self.root + "temp/" + bundle_id + "/" + file_name, self.root + "temp/" + bundle_id) os.remove(self.root + "temp/" + bundle_id + "/" + file_name) os.remove(self.root + "temp/" + bundle_id + "/control") else: # TODO: Update DpkgPy to generate DEB files without dependencies (for improved win32 support) call(["dpkg-deb", "-b", "-Zgzip", self.root + "temp/" + bundle_id], cwd=self.root + "temp/") # Compile DEB
def __init__(self, binary_path): self.binary_path = binary_path self.headers = {} pkg = Dpkg(self.binary_path) # build the information for the apt repo self.headers = pkg.headers self.headers['Filename'] = pkg.filename.replace("\\", '/') self.headers['Size'] = pkg.filesize self.headers['MD5sum'] = pkg.md5 self.headers['SHA1'] = pkg.sha1 self.headers['SHA256'] = pkg.sha256
def parse_files(self): # dsc file dsc = Dsc(self.dsc) depends = set(safe_split(dsc.headers['Build-Depends'])) self.product = dsc.headers['Source'] self.binary = dsc.headers['Binary'] self.version = dsc.headers['Version'] self.package_list = dsc.headers['Package-List'] # changelog debian_time = parse_changelog(self.changelog) if debian_time: self.timestamp = convert_debian_time_to_unix(debian_time) else: self.timestamp = -1 # deb and udeb files for deb in self.debs: dpkg = Dpkg(deb) depends.update(safe_split(dpkg.headers['Depends'])) # Set forge as debian because they declared as Debian packages debian_dependencies = [ parse_dependency(dep, 'debian') for dep in depends ] self.dependencies.extend(debian_dependencies)
def CheckForSilicaData(self): """ Ensures that a silica_data file exists and if it doesn't, try to create one with as much data as we have. If there is a DEB file, it will take data from its CONTROL file. It will also auto-update the version number. If there is no DEB file, it will use the name of the folder, version 1.0.0, try to guess some dependencies, and add some placeholder data. :return: """ for folder in os.listdir(self.root + "Packages"): if folder.lower() != ".ds_store": if not os.path.isdir(self.root + "Packages/" + folder + "/silica_data"): print("It seems like the package \"" + folder + "\" is not configured. Let's set it up!") is_deb = False deb_path = "" try: for file_name in os.listdir(self.root + "Packages/" + folder): if file_name.endswith(".deb"): is_deb = True deb_path = self.root + "Packages/" + folder + "/" + file_name except Exception: PackageLister.ErrorReporter(self, "Configuration Error!", "Please put your .deb file inside of " "its own folder. The \"Packages\" directory should be made of multiple folders that each " "contain data for a single package.\n Please fix this issue and try again.") # This will be the default scaffolding for our package. Eventually I'll neuter it to only be the # essential elements; it's also kinda a reference to me. output = { "bundle_id": "co.shuga.silica.unknown", "name": "Unknown Package", "version": "1.0.0", "tagline": "An unknown package.", "homepage": "https://shuga.co/", "developer": { "name": "Unknown", "email": "*****@*****.**" }, "maintainer": { "name": "Unknown", "email": "*****@*****.**" }, "section": "Themes", "works_min": "8.0", "works_max": "13.0", "featured": "false" } if is_deb: print("Extracting data from DEB...") deb = Dpkg(deb_path) output['name'] = deb.headers['Name'] output['bundle_id'] = deb.headers['Package'] try: output['tagline'] = deb.headers['Description'] except Exception: output['tagline'] = input("What is a brief description of the package? ") try: output['homepage'] = deb.headers['Homepage'] except Exception: pass try: remove_email_regex = re.compile('<.*?>') output['developer']['name'] = remove_email_regex.sub("", deb.headers['Author']) except Exception: output['developer']['name'] = input("Who originally made this package? This may be" " your name. ") output['developer']['email'] = input("What is the original author's email address? ") try: remove_email_regex = re.compile('<.*?>') output['maintainer']['name'] = remove_email_regex.sub("", deb.headers['Maintainer']) except Exception: output['maintainer']['name'] = input("Who maintains this package now?" " This is likely your name. ") output['maintainer']['email'] = input("What is the maintainer's email address? ") try: output['sponsor']['name'] = remove_email_regex.sub("", deb.headers['Sponsor']) except Exception: pass try: output['dependencies'] = deb.headers['Depends'] except Exception: pass try: output['section'] = deb.headers['Section'] except Exception: pass try: output['version'] = deb.headers['Version'] except Exception: output['version'] = "1.0.0" try: output['conflicts'] = deb.headers['Conflicts'] except Exception: pass try: output['replaces'] = deb.headers['Replaces'] except Exception: pass try: output['provides'] = deb.headers['Provides'] except Exception: pass try: output['build_depends'] = deb.headers['Build-Depends'] except Exception: pass try: output['recommends'] = deb.headers['Recommends'] except Exception: pass try: output['suggests'] = deb.headers['Suggests'] except Exception: pass try: output['enhances'] = deb.headers['Enhances'] except Exception: pass try: output['breaks'] = deb.headers['Breaks'] except Exception: pass try: output['tags'] = deb.headers['Tag'] except Exception: pass try: output['suggests'] = deb.headers['Suggests'] except Exception: pass # These still need data. output['works_min'] = input("What is the lowest iOS version the package works on? ") output['works_max'] = input("What is the highest iOS version the package works on? ") output['featured'] = input("Should this package be featured on your repo? (true/false) ") set_tint = input("What would you like this package's tint color to be? To keep it at" " the default, leave this blank: ") if set_tint != "": output['tint'] = set_tint print("All done! Please look over the generated \"index.json\" file and consider populating the" " \"silica_data\" folder with a description, screenshots, and an icon.") # Extract Control file and scripts from DEB DpkgPy.control_extract(self, deb_path, self.root + "Packages/" + folder + "/silica_data/scripts/") # Remove the Control; it's not needed. os.remove(self.root + "Packages/" + folder + "/silica_data/scripts/Control") if not os.listdir(self.root + "Packages/" + folder + "/silica_data/scripts/"): os.rmdir(self.root + "Packages/" + folder + "/silica_data/scripts/") else: print("Estimating dependencies...") # Use the filesystem to see if Zeppelin, Anemone, LockGlyph, XenHTML, and similar. # If one of these are found, set it as a dependency. # If multiple of these are found, use a hierarchy system, with Anemone as the highest priority, # for determining the category. output['dependencies'] = "" output['section'] = "Themes" if os.path.isdir(self.root + "Packages/" + folder + "/Library/Zeppelin"): output['section'] = "Themes (Zeppelin)" output['dependencies'] += "com.alexzielenski.zeppelin, " if os.path.isdir(self.root + "Packages/" + folder + "/Library/Application Support/LockGlyph"): output['section'] = "Themes (LockGlyph)" output['dependencies'] += "com.evilgoldfish.lockglypgh, " if os.path.isdir(self.root + "Packages/" + folder + "/var/mobile/Library/iWidgets"): output['section'] = "Widgets" output['dependencies'] += "com.matchstic.xenhtml, " if os.path.isdir(self.root + "Packages/" + folder + "/Library/Wallpaper"): output['section'] = "Wallpapers" if os.path.isdir(self.root + "Packages/" + folder + "/Library/Themes"): output['section'] = "Themes" output['dependencies'] += "com.anemonetheming.anemone, " if output['dependencies'] != "": output['dependencies'] = output['dependencies'][:-2] repo_settings = PackageLister.GetRepoSettings(self) # Ask for name output['name'] = input("What should we name this package? ") # Automatically generate a bundle ID from the package name. domain_breakup = repo_settings['cname'].split(".")[::-1] only_alpha_regex = re.compile('[^a-zA-Z]') machine_safe_name = only_alpha_regex.sub("", output['name']).lower() output['bundle_id'] = ".".join(str(x) for x in domain_breakup) + "." + machine_safe_name output['tagline'] = input("What is a brief description of the package? ") output['homepage'] = "https://" + repo_settings['cname'] # I could potentially default this to what is in settings.json but attribution may be an issue. output['developer']['name'] = input("Who made this package? This is likely your name. ") output['developer']['email'] = input("What is the author's email address? ") output['works_min'] = input("What is the lowest iOS version the package works on? ") output['works_max'] = input("What is the highest iOS version the package works on? ") output['featured'] = input("Should this package be featured on your repo? (true/false) ") PackageLister.CreateFolder(self, "Packages/" + folder + "/silica_data/") PackageLister.CreateFile(self, "Packages/" + folder + "/silica_data/index.json", json.dumps(output))
def CreateDEB(self, bundle_id, recorded_version): """ Creates a DEB from information stored in the "temp" folder. String bundle_id: The bundle id of the package to compress. String recorded_version: Object tweak_release: A "tweak release" object. """ # TODO: Find a Python-based method to safely delete all DS_Store files. call(["find", ".", "-name", ".DS_Store", "-delete"], cwd=self.root + "temp/" + bundle_id) # Remove .DS_Store. Kinda finicky. for file_name in os.listdir(self.root + "temp/" + bundle_id): if file_name.endswith(".deb"): # Check if the DEB is a newer version deb = Dpkg(self.root + "temp/" + bundle_id + "/" + file_name) if Dpkg.compare_versions(recorded_version, deb.version) == -1: # Update package stuff package_name = PackageLister.BundleIdToDirName(self, bundle_id) with open(self.root + "Packages/" + package_name + "/silica_data/index.json", "r") as content_file: update_json = json.load(content_file) update_json['version'] = deb.version changelog_entry = input("The DEB provided for \"" + update_json['name'] + "\" has a new version available (" + recorded_version + " -> " + deb.version + "). What changed in this version?\n(Add multiple lines" + " by using newline characters [\\n\\n] and use valid Markdown syntax): " ) try: update_json['changelog'].append( { "version": deb.version, "changes": changelog_entry } ) except Exception: # Make it a list! update_json['changelog'] = [] update_json['changelog'].append( { "version": deb.version, "changes": changelog_entry } ) # A small note: We already created the variables that contain the changelogs and, to # make matters worse, all the web assets. The only way to mitigate this is to re-create the # tweak_release variable again, which wrecks a lot of things (ie runtime). # A Silica rewrite is required to properly fix this bug. print("\nA small warning about adding changelogs mid-run:\n") print("Due to some less-than-ideal design decisions with Silica, for the changelog to show") print("up, you're going to have to run Silica again. Yes, I know this is annoying, and a proper") print("solution is in the works, but the under-the-hood changes that'll be needed to fix") print("it properly would require a rewrite [see issue #22].\n") print("I'm deeply sorry about this.\n - Shuga.\n") # Get human-readable folder name folder = PackageLister.BundleIdToDirName(self, bundle_id) deb_path = self.root + "Packages/" + folder + "/" + file_name # Extract Control file and scripts from DEB DpkgPy.control_extract(self, deb_path, self.root + "Packages/" + folder + "/silica_data/scripts/") # Remove the Control; it's not needed. os.remove(self.root + "Packages/" + folder + "/silica_data/scripts/Control") if not os.listdir(self.root + "Packages/" + folder + "/silica_data/scripts/"): os.rmdir(self.root + "Packages/" + folder + "/silica_data/scripts/") return_str = json.dumps(update_json) print("Updating package index.json...") PackageLister.CreateFile(self, "Packages/" + package_name + "/silica_data/index.json", return_str) pass DpkgPy.extract(self, self.root + "temp/" + bundle_id + "/" + file_name, self.root + "temp/" + bundle_id) try: os.remove(self.root + "temp/" + bundle_id + "/" + file_name) except: pass try: os.remove(self.root + "temp/" + bundle_id + "/control") except: pass else: # TODO: Update DpkgPy to generate DEB files without dependencies (for improved win32 support) # If the version is consistent, then assume the package is unchanged. Don't regenerate it. try: # Check for a DEB that already exists. docs_deb = Dpkg(self.root + "docs/pkg/" + bundle_id + ".deb") if docs_deb.version == recorded_version: shutil.copy(self.root + "docs/pkg/" + bundle_id + ".deb", self.root + "temp/" + bundle_id + ".deb") call_result = 0; else: # Sneaky swap. call_result = call(["dpkg-deb", "-b", "-Zgzip", self.root + "temp/" + bundle_id], cwd=self.root + "temp/") # Compile DEB except: # Create the DEB again. call_result = call(["dpkg-deb", "-b", "-Zgzip", self.root + "temp/" + bundle_id], cwd=self.root + "temp/") # Compile DEB if call_result != 0: # Did we run within WSL? if "Microsoft" in platform.release(): PackageLister.ErrorReporter(self, "Platform Error!", "dpkg-deb failed to run. " "This is likely due to improper configuration of WSL. Please check the Silcia README for " "how to set up WSL for dpkg-deb.") else: PackageLister.ErrorReporter(self, "DPKG Error!", "dpkg-deb failed to run. " "This could be due to a faulty system configuration.")
def setUp(self): dpkgfile = os.path.join(os.path.dirname(__file__), TEST_DPKG_FILE) self.dpkg = Dpkg(dpkgfile)
def add(args, repodb, repo): """Add packages""" # we check this here before we risk uploading to s3 if not validate_meta(args, repodb): return 1 success = 0 for fn in args.files: LOG.info('attempting to add file: %s', fn) if fn.endswith('.deb'): pkg = Dpkg(fn) elif fn.endswith('.dsc'): pkg = Dsc(fn) else: LOG.error('File "%s" is neither a deb nor a dsc files', fn) success += 1 continue try: if isinstance(pkg, Dsc): LOG.info('attempting to add source to s3: %s', os.path.basename(fn)) repo.add_source(pkg, dists=args.distribution, overwrite=args.overwrite) LOG.info('attempting to add source to simpledb: %s', os.path.basename(fn)) repodb.add_source(pkg, dists=args.distribution, comps=args.component, overwrite=args.overwrite, auto_purge=args.auto_purge) else: arch = pkg.architecture repodb.check_valid_archs([arch]) LOG.info('attempting to add package to s3: %s', os.path.basename(pkg.filename)) repo.add_package(pkg, dists=args.distribution, overwrite=args.overwrite) LOG.info('attempting to add package to simpledb: %s', os.path.basename(pkg.filename)) repodb.add_package(pkg, dists=args.distribution, comps=args.component, overwrite=args.overwrite, auto_purge=args.auto_purge) LOG.info('Successfully added %s to repoman!', fn) except InvalidArchitectureError: LOG.error('Package %s is built for the "%s" architecture, ' 'which this repo is not currently configured to ' 'serve; I will not add it. You may which to run ' '"repoman repo add_architecture %s"', fn, arch, arch) success += 1 continue except KeyExistsError: LOG.error('Package %s already exists in S3, you either want the ' '--overwrite flag or you want to move/copy the package ' 'within the repo. Skipping.', fn) success += 1 continue except ItemExistsError: LOG.error('Package %s already exists in simpledb, you either want ' 'the --overwrite flag or you want to move/copy the ' 'package within the repo. Skipping.', fn) success += 1 continue if success > 0: LOG.error('Not all packages uploadeded successfully; inspect ' 'the log output for errors.') return success
def __init__(self, package_path): """Initialize instance properties """ self.dpkg = Dpkg(package_path) self.headers = dict() self.get_package_meta()
def web_addpkg(self, reponame, name, version, fobj, dist): repo = get_repo(db(), reponame) dist = get_dist(db(), repo, dist) print("Dist:", dist) # - read f (write to temp storage if needed) and generate the hashes # - load with Dpkg to get name version and whatnot with TemporaryDirectory() as tdir: tmppkgpath = os.path.join(tdir, "temp.deb") with open(tmppkgpath, "wb") as fdest: fhashes = copyhash(fobj.file, fdest) fsize = os.path.getsize(tmppkgpath) p = Dpkg(tmppkgpath) pkgname = "{}_{}_{}.deb".format(p.message['Package'], p.message['Version'], p.message['Architecture']) #TODO keys can be duplicated in email.message.Message, does this cause any problems? fields = {key: p.message[key] for key in p.message.keys()} # repos/<reponame>/packages/f/foo.deb dpath = os.path.join(self.basepath, "repos", repo.name, "packages", dist.name, pkgname[0], pkgname) files = self.s3.list_objects(Bucket=self.bucket, Prefix=dpath).get("Contents") if files: print(f"will overwrite: {files}") pkg = AptPackage(repo=repo, dist=dist, name=p.message['Package'], version=p.message['Version'], arch=p.message['Architecture'], fname=pkgname, size=fsize, **fhashes, fields=json.dumps(fields)) db().add(pkg) db().commit() try: with open(tmppkgpath, "rb") as f: response = self.s3.put_object(Body=f, Bucket=self.bucket, Key=dpath) assert (response["ResponseMetadata"]["HTTPStatusCode"] == 200), f"Upload failed: {response}" except Exception: db().delete(pkg) db().commit() raise dist.dirty = True db().commit() self.regen_dist(dist.id) yield "package name: {}\n".format(pkgname) yield "package size: {}\n".format(fsize) yield "package message:\n-----------------\n{}\n-----------------\n".format( p.message) yield "package hashes: {}\n".format(fhashes)
def CreateDEB(self, bundle_id, recorded_version): """ Creates a DEB from information stored in the "temp" folder. String bundle_id: The bundle id of the package to compress. String recorded_version: Object tweak_release: A "tweak release" object. """ # TODO: Find a Python-based method to safely delete all DS_Store files. call(["find", ".", "-name", ".DS_Store", "-delete"], cwd=self.root + "temp/" + bundle_id) # Remove .DS_Store. Kinda finicky. for file_name in os.listdir(self.root + "temp/" + bundle_id): if file_name.endswith(".deb"): # Check if the DEB is a newer version deb = Dpkg(self.root + "temp/" + bundle_id + "/" + file_name) if Dpkg.compare_versions(recorded_version, deb.version) == -1: # Update package stuff package_name = PackageLister.BundleIdToDirName(self, bundle_id) with open(self.root + "Packages/" + package_name + "/silica_data/index.json", "r") as content_file: update_json = json.load(content_file) update_json['version'] = deb.version changelog_entry = input("The DEB provided for \"" + update_json['name'] + "\" has a new version available (" + recorded_version + " -> " + deb.version + "). What changed in this version?\n(Add multiple lines" + " by using newline characters [\\n\\n] and use valid Markdown syntax): " ) try: update_json['changelog'].append({ "version": deb.version, "changes": changelog_entry }) except Exception: update_json['changelog'] = { "version": deb.version, "changes": changelog_entry } # Get human-readable folder name folder = PackageLister.BundleIdToDirName(self, bundle_id) deb_path = self.root + "Packages/" + folder + "/" + file_name # Extract Control file and scripts from DEB DpkgPy.control_extract(self, deb_path, self.root + "Packages/" + folder + "/silica_data/scripts/") # Remove the Control; it's not needed. os.remove(self.root + "Packages/" + folder + "/silica_data/scripts/Control") if not os.listdir(self.root + "Packages/" + folder + "/silica_data/scripts/"): os.rmdir(self.root + "Packages/" + folder + "/silica_data/scripts/") return_str = json.dumps(update_json) print("Updating package index.json...") PackageLister.CreateFile(self, "Packages/" + package_name + "/silica_data/index.json", return_str) pass DpkgPy.extract(self, self.root + "temp/" + bundle_id + "/" + file_name, self.root + "temp/" + bundle_id) os.remove(self.root + "temp/" + bundle_id + "/" + file_name) os.remove(self.root + "temp/" + bundle_id + "/control") else: # TODO: Update DpkgPy to generate DEB files without dependencies (for improved win32 support) # If the version is consistent, then assume the package is unchanged. Don't regenerate it. try: docs_deb = Dpkg(self.root + "docs/pkg/" + bundle_id + ".deb") if docs_deb.version == recorded_version: shutil.copy(self.root + "docs/pkg/" + bundle_id + ".deb", self.root + "temp/" + bundle_id + ".deb") call_result = 0; else: # Sneaky swap. call_result = call(["dpkg-deb", "-b", "-Zgzip", self.root + "temp/" + bundle_id], cwd=self.root + "temp/") # Compile DEB except: call_result = call(["dpkg-deb", "-b", "-Zgzip", self.root + "temp/" + bundle_id], cwd=self.root + "temp/") # Compile DEB if call_result != 0: # Did we run within WSL? if "Microsoft" in platform.release(): PackageLister.ErrorReporter(self, "Platform Error!", "dpkg-deb failed to run. " "This is due to improper configuration of WSL. Please check the Silcia README for " "how to set up WSL for dpkg-deb.") else: PackageLister.ErrorReporter(self, "Platform Error!", "dpkg-deb failed to run. " "This may be due to a faulty system configuration.")