class DpkgTest(unittest.TestCase):
def setUp(self):
dpkgfile = os.path.join(os.path.dirname(__file__), TEST_DPKG_FILE)
self.dpkg = Dpkg(dpkgfile)
def test_get_versions(self):
self.assertEqual(self.dpkg.epoch, 1)
self.assertEqual(self.dpkg.upstream_version, '0.0.0')
self.assertEqual(self.dpkg.debian_revision, 'test')
def test_get_message_headers(self):
self.assertEqual(self.dpkg.package, 'testdeb')
self.assertEqual(self.dpkg.PACKAGE, 'testdeb')
self.assertEqual(self.dpkg['package'], 'testdeb')
self.assertEqual(self.dpkg['PACKAGE'], 'testdeb')
self.assertEqual(self.dpkg.get('package'), 'testdeb')
self.assertEqual(self.dpkg.get('PACKAGE'), 'testdeb')
self.assertEqual(self.dpkg.get('nonexistent'), None)
self.assertEqual(self.dpkg.get('nonexistent', 'foo'), 'foo')
def test_missing_header(self):
self.assertRaises(KeyError, self.dpkg.__getitem__, 'xyzzy')
self.assertRaises(AttributeError, self.dpkg.__getattr__, 'xyzzy')
def test_message(self):
self.assertIsInstance(self.dpkg.message, type(Message()))
def __get_packages(self):
# get all files
files = glob.glob( os.path.join(self.binary_path,"") + "*" + self.packageType )
for fname in files:
# extract the package information
pkg_info = DpkgInfo(fname)
# if arch is defined and does not match package, move on to the next
if self.arch is not None:
if str(pkg_info.headers['Architecture']) != self.arch:
continue
# if --multiversion switch is passed, append to the list
if self.multiversion==True:
self.packageList.append(pkg_info)
else:
# finf if package is already in the list
matchedItems = [(index,pkg) for (index,pkg) in enumerate(self.packageList) if self.packageList and pkg.headers['Package'] == pkg_info.headers['Package']]
if len(matchedItems)==0:
# add if not
self.packageList.append(pkg_info)
else:
# compare versions and add if newer
matchedIndex = matchedItems[0][0]
matchedItem = matchedItems[0][1]
dpkg = Dpkg(pkg_info.headers['Filename'])
if dpkg.compare_version_with(matchedItem.headers['Version']) == 1:
self.packageList[matchedIndex] = pkg_info
def CreateDEB(self, bundle_id, recorded_version):
"""
Creates a DEB from information stored in the "temp" folder.
String bundle_id: The bundle id of the package to compress.
"""
# TODO: Find a Python-based method to safely delete all DS_Store files.
call(["find", ".", "-name", ".DS_Store", "-delete"],
cwd=self.root + "temp/" +
bundle_id) # Remove .DS_Store. Kinda finicky.
for file_name in os.listdir(self.root + "temp/" + bundle_id):
if file_name.endswith(".deb"):
# Check if the DEB is a newer version
deb = Dpkg(self.root + "temp/" + bundle_id + "/" + file_name)
if Dpkg.compare_versions(recorded_version, deb.version) == -1:
# Update package stuff
package_name = PackageLister.BundleIdToDirName(
self, bundle_id)
with open(
self.root + "Packages/" + package_name +
"/silica_data/index.json", "r") as content_file:
update_json = json.load(content_file)
update_json['version'] = deb.version
changelog_entry = input(
"The DEB provided for \"" + update_json['name'] +
"\" has a new version available (" +
recorded_version + " -> " + deb.version +
"). What changed in this version?\n(Add multiple lines"
+
" by using a newline character [\\n] and use valid Markdown syntax.): "
)
try:
update_json['changelog'].append({
"version":
deb.version,
"changes":
changelog_entry
})
except Exception:
update_json['changelog'] = {
"version": deb.version,
"changes": changelog_entry
}
return_str = json.dumps(update_json)
print("Updating package index.json...")
PackageLister.CreateFile(
self, "Packages/" + package_name +
"/silica_data/index.json", return_str)
pass
DpkgPy.extract(
self, self.root + "temp/" + bundle_id + "/" + file_name,
self.root + "temp/" + bundle_id)
os.remove(self.root + "temp/" + bundle_id + "/" + file_name)
os.remove(self.root + "temp/" + bundle_id + "/control")
else:
# TODO: Update DpkgPy to generate DEB files without dependencies (for improved win32 support)
call(["dpkg-deb", "-b", "-Zgzip", self.root + "temp/" + bundle_id],
cwd=self.root + "temp/") # Compile DEB
def test_split_full_version(self):
self.assertEqual(Dpkg.split_full_version('00'), (0, '00', '0'))
self.assertEqual(Dpkg.split_full_version('00-00'), (0, '00', '00'))
self.assertEqual(Dpkg.split_full_version('0:0'), (0, '0', '0'))
self.assertEqual(Dpkg.split_full_version('0:0-0'), (0, '0', '0'))
self.assertEqual(Dpkg.split_full_version('0:0.0'), (0, '0.0', '0'))
self.assertEqual(Dpkg.split_full_version('0:0.0-0'), (0, '0.0', '0'))
self.assertEqual(Dpkg.split_full_version('0:0.0-00'), (0, '0.0', '00'))
def test_dstringcmp(self):
self.assertEqual(Dpkg.dstringcmp('~', '.'), -1)
self.assertEqual(Dpkg.dstringcmp('~', 'a'), -1)
self.assertEqual(Dpkg.dstringcmp('a', '.'), -1)
self.assertEqual(Dpkg.dstringcmp('a', '~'), 1)
self.assertEqual(Dpkg.dstringcmp('.', '~'), 1)
self.assertEqual(Dpkg.dstringcmp('.', 'a'), 1)
self.assertEqual(Dpkg.dstringcmp('.', '.'), 0)
self.assertEqual(Dpkg.dstringcmp('0', '0'), 0)
self.assertEqual(Dpkg.dstringcmp('a', 'a'), 0)
# taken from
# http://www.debian.org/doc/debian-policy/ch-controlfields.html#s-f-Version
self.assertEqual(
sorted(['a', '', '~', '~~a', '~~'], key=Dpkg.dstringcmp_key),
['~~', '~~a', '~', '', 'a'])
def __init__(self, binary_path):
self.binary_path = binary_path
self.headers = {}
pkg = Dpkg(self.binary_path)
# build the information for the apt repo
self.headers = pkg.headers
self.headers['Filename'] = pkg.filename.replace("\\", '/')
self.headers['Size'] = pkg.filesize
self.headers['MD5sum'] = pkg.md5
self.headers['SHA1'] = pkg.sha1
self.headers['SHA256'] = pkg.sha256
def create_table_data(results, headers):
table = []
for name in sorted(results):
for dist in sorted(results[name]):
for comp in sorted(results[name][dist]):
for arch in sorted(results[name][dist][comp]):
pkgs = results[name][dist][comp][arch]
pkgs.sort(
key=lambda x: Dpkg.compare_versions_key(x['version']))
for pkg in pkgs:
table.append([pkg[x] for x in headers])
return table
def test_get_alpha(self):
self.assertEqual(Dpkg.get_alphas(''), ('', ''))
self.assertEqual(Dpkg.get_alphas('0'), ('', '0'))
self.assertEqual(Dpkg.get_alphas('00'), ('', '00'))
self.assertEqual(Dpkg.get_alphas('0a'), ('', '0a'))
self.assertEqual(Dpkg.get_alphas('a'), ('a', ''))
self.assertEqual(Dpkg.get_alphas('a0'), ('a', '0'))
def test_listify(self):
self.assertEqual(Dpkg.listify('0'), ['', 0])
self.assertEqual(Dpkg.listify('00'), ['', 0])
self.assertEqual(Dpkg.listify('0a'), ['', 0, 'a', 0])
self.assertEqual(Dpkg.listify('a0'), ['a', 0])
self.assertEqual(Dpkg.listify('a00'), ['a', 0])
self.assertEqual(Dpkg.listify('a'), ['a', 0])
def getDebRepositoryLatestVersion(self, repository, packageName):
repository = repository.split()
URL = '{}/dists/{}'.format(repository[1], repository[2])
packages = list()
for i in range(3, len(repository)):
packagesURL = '{}/{}/binary-amd64/Packages.gz'.format(URL, repository[i])
data = zlib.decompress(requests.get(packagesURL).content, 16+zlib.MAX_WBITS).decode('UTF-8')
for package in filter(None, data.split('\n\n')):
packageDetails = self.convertDebPackagesSectionToYaml(package)
if packageDetails['Package'] == packageName:
packages.append(packageDetails)
version = sorted(packages, key = lambda x:Dpkg.compare_versions_key(x['Version']), reverse=True)
version = version[0]['Version']
self.log.info('Package: {}, latest version: {}'.format(packageName, version))
return version
class Package(object):
"""Represents a .deb package
"""
def __init__(self, package_path):
"""Initialize instance properties
"""
self.dpkg = Dpkg(package_path)
self.headers = dict()
self.get_package_meta()
def get_package_meta(self):
"""Reads in the package metadata and incorporates it into
the class
"""
for key, value in HEADER_KEYS.items():
self.headers[key] = self.dpkg.get(value)
def render_packages_file(self):
"""Dumps out the data that would go in a Packages file
for an apt repo
"""
ret = list()
for key, value in HEADER_KEYS.items():
ret.append("{0}: {1}".format(key, self.headers[key]))
ret.append('')
return "\n".join(ret)
def __getattribute__(self, name):
"""Make the dpkg header keys case-insensitive
"""
headers = [x.lower() for x in HEADER_KEYS]
if name.lower() in headers:
for key in HEADER_KEYS.keys():
if key.lower() == name.lower():
return self.headers[key]
else:
return object.__getattribute__(self, name)
def compareVersions(vulnPackages, vulnFixedVersion):
fixedVer = vulnFixedVersion
for ver in vulnPackages:
#print("ver : ", ver)
currVer = vulnPackages[ver]
#print("currVer : ", currVer)
#print("fixedVer : ", fixedVer)
compareResult = Dpkg.compare_versions(currVer, fixedVer)
if compareResult == 0:
printOutput.append(
'\nThe package version in the product is the same as the Fixed Version ==> CONGRATULATIONS the product is NOT vulnerable.'
)
elif compareResult == 1:
printOutput.append(
'\nThe package version in the product is higher than the Fixed Version ==> CONGRATULATIONS the product is NOT vulnerable.'
)
elif compareResult == -1:
printOutput.append(
'\nThe package version in the product is LOWER than the Fixed Version ==> The product is VULNERABLE!'
)
return compareResult
def parse_files(self):
# dsc file
dsc = Dsc(self.dsc)
depends = set(safe_split(dsc.headers['Build-Depends']))
self.product = dsc.headers['Source']
self.binary = dsc.headers['Binary']
self.version = dsc.headers['Version']
self.package_list = dsc.headers['Package-List']
# changelog
debian_time = parse_changelog(self.changelog)
if debian_time:
self.timestamp = convert_debian_time_to_unix(debian_time)
else:
self.timestamp = -1
# deb and udeb files
for deb in self.debs:
dpkg = Dpkg(deb)
depends.update(safe_split(dpkg.headers['Depends']))
# Set forge as debian because they declared as Debian packages
debian_dependencies = [
parse_dependency(dep, 'debian') for dep in depends
]
self.dependencies.extend(debian_dependencies)
def test_compare_revision_strings(self):
# note that these are testing a single revision string, not the full
# upstream+debian version. IOW, "0.0.9-foo" is an upstream or debian
# revision onto itself, not an upstream of 0.0.9 and a debian of foo.
# equals
self.assertEqual(Dpkg.compare_revision_strings('0', '0'), 0)
self.assertEqual(Dpkg.compare_revision_strings('0', '00'), 0)
self.assertEqual(Dpkg.compare_revision_strings('00.0.9', '0.0.9'), 0)
self.assertEqual(Dpkg.compare_revision_strings('0.00.9-foo', '0.0.9-foo'), 0)
self.assertEqual(Dpkg.compare_revision_strings('0.0.9-1.00foo', '0.0.9-1.0foo'), 0)
# less than
self.assertEqual(Dpkg.compare_revision_strings('0.0.9', '0.0.10'), -1)
self.assertEqual(Dpkg.compare_revision_strings('0.0.9-foo', '0.0.10-foo'), -1)
self.assertEqual(Dpkg.compare_revision_strings('0.0.9-foo', '0.0.10-goo'), -1)
self.assertEqual(Dpkg.compare_revision_strings('0.0.9-foo', '0.0.9-goo'), -1)
self.assertEqual(Dpkg.compare_revision_strings('0.0.10-foo', '0.0.10-goo'), -1)
self.assertEqual(Dpkg.compare_revision_strings('0.0.9-1.0foo', '0.0.9-1.1foo'), -1)
# greater than
self.assertEqual(Dpkg.compare_revision_strings('0.0.10', '0.0.9'), 1)
self.assertEqual(Dpkg.compare_revision_strings('0.0.10-foo', '0.0.9-foo'), 1)
self.assertEqual(Dpkg.compare_revision_strings('0.0.10-foo', '0.0.9-goo'), 1)
self.assertEqual(Dpkg.compare_revision_strings('0.0.9-1.0foo', '0.0.9-1.0bar'), 1)
def test_get_digits(self):
self.assertEqual(Dpkg.get_digits('00'), (0, ''))
self.assertEqual(Dpkg.get_digits('0'), (0, ''))
self.assertEqual(Dpkg.get_digits('0a'), (0, 'a'))
self.assertEqual(Dpkg.get_digits('a'), (0, 'a'))
self.assertEqual(Dpkg.get_digits('a0'), (0, 'a0'))
def web_addpkg(self, reponame, name, version, fobj, dist):
repo = get_repo(db(), reponame)
dist = get_dist(db(), repo, dist)
print("Dist:", dist)
# - read f (write to temp storage if needed) and generate the hashes
# - load with Dpkg to get name version and whatnot
with TemporaryDirectory() as tdir:
tmppkgpath = os.path.join(tdir, "temp.deb")
with open(tmppkgpath, "wb") as fdest:
fhashes = copyhash(fobj.file, fdest)
fsize = os.path.getsize(tmppkgpath)
p = Dpkg(tmppkgpath)
pkgname = "{}_{}_{}.deb".format(p.message['Package'],
p.message['Version'],
p.message['Architecture'])
#TODO keys can be duplicated in email.message.Message, does this cause any problems?
fields = {key: p.message[key] for key in p.message.keys()}
# repos/<reponame>/packages/f/foo.deb
dpath = os.path.join(self.basepath, "repos", repo.name, "packages",
dist.name, pkgname[0], pkgname)
files = self.s3.list_objects(Bucket=self.bucket,
Prefix=dpath).get("Contents")
if files:
print(f"will overwrite: {files}")
pkg = AptPackage(repo=repo,
dist=dist,
name=p.message['Package'],
version=p.message['Version'],
arch=p.message['Architecture'],
fname=pkgname,
size=fsize,
**fhashes,
fields=json.dumps(fields))
db().add(pkg)
db().commit()
try:
with open(tmppkgpath, "rb") as f:
response = self.s3.put_object(Body=f,
Bucket=self.bucket,
Key=dpath)
assert (response["ResponseMetadata"]["HTTPStatusCode"] ==
200), f"Upload failed: {response}"
except Exception:
db().delete(pkg)
db().commit()
raise
dist.dirty = True
db().commit()
self.regen_dist(dist.id)
yield "package name: {}\n".format(pkgname)
yield "package size: {}\n".format(fsize)
yield "package message:\n-----------------\n{}\n-----------------\n".format(
p.message)
yield "package hashes: {}\n".format(fhashes)
def add(args, repodb, repo):
"""Add packages"""
# we check this here before we risk uploading to s3
if not validate_meta(args, repodb):
return 1
success = 0
for fn in args.files:
LOG.info('attempting to add file: %s', fn)
if fn.endswith('.deb'):
pkg = Dpkg(fn)
elif fn.endswith('.dsc'):
pkg = Dsc(fn)
else:
LOG.error('File "%s" is neither a deb nor a dsc files', fn)
success += 1
continue
try:
if isinstance(pkg, Dsc):
LOG.info('attempting to add source to s3: %s',
os.path.basename(fn))
repo.add_source(pkg,
dists=args.distribution,
overwrite=args.overwrite)
LOG.info('attempting to add source to simpledb: %s',
os.path.basename(fn))
repodb.add_source(pkg,
dists=args.distribution,
comps=args.component,
overwrite=args.overwrite,
auto_purge=args.auto_purge)
else:
arch = pkg.architecture
repodb.check_valid_archs([arch])
LOG.info('attempting to add package to s3: %s',
os.path.basename(pkg.filename))
repo.add_package(pkg,
dists=args.distribution,
overwrite=args.overwrite)
LOG.info('attempting to add package to simpledb: %s',
os.path.basename(pkg.filename))
repodb.add_package(pkg,
dists=args.distribution,
comps=args.component,
overwrite=args.overwrite,
auto_purge=args.auto_purge)
LOG.info('Successfully added %s to repoman!', fn)
except InvalidArchitectureError:
LOG.error('Package %s is built for the "%s" architecture, '
'which this repo is not currently configured to '
'serve; I will not add it. You may which to run '
'"repoman repo add_architecture %s"', fn, arch, arch)
success += 1
continue
except KeyExistsError:
LOG.error('Package %s already exists in S3, you either want the '
'--overwrite flag or you want to move/copy the package '
'within the repo. Skipping.', fn)
success += 1
continue
except ItemExistsError:
LOG.error('Package %s already exists in simpledb, you either want '
'the --overwrite flag or you want to move/copy the '
'package within the repo. Skipping.', fn)
success += 1
continue
if success > 0:
LOG.error('Not all packages uploadeded successfully; inspect '
'the log output for errors.')
return success
def __init__(self, package_path):
"""Initialize instance properties
"""
self.dpkg = Dpkg(package_path)
self.headers = dict()
self.get_package_meta()
def test_get_upstream(self):
self.assertEqual(Dpkg.get_upstream('00'), ('00', '0'))
self.assertEqual(Dpkg.get_upstream('foo'), ('foo', '0'))
self.assertEqual(Dpkg.get_upstream('foo-bar'), ('foo', 'bar'))
self.assertEqual(Dpkg.get_upstream('foo-bar-baz'), ('foo-bar', 'baz'))
def test_get_epoch(self):
self.assertEqual(Dpkg.get_epoch('0'), (0, '0'))
self.assertEqual(Dpkg.get_epoch('0:0'), (0, '0'))
self.assertEqual(Dpkg.get_epoch('1:0'), (1, '0'))
self.assertRaises(DpkgVersionError, Dpkg.get_epoch, '1a:0')
def test_compare_versions(self):
# "This [the epoch] is a single (generally small) unsigned integer.
# It may be omitted, in which case zero is assumed."
self.assertEqual(Dpkg.compare_versions('0.0.0', '0:0.0.0'), 0)
self.assertEqual(Dpkg.compare_versions('0:0.0.0-foo', '0.0.0-foo'), 0)
self.assertEqual(Dpkg.compare_versions('0.0.0-a', '0:0.0.0-a'), 0)
# "The absence of a debian_revision is equivalent to a debian_revision
# of 0."
self.assertEqual(Dpkg.compare_versions('0.0.0', '0.0.0-0'), 0)
# tricksy:
self.assertEqual(Dpkg.compare_versions('0.0.0', '0.0.0-00'), 0)
# combining the above
self.assertEqual(Dpkg.compare_versions('0.0.0-0', '0:0.0.0'), 0)
# explicitly equal
self.assertEqual(Dpkg.compare_versions('0.0.0', '0.0.0'), 0)
self.assertEqual(Dpkg.compare_versions('1:0.0.0', '1:0.0.0'), 0)
self.assertEqual(Dpkg.compare_versions('0.0.0-10', '0.0.0-10'), 0)
self.assertEqual(Dpkg.compare_versions('2:0.0.0-1', '2:0.0.0-1'), 0)
self.assertEqual(Dpkg.compare_versions('0:a.0.0-foo', '0:a.0.0-foo'), 0)
# less than
self.assertEqual(Dpkg.compare_versions('0.0.0-0', '0:0.0.1'), -1)
self.assertEqual(Dpkg.compare_versions('0.0.0-0', '0:0.0.0-a'), -1)
self.assertEqual(Dpkg.compare_versions('0.0.0-0', '0:0.0.0-1'), -1)
self.assertEqual(Dpkg.compare_versions('0.0.9', '0.0.10'), -1)
self.assertEqual(Dpkg.compare_versions('0.9.0', '0.10.0'), -1)
self.assertEqual(Dpkg.compare_versions('9.0.0', '10.0.0'), -1)
# greater than
self.assertEqual(Dpkg.compare_versions('0.0.1-0', '0:0.0.0'), 1)
self.assertEqual(Dpkg.compare_versions('0.0.0-a', '0:0.0.0-1'), 1)
self.assertEqual(Dpkg.compare_versions('0.0.0-a', '0:0.0.0-0'), 1)
self.assertEqual(Dpkg.compare_versions('0.0.9', '0.0.1'), 1)
self.assertEqual(Dpkg.compare_versions('0.9.0', '0.1.0'), 1)
self.assertEqual(Dpkg.compare_versions('9.0.0', '1.0.0'), 1)
# unicode me harder
self.assertEqual(Dpkg.compare_versions(u'2:0.0.44-1', u'2:0.0.44-nobin'), -1)
self.assertEqual(Dpkg.compare_versions(u'2:0.0.44-nobin', u'2:0.0.44-1'), 1)
self.assertEqual(Dpkg.compare_versions(u'2:0.0.44-1', u'2:0.0.44-1'), 0)
def setUp(self):
dpkgfile = os.path.join(os.path.dirname(__file__), TEST_DPKG_FILE)
self.dpkg = Dpkg(dpkgfile)
def CreateDEB(self, bundle_id, recorded_version):
"""
Creates a DEB from information stored in the "temp" folder.
String bundle_id: The bundle id of the package to compress.
String recorded_version:
Object tweak_release: A "tweak release" object.
"""
# TODO: Find a Python-based method to safely delete all DS_Store files.
call(["find", ".", "-name", ".DS_Store", "-delete"],
cwd=self.root + "temp/" + bundle_id) # Remove .DS_Store. Kinda finicky.
for file_name in os.listdir(self.root + "temp/" + bundle_id):
if file_name.endswith(".deb"):
# Check if the DEB is a newer version
deb = Dpkg(self.root + "temp/" + bundle_id + "/" + file_name)
if Dpkg.compare_versions(recorded_version, deb.version) == -1:
# Update package stuff
package_name = PackageLister.BundleIdToDirName(self, bundle_id)
with open(self.root + "Packages/" + package_name + "/silica_data/index.json", "r") as content_file:
update_json = json.load(content_file)
update_json['version'] = deb.version
changelog_entry = input("The DEB provided for \"" + update_json['name'] +
"\" has a new version available (" + recorded_version + " -> " +
deb.version + "). What changed in this version?\n(Add multiple lines" +
" by using newline characters [\\n\\n] and use valid Markdown syntax): "
)
try:
update_json['changelog'].append(
{
"version": deb.version,
"changes": changelog_entry
}
)
except Exception:
# Make it a list!
update_json['changelog'] = []
update_json['changelog'].append(
{
"version": deb.version,
"changes": changelog_entry
}
)
# A small note: We already created the variables that contain the changelogs and, to
# make matters worse, all the web assets. The only way to mitigate this is to re-create the
# tweak_release variable again, which wrecks a lot of things (ie runtime).
# A Silica rewrite is required to properly fix this bug.
print("\nA small warning about adding changelogs mid-run:\n")
print("Due to some less-than-ideal design decisions with Silica, for the changelog to show")
print("up, you're going to have to run Silica again. Yes, I know this is annoying, and a proper")
print("solution is in the works, but the under-the-hood changes that'll be needed to fix")
print("it properly would require a rewrite [see issue #22].\n")
print("I'm deeply sorry about this.\n - Shuga.\n")
# Get human-readable folder name
folder = PackageLister.BundleIdToDirName(self, bundle_id)
deb_path = self.root + "Packages/" + folder + "/" + file_name
# Extract Control file and scripts from DEB
DpkgPy.control_extract(self, deb_path, self.root + "Packages/" + folder +
"/silica_data/scripts/")
# Remove the Control; it's not needed.
os.remove(self.root + "Packages/" + folder + "/silica_data/scripts/Control")
if not os.listdir(self.root + "Packages/" + folder + "/silica_data/scripts/"):
os.rmdir(self.root + "Packages/" + folder + "/silica_data/scripts/")
return_str = json.dumps(update_json)
print("Updating package index.json...")
PackageLister.CreateFile(self, "Packages/" + package_name +
"/silica_data/index.json", return_str)
pass
DpkgPy.extract(self, self.root + "temp/" + bundle_id + "/" + file_name, self.root + "temp/" + bundle_id)
try:
os.remove(self.root + "temp/" + bundle_id + "/" + file_name)
except:
pass
try:
os.remove(self.root + "temp/" + bundle_id + "/control")
except:
pass
else:
# TODO: Update DpkgPy to generate DEB files without dependencies (for improved win32 support)
# If the version is consistent, then assume the package is unchanged. Don't regenerate it.
try:
# Check for a DEB that already exists.
docs_deb = Dpkg(self.root + "docs/pkg/" + bundle_id + ".deb")
if docs_deb.version == recorded_version:
shutil.copy(self.root + "docs/pkg/" + bundle_id + ".deb", self.root + "temp/" + bundle_id + ".deb")
call_result = 0;
else:
# Sneaky swap.
call_result = call(["dpkg-deb", "-b", "-Zgzip", self.root + "temp/" + bundle_id], cwd=self.root + "temp/") # Compile DEB
except:
# Create the DEB again.
call_result = call(["dpkg-deb", "-b", "-Zgzip", self.root + "temp/" + bundle_id], cwd=self.root + "temp/") # Compile DEB
if call_result != 0:
# Did we run within WSL?
if "Microsoft" in platform.release():
PackageLister.ErrorReporter(self, "Platform Error!", "dpkg-deb failed to run. "
"This is likely due to improper configuration of WSL. Please check the Silcia README for "
"how to set up WSL for dpkg-deb.")
else:
PackageLister.ErrorReporter(self, "DPKG Error!", "dpkg-deb failed to run. "
"This could be due to a faulty system configuration.")
def CheckForSilicaData(self):
"""
Ensures that a silica_data file exists and if it doesn't, try to create one with as much data as we have.
If there is a DEB file, it will take data from its CONTROL file. It will also auto-update the version number.
If there is no DEB file, it will use the name of the folder, version 1.0.0, try to guess some dependencies,
and add some placeholder data.
:return:
"""
for folder in os.listdir(self.root + "Packages"):
if folder.lower() != ".ds_store":
if not os.path.isdir(self.root + "Packages/" + folder + "/silica_data"):
print("It seems like the package \"" + folder + "\" is not configured. Let's set it up!")
is_deb = False
deb_path = ""
try:
for file_name in os.listdir(self.root + "Packages/" + folder):
if file_name.endswith(".deb"):
is_deb = True
deb_path = self.root + "Packages/" + folder + "/" + file_name
except Exception:
PackageLister.ErrorReporter(self, "Configuration Error!", "Please put your .deb file inside of "
"its own folder. The \"Packages\" directory should be made of multiple folders that each "
"contain data for a single package.\n Please fix this issue and try again.")
# This will be the default scaffolding for our package. Eventually I'll neuter it to only be the
# essential elements; it's also kinda a reference to me.
output = {
"bundle_id": "co.shuga.silica.unknown",
"name": "Unknown Package",
"version": "1.0.0",
"tagline": "An unknown package.",
"homepage": "https://shuga.co/",
"developer": {
"name": "Unknown",
"email": "*****@*****.**"
},
"maintainer": {
"name": "Unknown",
"email": "*****@*****.**"
},
"section": "Themes",
"works_min": "8.0",
"works_max": "13.0",
"featured": "false"
}
if is_deb:
print("Extracting data from DEB...")
deb = Dpkg(deb_path)
output['name'] = deb.headers['Name']
output['bundle_id'] = deb.headers['Package']
try:
output['tagline'] = deb.headers['Description']
except Exception:
output['tagline'] = input("What is a brief description of the package? ")
try:
output['homepage'] = deb.headers['Homepage']
except Exception:
pass
try:
remove_email_regex = re.compile('<.*?>')
output['developer']['name'] = remove_email_regex.sub("", deb.headers['Author'])
except Exception:
output['developer']['name'] = input("Who originally made this package? This may be"
" your name. ")
output['developer']['email'] = input("What is the original author's email address? ")
try:
remove_email_regex = re.compile('<.*?>')
output['maintainer']['name'] = remove_email_regex.sub("", deb.headers['Maintainer'])
except Exception:
output['maintainer']['name'] = input("Who maintains this package now?"
" This is likely your name. ")
output['maintainer']['email'] = input("What is the maintainer's email address? ")
try:
output['sponsor']['name'] = remove_email_regex.sub("", deb.headers['Sponsor'])
except Exception:
pass
try:
output['dependencies'] = deb.headers['Depends']
except Exception:
pass
try:
output['section'] = deb.headers['Section']
except Exception:
pass
try:
output['version'] = deb.headers['Version']
except Exception:
output['version'] = "1.0.0"
try:
output['conflicts'] = deb.headers['Conflicts']
except Exception:
pass
try:
output['replaces'] = deb.headers['Replaces']
except Exception:
pass
try:
output['provides'] = deb.headers['Provides']
except Exception:
pass
try:
output['build_depends'] = deb.headers['Build-Depends']
except Exception:
pass
try:
output['recommends'] = deb.headers['Recommends']
except Exception:
pass
try:
output['suggests'] = deb.headers['Suggests']
except Exception:
pass
try:
output['enhances'] = deb.headers['Enhances']
except Exception:
pass
try:
output['breaks'] = deb.headers['Breaks']
except Exception:
pass
try:
output['tags'] = deb.headers['Tag']
except Exception:
pass
try:
output['suggests'] = deb.headers['Suggests']
except Exception:
pass
# These still need data.
output['works_min'] = input("What is the lowest iOS version the package works on? ")
output['works_max'] = input("What is the highest iOS version the package works on? ")
output['featured'] = input("Should this package be featured on your repo? (true/false) ")
set_tint = input("What would you like this package's tint color to be? To keep it at"
" the default, leave this blank: ")
if set_tint != "":
output['tint'] = set_tint
print("All done! Please look over the generated \"index.json\" file and consider populating the"
" \"silica_data\" folder with a description, screenshots, and an icon.")
# Extract Control file and scripts from DEB
DpkgPy.control_extract(self, deb_path, self.root + "Packages/" + folder +
"/silica_data/scripts/")
# Remove the Control; it's not needed.
os.remove(self.root + "Packages/" + folder + "/silica_data/scripts/Control")
if not os.listdir(self.root + "Packages/" + folder + "/silica_data/scripts/"):
os.rmdir(self.root + "Packages/" + folder + "/silica_data/scripts/")
else:
print("Estimating dependencies...")
# Use the filesystem to see if Zeppelin, Anemone, LockGlyph, XenHTML, and similar.
# If one of these are found, set it as a dependency.
# If multiple of these are found, use a hierarchy system, with Anemone as the highest priority,
# for determining the category.
output['dependencies'] = ""
output['section'] = "Themes"
if os.path.isdir(self.root + "Packages/" + folder + "/Library/Zeppelin"):
output['section'] = "Themes (Zeppelin)"
output['dependencies'] += "com.alexzielenski.zeppelin, "
if os.path.isdir(self.root + "Packages/" + folder + "/Library/Application Support/LockGlyph"):
output['section'] = "Themes (LockGlyph)"
output['dependencies'] += "com.evilgoldfish.lockglypgh, "
if os.path.isdir(self.root + "Packages/" + folder + "/var/mobile/Library/iWidgets"):
output['section'] = "Widgets"
output['dependencies'] += "com.matchstic.xenhtml, "
if os.path.isdir(self.root + "Packages/" + folder + "/Library/Wallpaper"):
output['section'] = "Wallpapers"
if os.path.isdir(self.root + "Packages/" + folder + "/Library/Themes"):
output['section'] = "Themes"
output['dependencies'] += "com.anemonetheming.anemone, "
if output['dependencies'] != "":
output['dependencies'] = output['dependencies'][:-2]
repo_settings = PackageLister.GetRepoSettings(self)
# Ask for name
output['name'] = input("What should we name this package? ")
# Automatically generate a bundle ID from the package name.
domain_breakup = repo_settings['cname'].split(".")[::-1]
only_alpha_regex = re.compile('[^a-zA-Z]')
machine_safe_name = only_alpha_regex.sub("", output['name']).lower()
output['bundle_id'] = ".".join(str(x) for x in domain_breakup) + "." + machine_safe_name
output['tagline'] = input("What is a brief description of the package? ")
output['homepage'] = "https://" + repo_settings['cname']
# I could potentially default this to what is in settings.json but attribution may be an issue.
output['developer']['name'] = input("Who made this package? This is likely your name. ")
output['developer']['email'] = input("What is the author's email address? ")
output['works_min'] = input("What is the lowest iOS version the package works on? ")
output['works_max'] = input("What is the highest iOS version the package works on? ")
output['featured'] = input("Should this package be featured on your repo? (true/false) ")
PackageLister.CreateFolder(self, "Packages/" + folder + "/silica_data/")
PackageLister.CreateFile(self, "Packages/" + folder + "/silica_data/index.json", json.dumps(output))
def CreateDEB(self, bundle_id, recorded_version):
"""
Creates a DEB from information stored in the "temp" folder.
String bundle_id: The bundle id of the package to compress.
String recorded_version:
Object tweak_release: A "tweak release" object.
"""
# TODO: Find a Python-based method to safely delete all DS_Store files.
call(["find", ".", "-name", ".DS_Store", "-delete"],
cwd=self.root + "temp/" + bundle_id) # Remove .DS_Store. Kinda finicky.
for file_name in os.listdir(self.root + "temp/" + bundle_id):
if file_name.endswith(".deb"):
# Check if the DEB is a newer version
deb = Dpkg(self.root + "temp/" + bundle_id + "/" + file_name)
if Dpkg.compare_versions(recorded_version, deb.version) == -1:
# Update package stuff
package_name = PackageLister.BundleIdToDirName(self, bundle_id)
with open(self.root + "Packages/" + package_name + "/silica_data/index.json", "r") as content_file:
update_json = json.load(content_file)
update_json['version'] = deb.version
changelog_entry = input("The DEB provided for \"" + update_json['name'] +
"\" has a new version available (" + recorded_version + " -> " +
deb.version + "). What changed in this version?\n(Add multiple lines" +
" by using newline characters [\\n\\n] and use valid Markdown syntax): "
)
try:
update_json['changelog'].append({
"version": deb.version,
"changes": changelog_entry
})
except Exception:
update_json['changelog'] = {
"version": deb.version,
"changes": changelog_entry
}
# Get human-readable folder name
folder = PackageLister.BundleIdToDirName(self, bundle_id)
deb_path = self.root + "Packages/" + folder + "/" + file_name
# Extract Control file and scripts from DEB
DpkgPy.control_extract(self, deb_path, self.root + "Packages/" + folder +
"/silica_data/scripts/")
# Remove the Control; it's not needed.
os.remove(self.root + "Packages/" + folder + "/silica_data/scripts/Control")
if not os.listdir(self.root + "Packages/" + folder + "/silica_data/scripts/"):
os.rmdir(self.root + "Packages/" + folder + "/silica_data/scripts/")
return_str = json.dumps(update_json)
print("Updating package index.json...")
PackageLister.CreateFile(self, "Packages/" + package_name +
"/silica_data/index.json", return_str)
pass
DpkgPy.extract(self, self.root + "temp/" + bundle_id + "/" + file_name, self.root + "temp/" + bundle_id)
os.remove(self.root + "temp/" + bundle_id + "/" + file_name)
os.remove(self.root + "temp/" + bundle_id + "/control")
else:
# TODO: Update DpkgPy to generate DEB files without dependencies (for improved win32 support)
# If the version is consistent, then assume the package is unchanged. Don't regenerate it.
try:
docs_deb = Dpkg(self.root + "docs/pkg/" + bundle_id + ".deb")
if docs_deb.version == recorded_version:
shutil.copy(self.root + "docs/pkg/" + bundle_id + ".deb", self.root + "temp/" + bundle_id + ".deb")
call_result = 0;
else:
# Sneaky swap.
call_result = call(["dpkg-deb", "-b", "-Zgzip", self.root + "temp/" + bundle_id], cwd=self.root + "temp/") # Compile DEB
except:
call_result = call(["dpkg-deb", "-b", "-Zgzip", self.root + "temp/" + bundle_id], cwd=self.root + "temp/") # Compile DEB
if call_result != 0:
# Did we run within WSL?
if "Microsoft" in platform.release():
PackageLister.ErrorReporter(self, "Platform Error!", "dpkg-deb failed to run. "
"This is due to improper configuration of WSL. Please check the Silcia README for "
"how to set up WSL for dpkg-deb.")
else:
PackageLister.ErrorReporter(self, "Platform Error!", "dpkg-deb failed to run. "
"This may be due to a faulty system configuration.")