def setUp(self): current_dir = os.path.dirname(__file__) filename = os.path.join(current_dir, 'testdata', 'Packages.txt') with open(filename) as f: data = f.read() self.data = data self.mirror_url = "http://debian.org" self.package_prefix = "http://dummy/prefix/" self.debian_repo_metadata = parse_package_metadata( self.data, self.mirror_url, "20170701", "") self.arbitrary_repo_metadata = parse_package_metadata( self.data, "", "", self.package_prefix)
def download_package_list(mirror_url, distro, arch, snapshot, sha256, packages_gz_url, package_prefix): """Downloads a debian package list, expands the relative urls, and saves the metadata as a json file A debian package list is a gzipped, newline delimited, colon separated file with metadata about all the packages available in that repository. Multiline keys are indented with spaces. An example package looks like: Package: newmail Version: 0.5-2 Installed-Size: 76 Maintainer: Martin Schulze <*****@*****.**> Architecture: amd64 Depends: libc6 (>= 2.7-1) Description: Notificator for incoming mail Homepage: http://www.infodrom.org/projects/newmail/ Description-md5: 49b0168ce625e668ce3031036ad2f541 Tag: interface::commandline, mail::notification, role::program, scope::utility, works-with::mail Section: mail Priority: optional Filename: pool/main/n/newmail/newmail_0.5-2_amd64.deb Size: 14154 MD5sum: 5cd31aab55877339145517fb6d5646cb SHA1: 869934a25a8bb3def0f17fef9221bed2d3a460f9 SHA256: 52ec3ac93cf8ba038fbcefe1e78f26ca1d59356cdc95e60f987c3f52b3f5e7ef """ if bool(packages_gz_url) != bool(package_prefix): raise Exception( "packages_gz_url and package_prefix must be specified or skipped at the same time." ) if (not packages_gz_url) and (not mirror_url or not snapshot or not distro or not arch): raise Exception( "If packages_gz_url is not specified, all of mirror_url, snapshot, " "distro and arch must be specified.") url = packages_gz_url if not url: url = "%s/debian/%s/dists/%s/main/binary-%s/Packages.gz" % ( mirror_url, snapshot, distro, arch) download_and_save(url, "Packages.gz") actual_sha256 = util.sha256_checksum("Packages.gz") if sha256 != actual_sha256: raise Exception( "sha256 of Packages.gz don't match: Expected: %s, Actual:%s" % (sha256, actual_sha256)) with gzip.open("Packages.gz", 'rb') as f: data = f.read() metadata = parse_package_metadata(data, mirror_url, snapshot, package_prefix) with open(PACKAGES_FILE_NAME, 'w') as f: json.dump(metadata, f)
def setUp(self): current_dir = os.path.dirname(__file__) filename = os.path.join(current_dir, 'testdata', 'Packages.txt') with open(filename) as f: data = f.read() self.data = data self.mirror_url = "http://debian.org" self.metadata = parse_package_metadata(self.data, self.mirror_url)
def download_package_list(mirror_url, distro, arch): """Downloads a debian package list, expands the relative urls, and saves the metadata as a json file A debian package list is a gzipped, newline delimited, colon separated file with metadata about all the packages available in that repository. Multiline keys are indented with spaces. An example package looks like: Package: newmail Version: 0.5-2 Installed-Size: 76 Maintainer: Martin Schulze <*****@*****.**> Architecture: amd64 Depends: libc6 (>= 2.7-1) Description: Notificator for incoming mail Homepage: http://www.infodrom.org/projects/newmail/ Description-md5: 49b0168ce625e668ce3031036ad2f541 Tag: interface::commandline, mail::notification, role::program, scope::utility, works-with::mail Section: mail Priority: optional Filename: pool/main/n/newmail/newmail_0.5-2_amd64.deb Size: 14154 MD5sum: 5cd31aab55877339145517fb6d5646cb SHA1: 869934a25a8bb3def0f17fef9221bed2d3a460f9 SHA256: 52ec3ac93cf8ba038fbcefe1e78f26ca1d59356cdc95e60f987c3f52b3f5e7ef """ url = "%s/debian/dists/%s/main/binary-%s/Packages.gz" % (mirror_url, distro, arch) buf = urllib2.urlopen(url) f = gzip.GzipFile(fileobj=io.BytesIO(buf.read())) data = f.read() metadata = parse_package_metadata(data, mirror_url) with open(PACKAGES_FILE_NAME, 'w') as f: json.dump(metadata, f)