def download_dpkg(package_files, packages, workspace_name): """ Using an unzipped, json package file with full urls, downloads a .deb package Uses the 'Filename' key to download the .deb package """ pkg_vals_to_package_file_and_sha256 = {} package_to_rule_map = {} package_file_to_metadata = {} for pkg_vals in set(packages.split(",")): pkg_split = pkg_vals.split("=") if len(pkg_split) != 2: pkg_name = pkg_vals pkg_version = "" else: pkg_name, pkg_version = pkg_split for package_file in package_files.split(","): if package_file not in package_file_to_metadata: with open(package_file, 'rb') as f: package_file_to_metadata[package_file] = json.load(f) metadata = package_file_to_metadata[package_file] if (pkg_name in metadata and (pkg_version == "" or pkg_version == metadata[pkg_name][VERSION_KEY])): pkg = metadata[pkg_name] buf = urllib2.urlopen(pkg[FILENAME_KEY]) package_to_rule_map[pkg_name] = util.package_to_rule( workspace_name, pkg_name) out_file = os.path.join("file", util.encode_package_name(pkg_name)) with open(out_file, 'w') as f: f.write(buf.read()) expected_checksum = util.sha256_checksum(out_file) actual_checksum = pkg[SHA256_KEY] if actual_checksum != expected_checksum: raise Exception( "Wrong checksum for package %s. Expected: %s, Actual: %s" % (pkg_name, expected_checksum, actual_checksum)) if pkg_version == "": break if (pkg_vals in pkg_vals_to_package_file_and_sha256 and pkg_vals_to_package_file_and_sha256[pkg_vals][1] != actual_checksum): raise Exception( "Conflicting checksums for package %s, version %s. Conflicting checksums: %s:%s, %s:%s" % (pkg_name, pkg_version, pkg_vals_to_package_file_and_sha256[pkg_vals][0], pkg_vals_to_package_file_and_sha256[pkg_vals][1], package_file, actual_checksum)) else: pkg_vals_to_package_file_and_sha256[pkg_vals] = [ package_file, actual_checksum ] break else: raise Exception( "Package: %s, Version: %s not found in any of the sources" % (pkg_name, pkg_version)) with open(PACKAGE_MAP_FILE_NAME, 'w') as f: f.write("packages = " + json.dumps(package_to_rule_map))
def download_dpkg(package_files, packages, workspace_name): """ Using an unzipped, json package file with full urls, downloads a .deb package Uses the 'Filename' key to download the .deb package """ package_to_rule_map = {} for pkg_name in packages.split(","): for package_file in package_files.split(","): with open(package_file, 'rb') as f: metadata = json.load(f) if pkg_name in metadata: pkg = metadata[pkg_name] buf = urllib2.urlopen(pkg[FILENAME_KEY]) package_to_rule_map[pkg_name] = util.package_to_rule( workspace_name, pkg_name) out_file = os.path.join("file", util.encode_package_name(pkg_name)) with open(out_file, 'w') as f: f.write(buf.read()) expected_checksum = util.sha256_checksum(out_file) actual_checksum = pkg[SHA256_KEY] if actual_checksum != expected_checksum: raise Exception( "Wrong checksum for package %s. Expected: %s, Actual: %s", pkg_name, expected_checksum, actual_checksum) break else: raise Exception("Package %s not found in any of the sources" % pkg_name) with open(PACKAGE_MAP_FILE_NAME, 'w') as f: f.write("packages = " + json.dumps(package_to_rule_map))
def download_package_list(mirror_url, distro, arch, snapshot, sha256, packages_gz_url, package_prefix): """Downloads a debian package list, expands the relative urls, and saves the metadata as a json file A debian package list is a gzipped, newline delimited, colon separated file with metadata about all the packages available in that repository. Multiline keys are indented with spaces. An example package looks like: Package: newmail Version: 0.5-2 Installed-Size: 76 Maintainer: Martin Schulze <*****@*****.**> Architecture: amd64 Depends: libc6 (>= 2.7-1) Description: Notificator for incoming mail Homepage: http://www.infodrom.org/projects/newmail/ Description-md5: 49b0168ce625e668ce3031036ad2f541 Tag: interface::commandline, mail::notification, role::program, scope::utility, works-with::mail Section: mail Priority: optional Filename: pool/main/n/newmail/newmail_0.5-2_amd64.deb Size: 14154 MD5sum: 5cd31aab55877339145517fb6d5646cb SHA1: 869934a25a8bb3def0f17fef9221bed2d3a460f9 SHA256: 52ec3ac93cf8ba038fbcefe1e78f26ca1d59356cdc95e60f987c3f52b3f5e7ef """ if bool(packages_gz_url) != bool(package_prefix): raise Exception( "packages_gz_url and package_prefix must be specified or skipped at the same time." ) if (not packages_gz_url) and (not mirror_url or not snapshot or not distro or not arch): raise Exception( "If packages_gz_url is not specified, all of mirror_url, snapshot, " "distro and arch must be specified.") url = packages_gz_url if not url: url = "%s/debian/%s/dists/%s/main/binary-%s/Packages.gz" % ( mirror_url, snapshot, distro, arch) download_and_save(url, "Packages.gz") actual_sha256 = util.sha256_checksum("Packages.gz") if sha256 != actual_sha256: raise Exception( "sha256 of Packages.gz don't match: Expected: %s, Actual:%s" % (sha256, actual_sha256)) with gzip.open("Packages.gz", 'rb') as f: data = f.read() metadata = parse_package_metadata(data, mirror_url, snapshot, package_prefix) with open(PACKAGES_FILE_NAME, 'w') as f: json.dump(metadata, f)
def download_dpkg(package_files, packages, workspace_name, versionsfile): """ Using an unzipped, json package file with full urls, downloads a .deb package Uses the 'Filename' key to download the .deb package """ package_to_rule_map = {} package_to_version_map = {} package_file_to_metadata = {} for pkg_name in set(packages.split(",")): pkg = {} for package_file in package_files.split(","): if package_file not in package_file_to_metadata: with open(package_file, 'rb') as f: data = f.read() package_file_to_metadata[package_file] = json.loads( data.decode('utf-8')) metadata = package_file_to_metadata[package_file] if (pkg_name in metadata and (not VERSION_KEY in pkg or compare_versions( metadata[pkg_name][VERSION_KEY], pkg[VERSION_KEY]) > 0)): pkg = metadata[pkg_name] if (not pkg): raise Exception("Package: %s not found in any of the sources" % pkg_name) else: out_file = os.path.join("file", util.encode_package_name(pkg_name)) download_and_save(pkg_name, pkg[FILENAME_KEY], out_file) package_to_rule_map[pkg_name] = util.package_to_rule( workspace_name, pkg_name) package_to_version_map[pkg_name] = pkg[VERSION_KEY] actual_checksum = util.sha256_checksum(out_file) expected_checksum = pkg[SHA256_KEY] if actual_checksum != expected_checksum: raise Exception( "Wrong checksum for package %s (%s). Expected: %s, Actual: %s" % (pkg_name, pkg[FILENAME_KEY], expected_checksum, actual_checksum)) with open(PACKAGE_MAP_FILE_NAME, 'w') as f: f.write("packages = " + json.dumps(package_to_rule_map)) f.write("\nversions = " + json.dumps(package_to_version_map)) if versionsfile: with open(versionsfile, 'w') as f: f.write( json.dumps(package_to_version_map, sort_keys=True, indent=4, separators=(',', ': '))) f.write('\n')
def test_sha256(self): current_dir = os.path.dirname(__file__) filename = os.path.join(current_dir, 'testdata', 'checksum.txt') actual = util.sha256_checksum(filename) self.assertEqual(CHECKSUM_TXT, actual)
def test_sha256(self): actual = util.sha256_checksum("checksum_txt") self.assertEqual(CHECKSUM_TXT, actual)