def extract(repo="DOE-ICoM/icom-mesh-data", vtag="v0.1.0"): """ EXTRACT: download and unpack data assets for icom-mesh. """ # Authors: Darren Engwirda cwd_pointer = os.getcwd() try: print("Extracting to:", os.path.join(HERE, "data")) print("") directory = os.path.join(HERE, "data") os.chdir(directory) gh_asset_download(repo, vtag) for item in os.listdir(directory): if item.endswith(".zip"): file_name = os.path.abspath(item) zip_func = zipfile.ZipFile(file_name) zip_func.extractall(directory) zip_func.close() print("") print("Done downloading + unpacking assets.") print("") finally: os.chdir(cwd_pointer) return
def sign_release(self, release, other_names, asc_names, is_newest_release): """Download/sign unsigned assets, upload .asc counterparts. Create SHA256SUMS.txt with all assets included and upload it with SHA256SUMS.txt.asc counterpart. """ repo = self.repo tag = release.get('tag_name', None) if not tag: print('Release have no tag name, skip release\n') return with ChdirTemporaryDirectory() as tmpdir: with open(SHA_FNAME, 'w') as fdw: sdist_match = None for name in other_names: if name == SHA_FNAME: continue gh_asset_download(repo, tag, name) if not self.no_ppa: sdist_match = sdist_match \ or SDIST_NAME_PATTERN.match(name) apk_match = UNSIGNED_APK_PATTERN.match(name) if apk_match: unsigned_name = name name = self.sign_apk(unsigned_name, apk_match.group(1)) gh_asset_upload(repo, tag, name, dry_run=self.dry_run) gh_asset_delete(repo, tag, unsigned_name, dry_run=self.dry_run) if not '%s.asc' % name in asc_names or self.force: self.sign_file_name(name) if self.force: gh_asset_delete(repo, tag, '%s.asc' % name, dry_run=self.dry_run) gh_asset_upload(repo, tag, '%s.asc' % name, dry_run=self.dry_run) sumline = '%s %s\n' % (sha256_checksum(name), name) fdw.write(sumline) self.sign_file_name(SHA_FNAME, detach=False) gh_asset_delete(repo, tag, '%s.asc' % SHA_FNAME, dry_run=self.dry_run) gh_asset_upload(repo, tag, '%s.asc' % SHA_FNAME, dry_run=self.dry_run) if sdist_match and is_newest_release: self.make_ppa(sdist_match, tmpdir, tag)
def test_download_twice(tmpdir): _download_test_prerequisites(tmpdir) with push_dir(tmpdir): download_count = ghr.gh_asset_download( REPO_NAME, tag_name="1.0.0", pattern="asset_1_a" ) assert download_count == 1 download_count = ghr.gh_asset_download( REPO_NAME, tag_name="1.0.0", pattern="asset_1_a" ) assert download_count == 0
def test_download_twice(tmpdir): _download_test_prerequisites(tmpdir) with push_dir(tmpdir): download_count = ghr.gh_asset_download(REPO_NAME, tag_name="1.0.0", pattern="asset_1_a") assert download_count == 1 download_count = ghr.gh_asset_download(REPO_NAME, tag_name="1.0.0", pattern="asset_1_a") assert download_count == 0
def sign_release(self, release, other_names, asc_names): """Download/sign unsigned assets, upload .asc counterparts. Create SHA256SUMS.txt with all assets included and upload it with SHA256SUMS.txt.asc counterpart. """ repo = self.repo tag = release.get('tag_name', None) if not tag: print 'Release have no tag name, skip release\n' return with ChdirTemporaryDirectory(): with open(SHA_FNAME, 'w') as fdw: for name in other_names: if name == SHA_FNAME: continue gh_asset_download(repo, tag, name) if not '%s.asc' % name in asc_names or self.force: self.sign_file_name(name) if self.force: gh_asset_delete(repo, tag, '%s.asc' % name, dry_run=self.dry_run) gh_asset_upload(repo, tag, '%s.asc' % name, dry_run=self.dry_run) sumline = '%s %s\n' % (sha256_checksum(name), name) fdw.write(sumline) self.sign_file_name(SHA_FNAME, detach=False) gh_asset_delete(repo, tag, '%s.asc' % SHA_FNAME, dry_run=self.dry_run) gh_asset_upload(repo, tag, '%s.asc' % SHA_FNAME, dry_run=self.dry_run)
def download(repo_name, root_dir, download_dir, hashalgo, github_token=None): """Download files associated with HASHALGO release into directory (root_dir)/(hashalgo). List of files is taken from (root_dir)/(hashalgo).csv. If multiple hashes associated with the same filename then the last entry will be used. """ if github_token: github_release._github_token_cli_arg = github_token if not os.path.isdir(download_dir): os.mkdir(download_dir) hashalgo_dir = os.path.join(root_dir, hashalgo) if not os.path.isdir(hashalgo_dir): os.mkdir(hashalgo_dir) hashalgo_csv = download_fileindex_csv( repo_name, hashalgo_dir, hashalgo, github_token ) fileindex = read_fileindex_csv(hashalgo_csv) logging.debug(hashalgo + ": downloading release assets") # Find out which filenames are present in multiple versions (need to give them unique names) filenames = [checksum_filename[1] for checksum_filename in fileindex] from collections import Counter filenames_counter = Counter(filenames) # download saves files to current working directory, so we need to temporarily # change working dir to hashalgo_dir folder with cd(hashalgo_dir): for [checksum, filename] in fileindex: filepath = os.path.join(hashalgo_dir, checksum) if not os.path.isfile(filepath): if not github_release.gh_asset_download(repo_name, hashalgo, checksum): logging.error( hashalgo + ": failed to download " + filename + " (" + checksum + ")" ) continue logging.debug( hashalgo + ": downloaded " + filename + " (" + checksum + ")" ) # copying to download folder with real filename if filenames_counter[filename] == 1: # unique filename copyfile(filepath, os.path.join(download_dir, filename)) else: # multiple versions of the filename with different content # add checksum as suffix to distinguish them copyfile( filepath, os.path.join(download_dir, filename + "." + checksum) )
def download_fileindex_csv(repo_name, download_dir, hashalgo, github_token=None): if github_token: github_release._github_token_cli_arg = github_token fileindex_csv = os.path.join(download_dir, hashalgo + ".csv") if os.path.isfile(fileindex_csv): os.remove(fileindex_csv) with cd(download_dir): if not github_release.gh_asset_download(repo_name, hashalgo, hashalgo + ".csv"): raise ValueError("Failed to download " + hashalgo + ".csv") return fileindex_csv
def test_download_all_tag(tmpdir): _download_test_prerequisites(tmpdir) with push_dir(tmpdir): download_count = ghr.gh_asset_download(REPO_NAME, "2.0.0") assert download_count == 3 for asset_name in ["asset_2_a", "asset_2_boo", "asset_2_bar"]: asset_file = tmpdir.join(asset_name) assert asset_file.check() assert asset_file.read() == asset_name
def test_download_tag_pattern(tmpdir): _download_test_prerequisites(tmpdir) with push_dir(tmpdir): download_count = ghr.gh_asset_download(REPO_NAME, tag_name="2.0.0", pattern="*_b*") assert download_count == 2 for asset_name in ["asset_2_boo", "asset_2_bar"]: asset_file = tmpdir.join(asset_name) assert asset_file.check() assert asset_file.read() == asset_name
def test_download_all_pattern(tmpdir): _download_test_prerequisites(tmpdir) with push_dir(tmpdir): download_count = ghr.gh_asset_download(REPO_NAME, pattern="*_b*") assert download_count == 4 for asset_name in [ "asset_1_boo", "asset_1_bar", "asset_2_boo", "asset_2_bar" ]: asset_file = tmpdir.join(asset_name) assert asset_file.check() assert asset_file.read() == asset_name
def download_unpack_assets(repo, tag, asset_name, dest_dir): dest_dir = os.path.abspath(dest_dir) os.makedirs(dest_dir, exist_ok=True) with tempfile.TemporaryDirectory() as tmpdir: curdir = os.getcwd() os.chdir(tmpdir) try: downloaded = github_release.gh_asset_download( repo, tag, asset_name) except: raise else: if not downloaded: raise ValueError( "no assets found for {0!r} with name {1!r}".format( tag, asset_name)) for archive in glob.glob(asset_name): shutil.unpack_archive(archive, dest_dir) finally: os.chdir(curdir)
def download(repo_name, root_dir, download_dir, hashalgo, github_token=None): """Download files associated with HASHALGO release into directory (root_dir)/(hashalgo). List of files is taken from (root_dir)/(hashalgo).csv. If multiple hashes associated with the same filename then the last entry will be used. """ if github_token: github_release._github_token_cli_arg = github_token if not os.path.isdir(download_dir): os.mkdir(download_dir) hashalgo_dir = os.path.join(root_dir, hashalgo) if not os.path.isdir(hashalgo_dir): os.mkdir(hashalgo_dir) hashalgo_csv = download_fileindex_csv(repo_name, hashalgo_dir, hashalgo, github_token) fileindex = read_fileindex_csv(hashalgo_csv) logging.debug(hashalgo + ": downloading release assets") # Find out which filenames are present in multiple versions (need to give them unique names) filenames = [checksum_filename[1] for checksum_filename in fileindex] from collections import Counter # Sort based on filename and filedate fileindex.sort( key=lambda a: (a[COLUMN_FILENAME].casefold(), a[COLUMN_FILEDATE])) filenames_counter = Counter(filenames) # download saves files to current working directory, so we need to temporarily # change working dir to hashalgo_dir folder with cd(hashalgo_dir): fileindex_with_local_filename = [] for fileindex_item in fileindex: checksum = fileindex_item[COLUMN_CHECKSUM] filename = fileindex_item[COLUMN_FILENAME] filedate = fileindex_item[COLUMN_FILEDATE] if len( fileindex_item) > COLUMN_FILEDATE else "" filepath = os.path.join(hashalgo_dir, checksum) if not os.path.isfile(filepath): if not github_release.gh_asset_download( repo_name, hashalgo, checksum): logging.error(hashalgo + ": failed to download " + filename + " (" + checksum + ")") continue logging.debug(hashalgo + ": downloaded " + filename + " (" + checksum + ")") # determine local filename if filenames_counter[filename] == 1: # unique filename local_filename = filename else: # multiple versions of the filename with different content # add checksum as suffix to distinguish them local_filename = filename + "." + checksum local_filepath = os.path.join(download_dir, local_filename) # set file name and date from index copyfile(filepath, local_filepath) set_filedate( local_filepath, date_from_utc_string( filedate if filedate else DEFAULT_FILE_DATE_UTC_STRING)) # save local fileindex fileindex_with_local_filename.append( [checksum, filename, filedate, local_filename]) # Create new hashalgo.csv from existing and incoming files write_fileindex_csv(hashalgo_csv, fileindex) hashalgo_local_md = os.path.join(download_dir, hashalgo + "_local.md") write_fileindex_md(hashalgo_local_md, fileindex_with_local_filename, repo_name, hashalgo, include_local_filename=True)