def extract(repo="DOE-ICoM/icom-mesh-data", vtag="v0.1.0"):
"""
EXTRACT: download and unpack data assets for icom-mesh.
"""
# Authors: Darren Engwirda
cwd_pointer = os.getcwd()
try:
print("Extracting to:", os.path.join(HERE, "data"))
print("")
directory = os.path.join(HERE, "data")
os.chdir(directory)
gh_asset_download(repo, vtag)
for item in os.listdir(directory):
if item.endswith(".zip"):
file_name = os.path.abspath(item)
zip_func = zipfile.ZipFile(file_name)
zip_func.extractall(directory)
zip_func.close()
print("")
print("Done downloading + unpacking assets.")
print("")
finally:
os.chdir(cwd_pointer)
return
def sign_release(self, release, other_names, asc_names, is_newest_release):
"""Download/sign unsigned assets, upload .asc counterparts.
Create SHA256SUMS.txt with all assets included and upload it
with SHA256SUMS.txt.asc counterpart.
"""
repo = self.repo
tag = release.get('tag_name', None)
if not tag:
print('Release have no tag name, skip release\n')
return
with ChdirTemporaryDirectory() as tmpdir:
with open(SHA_FNAME, 'w') as fdw:
sdist_match = None
for name in other_names:
if name == SHA_FNAME:
continue
gh_asset_download(repo, tag, name)
if not self.no_ppa:
sdist_match = sdist_match \
or SDIST_NAME_PATTERN.match(name)
apk_match = UNSIGNED_APK_PATTERN.match(name)
if apk_match:
unsigned_name = name
name = self.sign_apk(unsigned_name, apk_match.group(1))
gh_asset_upload(repo, tag, name, dry_run=self.dry_run)
gh_asset_delete(repo, tag, unsigned_name,
dry_run=self.dry_run)
if not '%s.asc' % name in asc_names or self.force:
self.sign_file_name(name)
if self.force:
gh_asset_delete(repo, tag, '%s.asc' % name,
dry_run=self.dry_run)
gh_asset_upload(repo, tag, '%s.asc' % name,
dry_run=self.dry_run)
sumline = '%s %s\n' % (sha256_checksum(name), name)
fdw.write(sumline)
self.sign_file_name(SHA_FNAME, detach=False)
gh_asset_delete(repo, tag, '%s.asc' % SHA_FNAME,
dry_run=self.dry_run)
gh_asset_upload(repo, tag, '%s.asc' % SHA_FNAME,
dry_run=self.dry_run)
if sdist_match and is_newest_release:
self.make_ppa(sdist_match, tmpdir, tag)
def sign_release(self, release, other_names, asc_names, is_newest_release):
"""Download/sign unsigned assets, upload .asc counterparts.
Create SHA256SUMS.txt with all assets included and upload it
with SHA256SUMS.txt.asc counterpart.
"""
repo = self.repo
tag = release.get('tag_name', None)
if not tag:
print('Release have no tag name, skip release\n')
return
with ChdirTemporaryDirectory() as tmpdir:
with open(SHA_FNAME, 'w') as fdw:
sdist_match = None
for name in other_names:
if name == SHA_FNAME:
continue
gh_asset_download(repo, tag, name)
if not self.no_ppa:
sdist_match = sdist_match \
or SDIST_NAME_PATTERN.match(name)
apk_match = UNSIGNED_APK_PATTERN.match(name)
if apk_match:
unsigned_name = name
name = self.sign_apk(unsigned_name, apk_match.group(1))
gh_asset_upload(repo, tag, name, dry_run=self.dry_run)
gh_asset_delete(repo, tag, unsigned_name,
dry_run=self.dry_run)
if not '%s.asc' % name in asc_names or self.force:
self.sign_file_name(name)
if self.force:
gh_asset_delete(repo, tag, '%s.asc' % name,
dry_run=self.dry_run)
gh_asset_upload(repo, tag, '%s.asc' % name,
dry_run=self.dry_run)
sumline = '%s %s\n' % (sha256_checksum(name), name)
fdw.write(sumline)
self.sign_file_name(SHA_FNAME, detach=False)
gh_asset_delete(repo, tag, '%s.asc' % SHA_FNAME,
dry_run=self.dry_run)
gh_asset_upload(repo, tag, '%s.asc' % SHA_FNAME,
dry_run=self.dry_run)
if sdist_match and is_newest_release:
self.make_ppa(sdist_match, tmpdir, tag)
def test_download_twice(tmpdir):
_download_test_prerequisites(tmpdir)
with push_dir(tmpdir):
download_count = ghr.gh_asset_download(
REPO_NAME, tag_name="1.0.0", pattern="asset_1_a"
)
assert download_count == 1
download_count = ghr.gh_asset_download(
REPO_NAME, tag_name="1.0.0", pattern="asset_1_a"
)
assert download_count == 0
def test_download_twice(tmpdir):
_download_test_prerequisites(tmpdir)
with push_dir(tmpdir):
download_count = ghr.gh_asset_download(REPO_NAME,
tag_name="1.0.0",
pattern="asset_1_a")
assert download_count == 1
download_count = ghr.gh_asset_download(REPO_NAME,
tag_name="1.0.0",
pattern="asset_1_a")
assert download_count == 0
def sign_release(self, release, other_names, asc_names):
"""Download/sign unsigned assets, upload .asc counterparts.
Create SHA256SUMS.txt with all assets included and upload it
with SHA256SUMS.txt.asc counterpart.
"""
repo = self.repo
tag = release.get('tag_name', None)
if not tag:
print 'Release have no tag name, skip release\n'
return
with ChdirTemporaryDirectory():
with open(SHA_FNAME, 'w') as fdw:
for name in other_names:
if name == SHA_FNAME:
continue
gh_asset_download(repo, tag, name)
if not '%s.asc' % name in asc_names or self.force:
self.sign_file_name(name)
if self.force:
gh_asset_delete(repo,
tag,
'%s.asc' % name,
dry_run=self.dry_run)
gh_asset_upload(repo,
tag,
'%s.asc' % name,
dry_run=self.dry_run)
sumline = '%s %s\n' % (sha256_checksum(name), name)
fdw.write(sumline)
self.sign_file_name(SHA_FNAME, detach=False)
gh_asset_delete(repo,
tag,
'%s.asc' % SHA_FNAME,
dry_run=self.dry_run)
gh_asset_upload(repo,
tag,
'%s.asc' % SHA_FNAME,
dry_run=self.dry_run)
def download(repo_name, root_dir, download_dir, hashalgo, github_token=None):
"""Download files associated with HASHALGO release into directory (root_dir)/(hashalgo).
List of files is taken from (root_dir)/(hashalgo).csv. If multiple hashes associated with
the same filename then the last entry will be used.
"""
if github_token:
github_release._github_token_cli_arg = github_token
if not os.path.isdir(download_dir):
os.mkdir(download_dir)
hashalgo_dir = os.path.join(root_dir, hashalgo)
if not os.path.isdir(hashalgo_dir):
os.mkdir(hashalgo_dir)
hashalgo_csv = download_fileindex_csv(
repo_name, hashalgo_dir, hashalgo, github_token
)
fileindex = read_fileindex_csv(hashalgo_csv)
logging.debug(hashalgo + ": downloading release assets")
# Find out which filenames are present in multiple versions (need to give them unique names)
filenames = [checksum_filename[1] for checksum_filename in fileindex]
from collections import Counter
filenames_counter = Counter(filenames)
# download saves files to current working directory, so we need to temporarily
# change working dir to hashalgo_dir folder
with cd(hashalgo_dir):
for [checksum, filename] in fileindex:
filepath = os.path.join(hashalgo_dir, checksum)
if not os.path.isfile(filepath):
if not github_release.gh_asset_download(repo_name, hashalgo, checksum):
logging.error(
hashalgo
+ ": failed to download "
+ filename
+ " ("
+ checksum
+ ")"
)
continue
logging.debug(
hashalgo + ": downloaded " + filename + " (" + checksum + ")"
)
# copying to download folder with real filename
if filenames_counter[filename] == 1:
# unique filename
copyfile(filepath, os.path.join(download_dir, filename))
else:
# multiple versions of the filename with different content
# add checksum as suffix to distinguish them
copyfile(
filepath, os.path.join(download_dir, filename + "." + checksum)
)
def download_fileindex_csv(repo_name, download_dir, hashalgo, github_token=None):
if github_token:
github_release._github_token_cli_arg = github_token
fileindex_csv = os.path.join(download_dir, hashalgo + ".csv")
if os.path.isfile(fileindex_csv):
os.remove(fileindex_csv)
with cd(download_dir):
if not github_release.gh_asset_download(repo_name, hashalgo, hashalgo + ".csv"):
raise ValueError("Failed to download " + hashalgo + ".csv")
return fileindex_csv
def test_download_all_tag(tmpdir):
_download_test_prerequisites(tmpdir)
with push_dir(tmpdir):
download_count = ghr.gh_asset_download(REPO_NAME, "2.0.0")
assert download_count == 3
for asset_name in ["asset_2_a", "asset_2_boo", "asset_2_bar"]:
asset_file = tmpdir.join(asset_name)
assert asset_file.check()
assert asset_file.read() == asset_name
def test_download_all_tag(tmpdir):
_download_test_prerequisites(tmpdir)
with push_dir(tmpdir):
download_count = ghr.gh_asset_download(REPO_NAME, "2.0.0")
assert download_count == 3
for asset_name in ["asset_2_a", "asset_2_boo", "asset_2_bar"]:
asset_file = tmpdir.join(asset_name)
assert asset_file.check()
assert asset_file.read() == asset_name
def test_download_tag_pattern(tmpdir):
_download_test_prerequisites(tmpdir)
with push_dir(tmpdir):
download_count = ghr.gh_asset_download(REPO_NAME,
tag_name="2.0.0",
pattern="*_b*")
assert download_count == 2
for asset_name in ["asset_2_boo", "asset_2_bar"]:
asset_file = tmpdir.join(asset_name)
assert asset_file.check()
assert asset_file.read() == asset_name
def test_download_all_pattern(tmpdir):
_download_test_prerequisites(tmpdir)
with push_dir(tmpdir):
download_count = ghr.gh_asset_download(REPO_NAME, pattern="*_b*")
assert download_count == 4
for asset_name in [
"asset_1_boo", "asset_1_bar",
"asset_2_boo", "asset_2_bar"
]:
asset_file = tmpdir.join(asset_name)
assert asset_file.check()
assert asset_file.read() == asset_name
def download_unpack_assets(repo, tag, asset_name, dest_dir):
dest_dir = os.path.abspath(dest_dir)
os.makedirs(dest_dir, exist_ok=True)
with tempfile.TemporaryDirectory() as tmpdir:
curdir = os.getcwd()
os.chdir(tmpdir)
try:
downloaded = github_release.gh_asset_download(
repo, tag, asset_name)
except:
raise
else:
if not downloaded:
raise ValueError(
"no assets found for {0!r} with name {1!r}".format(
tag, asset_name))
for archive in glob.glob(asset_name):
shutil.unpack_archive(archive, dest_dir)
finally:
os.chdir(curdir)
def download(repo_name, root_dir, download_dir, hashalgo, github_token=None):
"""Download files associated with HASHALGO release into directory (root_dir)/(hashalgo).
List of files is taken from (root_dir)/(hashalgo).csv. If multiple hashes associated with
the same filename then the last entry will be used.
"""
if github_token:
github_release._github_token_cli_arg = github_token
if not os.path.isdir(download_dir):
os.mkdir(download_dir)
hashalgo_dir = os.path.join(root_dir, hashalgo)
if not os.path.isdir(hashalgo_dir):
os.mkdir(hashalgo_dir)
hashalgo_csv = download_fileindex_csv(repo_name, hashalgo_dir, hashalgo,
github_token)
fileindex = read_fileindex_csv(hashalgo_csv)
logging.debug(hashalgo + ": downloading release assets")
# Find out which filenames are present in multiple versions (need to give them unique names)
filenames = [checksum_filename[1] for checksum_filename in fileindex]
from collections import Counter
# Sort based on filename and filedate
fileindex.sort(
key=lambda a: (a[COLUMN_FILENAME].casefold(), a[COLUMN_FILEDATE]))
filenames_counter = Counter(filenames)
# download saves files to current working directory, so we need to temporarily
# change working dir to hashalgo_dir folder
with cd(hashalgo_dir):
fileindex_with_local_filename = []
for fileindex_item in fileindex:
checksum = fileindex_item[COLUMN_CHECKSUM]
filename = fileindex_item[COLUMN_FILENAME]
filedate = fileindex_item[COLUMN_FILEDATE] if len(
fileindex_item) > COLUMN_FILEDATE else ""
filepath = os.path.join(hashalgo_dir, checksum)
if not os.path.isfile(filepath):
if not github_release.gh_asset_download(
repo_name, hashalgo, checksum):
logging.error(hashalgo + ": failed to download " +
filename + " (" + checksum + ")")
continue
logging.debug(hashalgo + ": downloaded " + filename + " (" +
checksum + ")")
# determine local filename
if filenames_counter[filename] == 1:
# unique filename
local_filename = filename
else:
# multiple versions of the filename with different content
# add checksum as suffix to distinguish them
local_filename = filename + "." + checksum
local_filepath = os.path.join(download_dir, local_filename)
# set file name and date from index
copyfile(filepath, local_filepath)
set_filedate(
local_filepath,
date_from_utc_string(
filedate if filedate else DEFAULT_FILE_DATE_UTC_STRING))
# save local fileindex
fileindex_with_local_filename.append(
[checksum, filename, filedate, local_filename])
# Create new hashalgo.csv from existing and incoming files
write_fileindex_csv(hashalgo_csv, fileindex)
hashalgo_local_md = os.path.join(download_dir, hashalgo + "_local.md")
write_fileindex_md(hashalgo_local_md,
fileindex_with_local_filename,
repo_name,
hashalgo,
include_local_filename=True)
