def esgf_upload(starting_directory, build_list, name, upload_flag=False, prerelease_flag=False, dryrun=False):
"""Upload binaries to GitHub release as assets."""
if upload_flag is None:
upload_flag = query_for_upload()
if not upload_flag:
return
if prerelease_flag:
print "Marking as prerelease"
print "build list in upload:", build_list
for repo in build_list:
print "repo:", repo
os.chdir(os.path.join(starting_directory, repo))
repo_handle = Repo(os.getcwd())
latest_tag = get_latest_tag(repo_handle)
print "latest_tag:", latest_tag
if not name:
release_name = latest_tag
else:
release_name = name
if latest_tag in get_releases("ESGF/{}".format(repo)):
print "Updating the assets for the latest tag {}".format(latest_tag)
gh_asset_upload("ESGF/{}".format(repo), latest_tag, "{}/{}/dist/*".format(starting_directory, repo), dry_run=dryrun, verbose=False)
else:
print "Creating release version {} for {}".format(latest_tag, repo)
gh_release_create("ESGF/{}".format(repo), "{}".format(latest_tag), publish=True, name=release_name, prerelease=prerelease_flag, dry_run=dryrun, asset_pattern="{}/{}/dist/*".format(starting_directory, repo))
print "Upload completed!"
def deployBinsToGithub():
global allCfgs, dry_run
import github_release as gh
repo_name = "OrganicOrchestra/LGML"
#checkNotReleased
rl = gh.get_releases(repo_name)
for r in rl:
assert (r['tag_name'] != desiredVersion)
tempD = '/tmp/githubExports'
if (not os.path.exists(tempD)):
os.makedirs(tempD)
# print(json.dumps(allCfgs,indent=4))
allAssets = []
for k, v in allCfgs.items():
ext = os.path.splitext(v["local_bin"])[1]
if v["local_bin"].endswith(".tar.gz"):
ext = ".tar.gz"
tmpRenamed = tempD + '/' + v["published_basename"] + ext
copy_dry(v["local_bin"], tmpRenamed)
allAssets += [tmpRenamed]
print(allAssets)
gh.gh_release_create(repo_name,
desiredVersion,
publish=True,
asset_pattern=allAssets,
name=desiredVersion,
body=notes,
dry_run=dry_run)
def get_published_releases(repo_name):
"""Return the versions of the releases that have been published to GitHub."""
published_releases = [
release["name"] for release in get_releases(repo_name)
]
print "published_releases:", published_releases
return published_releases
def reset():
# Reset to first commit
first_sha = run("git log --reverse --pretty=\"%H\"", limit=1)
run("git reset --hard %s" % first_sha)
run("git push --quiet origin master --force")
clear_github_release_and_tags()
# Remove local tags
for tag in run("git tag"):
run("git tag -d %s" % tag)
assert len(get_releases(repo_name=REPO_NAME)) == 0
def reset():
# Reset to first commit
first_sha = run("git log --reverse --pretty=\"%H\"", limit=1)
run("git reset --hard %s" % first_sha)
run("git push --quiet origin master --force")
clear_github_release_and_tags()
# Remove local tags
for tag in run("git tag"):
run("git tag -d %s" % tag)
assert len(get_releases(repo_name=REPO_NAME)) == 0
def reset():
pause("We will now reset '%s'" % REPO_NAME)
clear_package_directory()
# Reset to first commit
first_sha = run("git log --reverse --pretty=\"%H\"", limit=1)
run("git reset --hard %s" % first_sha)
run("git push origin master --force")
# Remove release and tags from GitHub
run("githubrelease release %s delete *" % REPO_NAME)
run("githubrelease ref %s delete --tags *" % REPO_NAME)
# Remove local tags
for tag in run("git tag"):
run("git tag -d %s" % tag)
assert len(get_releases(repo_name=REPO_NAME)) == 0
def clear_github_release_and_tags():
@contextmanager
def _ignore_4xx_errors():
try:
yield
except requests.exceptions.HTTPError as exc_info:
response = exc_info.response
if 400 <= response.status_code < 500:
print("Ignoring (%s Client Error: %s for url: %s)" %
(response.status_code, response.reason, response.url))
return
if sys.version_info[0] >= 3:
raise exc_info.with_traceback(sys.exc_info()[2])
else:
raise (sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2]
) # noqa: E999
# Remove release and tags from GitHub
for release in get_releases(REPO_NAME):
with _ignore_4xx_errors():
gh_release_delete(REPO_NAME, release["tag_name"])
with _ignore_4xx_errors():
gh_ref_delete(REPO_NAME, "*", keep_pattern="refs/heads/master")
def clear_github_release_and_tags():
@contextmanager
def _ignore_4xx_errors():
try:
yield
except requests.exceptions.HTTPError as exc_info:
response = exc_info.response
if 400 <= response.status_code < 500:
print("Ignoring (%s Client Error: %s for url: %s)" % (
response.status_code, response.reason, response.url))
return
if sys.version_info[0] >= 3:
raise exc_info.with_traceback(sys.exc_info()[2])
else:
raise (sys.exc_info()[0],
sys.exc_info()[1], sys.exc_info()[2]) # noqa: E999
# Remove release and tags from GitHub
for release in get_releases(REPO_NAME):
with _ignore_4xx_errors():
gh_release_delete(REPO_NAME, release["tag_name"])
with _ignore_4xx_errors():
gh_ref_delete(REPO_NAME, "*", keep_pattern="refs/heads/master")
def search_and_sign_unsinged(self):
"""Search through last 'count' releases with assets without
.asc counterparts or releases withouth SHA256SUMS.txt.asc
"""
print('Sign releases on repo: %s' % self.repo)
print(' With key: %s, %s\n' % (self.keyid, self.uid))
releases = get_releases(self.repo)
if self.tag_name:
releases = [r for r in releases
if r.get('tag_name', None) == self.tag_name]
if len(releases) == 0:
print('No release with tag "%s" found, exit' % self.tag_name)
sys.exit(1)
elif not self.sign_drafts:
releases = [r for r in releases if not r.get('draft', False)]
# cycle through releases sorted by by publication date
releases.sort(key=cmp_to_key(compare_published_times))
for r in releases[:self.count]:
tag_name = r.get('tag_name', 'No tag_name')
is_draft = r.get('draft', False)
is_prerelease = r.get('prerelease', False)
created_at = r.get('created_at', '')
msg = 'Found %s%s tagged: %s, created at: %s' % (
'draft ' if is_draft else '',
'prerelease' if is_prerelease else 'release',
tag_name,
created_at
)
if not is_draft:
msg += ', published at: %s' % r.get('published_at', '')
print(msg)
asset_names = [a['name'] for a in r['assets']]
if not asset_names:
print(' No assets found, skip release\n')
continue
asc_names = [a for a in asset_names if a.endswith('.asc')]
other_names = [a for a in asset_names if not a.endswith('.asc')]
need_to_sign = False
if asset_names and not asc_names:
need_to_sign = True
if not need_to_sign:
for name in other_names:
if not '%s.asc' % name in asc_names:
need_to_sign = True
break
if not need_to_sign:
need_to_sign = '%s.asc' % SHA_FNAME not in asc_names
if need_to_sign or self.force:
self.sign_release(r, other_names, asc_names, r==releases[0])
else:
print(' Seems already signed, skip release\n')
def check_releases(expected, releases=None): # noqa: C901
"""Return False if expected release data are missing or incorrect.
Expected data can be either a dictionary or a list of dictionaries.
Supported attributes are tag_name, name, draft, prerelease, package_count,
package_pattern, packages and tag_date.
* tag_name, name, and body are string
* draft and prerelease are boolean
* package_count is an integer
* packages is a list of strings
* package_pattern is either one tuple or a list of tuples of the
form (expected_count, pattern).
"""
def display_error():
print("-" * 80 + "\nERROR:\n")
if releases is None:
releases = get_releases(REPO_NAME)
if type(expected) is list:
# Check overall count
if len(releases) != len(expected):
display_error()
print("Numbers of releases is incorrect")
print(" expected: %s" % len(expected))
print(" current: %s" % len(releases))
print("")
return False
# Check each release
statuses = []
for _expected in expected:
statuses.append(check_releases(_expected, releases))
return reduce(operator.and_, statuses) if statuses else True
# Lookup release
current_release = None
for release in releases:
if release["tag_name"] == expected["tag_name"]:
current_release = release
break
if current_release is None:
display_error()
print("release [%s] not found" % expected["tag_name"])
print("")
return False
# Check simple attributes
for attribute in ["name", "draft", "prerelease", "body"]:
if attribute not in expected:
continue
if attribute not in release:
display_error()
print("Release [%s] is missing [%s] "
"attributes" % (expected["tag_name"], attribute))
display_release("Expected", expected)
display_release("Current", release)
print("")
return False
if expected[attribute] != release[attribute]:
display_error()
print("Release [%s]: attribute [%s] is "
"different" % (expected["tag_name"], attribute))
display_release("Expected", expected)
display_release("Current", release)
print("")
return False
if "package_count" in expected:
current_count = len(release["assets"])
expected_count = expected["package_count"]
if current_count != expected_count:
display_error()
print("Release [%s]: "
"Number of packages does not match" % expected["tag_name"])
print(" expected: %s" % expected["package_count"])
print(" current: %s" % current_count)
print("")
display_package_names(expected, release)
return False
if "package_pattern" in expected:
if "packages" in expected:
display_error()
print("Release [%s]: attributes 'package_pattern' and 'packages' "
"are exclusive. Use only one." % expected["tag_name"])
print("")
return False
if "package_count" in expected:
display_error()
print("Release [%s]: attributes 'package_pattern' and "
"'package_count' are exclusive. "
"Use only one." % expected["tag_name"])
print("")
return False
patterns = expected["package_pattern"]
if type(patterns) is not list:
patterns = [patterns]
for expected_package_count, pattern in patterns:
current_package_count = 0
for package in get_release_packages(release):
if fnmatch.fnmatch(package, pattern):
current_package_count += 1
continue
if expected_package_count != current_package_count:
display_error()
print("Release [%s]: "
"Number of packages associated with pattern [%s] "
"does not match" % (expected["tag_name"], pattern))
print(" expected: %s" % expected_package_count)
print(" current: %s" % current_package_count)
print("")
display_package_names(expected, release)
print("")
return False
if "packages" in expected:
diff = set(expected["packages"]) & set(get_release_packages(release))
if diff:
display_error()
print("Release [%s]: "
"List of packages names are different" %
expected["tag_name"])
display_package_names(expected, release)
print("")
return False
if "tag_date" in expected and release["draft"] is not True:
expected_tag_date = expected["tag_date"]
release_tag_date = get_commit_date(release["tag_name"])
if expected_tag_date != release_tag_date:
display_error()
print("Release [%s]: tag dates do not match" %
expected["tag_name"])
print(" expected tag_date: %s" % expected_tag_date)
print(" current tag_date: %s" % release_tag_date)
print("")
return False
# Check that expected attributes are correct (e.g without typo)
for attribute in [
"tag_name", "name", "body", "draft", "prerelease", "package_count",
"package_pattern", "packages", "tag_date"
]:
expected.pop(attribute, None)
if len(expected) > 0:
display_error()
print("Unknown expected attributes: %s\n" % ", ".join(expected))
return False
return True
def test_prerelease_mode():
"""In ``prerelease`` mode, the script is expected to create a
prerelease only if HEAD is not associated with a tag different
from the prerelease tag"""
global TEST_CASE
TEST_CASE = "test_prerelease_mode"
mode = "prerelease"
reset() # 2017-01-01
do_commit() # 2017-01-02
do_release("1.0.0") # 2017-01-03
do_commit(version="2.0.0", push=True) # 2017-01-04
publish_github_release(mode)
assert (check_releases([
# Release 1.0.0
{
"tag_name": "1.0.0",
"tag_date": "20170103",
"draft": False,
"prerelease": False,
"package_pattern": (16, "*1.0.0-*.whl")
},
# Prerelease
{
"tag_name": PRERELEASE_TAG,
"tag_date": "20170104",
"draft": False,
"prerelease": True,
"package_pattern": (16, "*2.0.0.dev*.whl")
}
]))
# Get 'published_at' value
initial_published_at = [
release["published_at"] for release in get_releases(REPO_NAME)
if release["tag_name"] == PRERELEASE_TAG
][0]
do_commit(push=True) # 2017-01-05
#
# Check that the correct assets are removed after
# publishing packages associated with each system.
#
publish_github_release(mode, system="manylinux1")
publish_github_release(mode, system="manylinux1", re_upload=True)
assert (check_releases([
# Release 1.0.0
{
"tag_name": "1.0.0",
"tag_date": "20170103",
"draft": False,
"prerelease": False,
"package_pattern": (16, "*1.0.0-*.whl")
},
# Prerelease
{
"tag_name":
PRERELEASE_TAG,
"tag_date":
"20170105",
"draft":
False,
"prerelease":
True,
"package_pattern": [(12, "*2.0.0.dev20170104*.whl"),
(4, "*2.0.0.dev20170105*.whl")]
}
]))
final_published_at = [
release["published_at"] for release in get_releases(REPO_NAME)
if release["tag_name"] == PRERELEASE_TAG
][0]
print("")
print("Check that 'published_at' was updated:")
print(" initial_published_at: %s" % initial_published_at)
print(" final_published_at: %s" % final_published_at)
assert initial_published_at != final_published_at
publish_github_release(mode, system="macosx")
assert (check_releases([
# Release 1.0.0
{
"tag_name": "1.0.0",
"tag_date": "20170103",
"draft": False,
"prerelease": False,
"package_pattern": (16, "*1.0.0-*.whl")
},
# Prerelease
{
"tag_name":
PRERELEASE_TAG,
"tag_date":
"20170105",
"draft":
False,
"prerelease":
True,
"package_pattern": [(8, "*2.0.0.dev20170104*.whl"),
(8, "*2.0.0.dev20170105*.whl")]
}
]))
publish_github_release(mode, system="win")
assert (check_releases([
# Release 1.0.0
{
"tag_name": "1.0.0",
"tag_date": "20170103",
"draft": False,
"prerelease": False,
"package_pattern": (16, "*1.0.0-*.whl")
},
# Prerelease
{
"tag_name":
PRERELEASE_TAG,
"tag_date":
"20170105",
"draft":
False,
"prerelease":
True,
"package_pattern": [(0, "*2.0.0.dev20170104*.whl"),
(16, "*2.0.0.dev20170105*.whl")]
}
]))
do_commit() # 2017-01-06
do_commit(release_tag="2.0.0", push=True) # 2017-01-07
#
# Check that no prerelease is created if HEAD is associated
# to a tag different from PRERELEASE_TAG
#
publish_github_release(mode, system="manylinux1")
assert (check_releases([
# Release 1.0.0
{
"tag_name": "1.0.0",
"tag_date": "20170103",
"draft": False,
"prerelease": False,
"package_pattern": (16, "*1.0.0-*.whl")
},
# Prerelease
{
"tag_name": PRERELEASE_TAG,
"tag_date": "20170105",
"draft": False,
"prerelease": True,
"package_pattern": (16, "*2.0.0.dev20170105*.whl")
}
]))
#
# Check that re-uploading the same packages does not raise an exception
#
publish_github_release(mode, system="manylinux1")
assert (check_releases([
# Release 1.0.0
{
"tag_name": "1.0.0",
"tag_date": "20170103",
"draft": False,
"prerelease": False,
"package_pattern": (16, "*1.0.0-*.whl")
},
# Prerelease
{
"tag_name": PRERELEASE_TAG,
"tag_date": "20170105",
"draft": False,
"prerelease": True,
"package_pattern": (16, "*2.0.0.dev20170105*.whl")
}
]))
def get_latest_release(repo):
releases = github_release.get_releases(repo)
if not releases:
raise ValueError("no releases found for {!r}".format(repo))
return releases[0]
def check_releases(expected, releases=None): # noqa: C901
"""Return False if expected release data are missing or incorrect.
Expected data can be either a dictionary or a list of dictionaries.
Supported attributes are tag_name, name, draft, prerelease, package_count,
package_pattern, packages and tag_date.
* tag_name, name, and body are string
* draft and prerelease are boolean
* package_count is an integer
* packages is a list of strings
* package_pattern is either one tuple or a list of tuples of the
form (expected_count, pattern).
"""
def display_error():
print("-" * 80 + "\nERROR:\n")
if releases is None:
releases = get_releases(REPO_NAME)
if type(expected) is list:
# Check overall count
if len(releases) != len(expected):
display_error()
print("Numbers of releases is incorrect")
print(" expected: %s" % len(expected))
print(" current: %s" % len(releases))
print("")
return False
# Check each release
statuses = []
for _expected in expected:
statuses.append(check_releases(_expected, releases))
return reduce(operator.and_, statuses) if statuses else True
# Lookup release
current_release = None
for release in releases:
if release["tag_name"] == expected["tag_name"]:
current_release = release
break
if current_release is None:
display_error()
print("release [%s] not found" % expected["tag_name"])
print("")
return False
# Check simple attributes
for attribute in ["name", "draft", "prerelease", "body"]:
if attribute not in expected:
continue
if attribute not in release:
display_error()
print("Release [%s] is missing [%s] "
"attributes" % (expected["tag_name"], attribute))
display_release("Expected", expected)
display_release("Current", release)
print("")
return False
if expected[attribute] != release[attribute]:
display_error()
print("Release [%s]: attribute [%s] is "
"different" % (expected["tag_name"], attribute))
display_release("Expected", expected)
display_release("Current", release)
print("")
return False
if "package_count" in expected:
current_count = len(release["assets"])
expected_count = expected["package_count"]
if current_count != expected_count:
display_error()
print("Release [%s]: "
"Number of packages does not match" % expected["tag_name"])
print(" expected: %s" % expected["package_count"])
print(" current: %s" % current_count)
print("")
display_package_names(expected, release)
return False
if "package_pattern" in expected:
if "packages" in expected:
display_error()
print("Release [%s]: attributes 'package_pattern' and 'packages' "
"are exclusive. Use only one." % expected["tag_name"])
print("")
return False
if "package_count" in expected:
display_error()
print("Release [%s]: attributes 'package_pattern' and "
"'package_count' are exclusive. "
"Use only one." % expected["tag_name"])
print("")
return False
patterns = expected["package_pattern"]
if type(patterns) is not list:
patterns = [patterns]
for expected_package_count, pattern in patterns:
current_package_count = 0
for package in get_release_packages(release):
if fnmatch.fnmatch(package, pattern):
current_package_count += 1
continue
if expected_package_count != current_package_count:
display_error()
print("Release [%s]: "
"Number of packages associated with pattern [%s] "
"does not match" % (expected["tag_name"], pattern))
print(" expected: %s" % expected_package_count)
print(" current: %s" % current_package_count)
print("")
display_package_names(expected, release)
print("")
return False
if "packages" in expected:
diff = set(expected["packages"]) & set(get_release_packages(release))
if diff:
display_error()
print("Release [%s]: "
"List of packages names are different" % expected["tag_name"])
display_package_names(expected, release)
print("")
return False
if "tag_date" in expected and release["draft"] is not True:
expected_tag_date = expected["tag_date"]
release_tag_date = get_commit_date(release["tag_name"])
if expected_tag_date != release_tag_date:
display_error()
print("Release [%s]: tag dates do not match" % expected["tag_name"])
print(" expected tag_date: %s" % expected_tag_date)
print(" current tag_date: %s" % release_tag_date)
print("")
return False
# Check that expected attributes are correct (e.g without typo)
for attribute in [
"tag_name", "name", "body", "draft", "prerelease",
"package_count", "package_pattern", "packages",
"tag_date"]:
expected.pop(attribute, None)
if len(expected) > 0:
display_error()
print("Unknown expected attributes: %s\n" % ", ".join(expected))
return False
return True
def search_and_sign_unsinged(self):
"""Search through last 'count' releases with assets without
.asc counterparts or releases withouth SHA256SUMS.txt.asc
"""
print('Sign releases on repo: %s' % self.repo)
print(' With key: %s, %s\n' % (self.keyid, self.uid))
releases = get_releases(self.repo)
if self.tag_name:
releases = [r for r in releases
if r.get('tag_name', None) == self.tag_name]
if len(releases) == 0:
print('No release with tag "%s" found, exit' % self.tag_name)
sys.exit(1)
elif not self.sign_drafts:
releases = [r for r in releases if not r.get('draft', False)]
# cycle through releases sorted by by publication date
releases.sort(key=cmp_to_key(compare_published_times))
for r in releases[:self.count]:
tag_name = r.get('tag_name', 'No tag_name')
is_draft = r.get('draft', False)
is_prerelease = r.get('prerelease', False)
created_at = r.get('created_at', '')
msg = 'Found %s%s tagged: %s, created at: %s' % (
'draft ' if is_draft else '',
'prerelease' if is_prerelease else 'release',
tag_name,
created_at
)
if not is_draft:
msg += ', published at: %s' % r.get('published_at', '')
print(msg)
asset_names = [a['name'] for a in r['assets']]
if not asset_names:
print(' No assets found, skip release\n')
continue
asc_names = [a for a in asset_names if a.endswith('.asc')]
other_names = [a for a in asset_names if not a.endswith('.asc')]
need_to_sign = False
if asset_names and not asc_names:
need_to_sign = True
if not need_to_sign:
for name in other_names:
if not '%s.asc' % name in asc_names:
need_to_sign = True
break
if not need_to_sign:
need_to_sign = '%s.asc' % SHA_FNAME not in asc_names
if need_to_sign or self.force:
self.sign_release(r, other_names, asc_names, r==releases[0])
else:
print(' Seems already signed, skip release\n')
