def esgf_upload(starting_directory, build_list, name, upload_flag=False, prerelease_flag=False, dryrun=False): """Upload binaries to GitHub release as assets.""" if upload_flag is None: upload_flag = query_for_upload() if not upload_flag: return if prerelease_flag: print "Marking as prerelease" print "build list in upload:", build_list for repo in build_list: print "repo:", repo os.chdir(os.path.join(starting_directory, repo)) repo_handle = Repo(os.getcwd()) latest_tag = get_latest_tag(repo_handle) print "latest_tag:", latest_tag if not name: release_name = latest_tag else: release_name = name if latest_tag in get_releases("ESGF/{}".format(repo)): print "Updating the assets for the latest tag {}".format(latest_tag) gh_asset_upload("ESGF/{}".format(repo), latest_tag, "{}/{}/dist/*".format(starting_directory, repo), dry_run=dryrun, verbose=False) else: print "Creating release version {} for {}".format(latest_tag, repo) gh_release_create("ESGF/{}".format(repo), "{}".format(latest_tag), publish=True, name=release_name, prerelease=prerelease_flag, dry_run=dryrun, asset_pattern="{}/{}/dist/*".format(starting_directory, repo)) print "Upload completed!"
def deployBinsToGithub(): global allCfgs, dry_run import github_release as gh repo_name = "OrganicOrchestra/LGML" #checkNotReleased rl = gh.get_releases(repo_name) for r in rl: assert (r['tag_name'] != desiredVersion) tempD = '/tmp/githubExports' if (not os.path.exists(tempD)): os.makedirs(tempD) # print(json.dumps(allCfgs,indent=4)) allAssets = [] for k, v in allCfgs.items(): ext = os.path.splitext(v["local_bin"])[1] if v["local_bin"].endswith(".tar.gz"): ext = ".tar.gz" tmpRenamed = tempD + '/' + v["published_basename"] + ext copy_dry(v["local_bin"], tmpRenamed) allAssets += [tmpRenamed] print(allAssets) gh.gh_release_create(repo_name, desiredVersion, publish=True, asset_pattern=allAssets, name=desiredVersion, body=notes, dry_run=dry_run)
def get_published_releases(repo_name): """Return the versions of the releases that have been published to GitHub.""" published_releases = [ release["name"] for release in get_releases(repo_name) ] print "published_releases:", published_releases return published_releases
def reset(): # Reset to first commit first_sha = run("git log --reverse --pretty=\"%H\"", limit=1) run("git reset --hard %s" % first_sha) run("git push --quiet origin master --force") clear_github_release_and_tags() # Remove local tags for tag in run("git tag"): run("git tag -d %s" % tag) assert len(get_releases(repo_name=REPO_NAME)) == 0
def reset(): pause("We will now reset '%s'" % REPO_NAME) clear_package_directory() # Reset to first commit first_sha = run("git log --reverse --pretty=\"%H\"", limit=1) run("git reset --hard %s" % first_sha) run("git push origin master --force") # Remove release and tags from GitHub run("githubrelease release %s delete *" % REPO_NAME) run("githubrelease ref %s delete --tags *" % REPO_NAME) # Remove local tags for tag in run("git tag"): run("git tag -d %s" % tag) assert len(get_releases(repo_name=REPO_NAME)) == 0
def clear_github_release_and_tags(): @contextmanager def _ignore_4xx_errors(): try: yield except requests.exceptions.HTTPError as exc_info: response = exc_info.response if 400 <= response.status_code < 500: print("Ignoring (%s Client Error: %s for url: %s)" % (response.status_code, response.reason, response.url)) return if sys.version_info[0] >= 3: raise exc_info.with_traceback(sys.exc_info()[2]) else: raise (sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2] ) # noqa: E999 # Remove release and tags from GitHub for release in get_releases(REPO_NAME): with _ignore_4xx_errors(): gh_release_delete(REPO_NAME, release["tag_name"]) with _ignore_4xx_errors(): gh_ref_delete(REPO_NAME, "*", keep_pattern="refs/heads/master")
def clear_github_release_and_tags(): @contextmanager def _ignore_4xx_errors(): try: yield except requests.exceptions.HTTPError as exc_info: response = exc_info.response if 400 <= response.status_code < 500: print("Ignoring (%s Client Error: %s for url: %s)" % ( response.status_code, response.reason, response.url)) return if sys.version_info[0] >= 3: raise exc_info.with_traceback(sys.exc_info()[2]) else: raise (sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2]) # noqa: E999 # Remove release and tags from GitHub for release in get_releases(REPO_NAME): with _ignore_4xx_errors(): gh_release_delete(REPO_NAME, release["tag_name"]) with _ignore_4xx_errors(): gh_ref_delete(REPO_NAME, "*", keep_pattern="refs/heads/master")
def search_and_sign_unsinged(self): """Search through last 'count' releases with assets without .asc counterparts or releases withouth SHA256SUMS.txt.asc """ print('Sign releases on repo: %s' % self.repo) print(' With key: %s, %s\n' % (self.keyid, self.uid)) releases = get_releases(self.repo) if self.tag_name: releases = [r for r in releases if r.get('tag_name', None) == self.tag_name] if len(releases) == 0: print('No release with tag "%s" found, exit' % self.tag_name) sys.exit(1) elif not self.sign_drafts: releases = [r for r in releases if not r.get('draft', False)] # cycle through releases sorted by by publication date releases.sort(key=cmp_to_key(compare_published_times)) for r in releases[:self.count]: tag_name = r.get('tag_name', 'No tag_name') is_draft = r.get('draft', False) is_prerelease = r.get('prerelease', False) created_at = r.get('created_at', '') msg = 'Found %s%s tagged: %s, created at: %s' % ( 'draft ' if is_draft else '', 'prerelease' if is_prerelease else 'release', tag_name, created_at ) if not is_draft: msg += ', published at: %s' % r.get('published_at', '') print(msg) asset_names = [a['name'] for a in r['assets']] if not asset_names: print(' No assets found, skip release\n') continue asc_names = [a for a in asset_names if a.endswith('.asc')] other_names = [a for a in asset_names if not a.endswith('.asc')] need_to_sign = False if asset_names and not asc_names: need_to_sign = True if not need_to_sign: for name in other_names: if not '%s.asc' % name in asc_names: need_to_sign = True break if not need_to_sign: need_to_sign = '%s.asc' % SHA_FNAME not in asc_names if need_to_sign or self.force: self.sign_release(r, other_names, asc_names, r==releases[0]) else: print(' Seems already signed, skip release\n')
def check_releases(expected, releases=None): # noqa: C901 """Return False if expected release data are missing or incorrect. Expected data can be either a dictionary or a list of dictionaries. Supported attributes are tag_name, name, draft, prerelease, package_count, package_pattern, packages and tag_date. * tag_name, name, and body are string * draft and prerelease are boolean * package_count is an integer * packages is a list of strings * package_pattern is either one tuple or a list of tuples of the form (expected_count, pattern). """ def display_error(): print("-" * 80 + "\nERROR:\n") if releases is None: releases = get_releases(REPO_NAME) if type(expected) is list: # Check overall count if len(releases) != len(expected): display_error() print("Numbers of releases is incorrect") print(" expected: %s" % len(expected)) print(" current: %s" % len(releases)) print("") return False # Check each release statuses = [] for _expected in expected: statuses.append(check_releases(_expected, releases)) return reduce(operator.and_, statuses) if statuses else True # Lookup release current_release = None for release in releases: if release["tag_name"] == expected["tag_name"]: current_release = release break if current_release is None: display_error() print("release [%s] not found" % expected["tag_name"]) print("") return False # Check simple attributes for attribute in ["name", "draft", "prerelease", "body"]: if attribute not in expected: continue if attribute not in release: display_error() print("Release [%s] is missing [%s] " "attributes" % (expected["tag_name"], attribute)) display_release("Expected", expected) display_release("Current", release) print("") return False if expected[attribute] != release[attribute]: display_error() print("Release [%s]: attribute [%s] is " "different" % (expected["tag_name"], attribute)) display_release("Expected", expected) display_release("Current", release) print("") return False if "package_count" in expected: current_count = len(release["assets"]) expected_count = expected["package_count"] if current_count != expected_count: display_error() print("Release [%s]: " "Number of packages does not match" % expected["tag_name"]) print(" expected: %s" % expected["package_count"]) print(" current: %s" % current_count) print("") display_package_names(expected, release) return False if "package_pattern" in expected: if "packages" in expected: display_error() print("Release [%s]: attributes 'package_pattern' and 'packages' " "are exclusive. Use only one." % expected["tag_name"]) print("") return False if "package_count" in expected: display_error() print("Release [%s]: attributes 'package_pattern' and " "'package_count' are exclusive. " "Use only one." % expected["tag_name"]) print("") return False patterns = expected["package_pattern"] if type(patterns) is not list: patterns = [patterns] for expected_package_count, pattern in patterns: current_package_count = 0 for package in get_release_packages(release): if fnmatch.fnmatch(package, pattern): current_package_count += 1 continue if expected_package_count != current_package_count: display_error() print("Release [%s]: " "Number of packages associated with pattern [%s] " "does not match" % (expected["tag_name"], pattern)) print(" expected: %s" % expected_package_count) print(" current: %s" % current_package_count) print("") display_package_names(expected, release) print("") return False if "packages" in expected: diff = set(expected["packages"]) & set(get_release_packages(release)) if diff: display_error() print("Release [%s]: " "List of packages names are different" % expected["tag_name"]) display_package_names(expected, release) print("") return False if "tag_date" in expected and release["draft"] is not True: expected_tag_date = expected["tag_date"] release_tag_date = get_commit_date(release["tag_name"]) if expected_tag_date != release_tag_date: display_error() print("Release [%s]: tag dates do not match" % expected["tag_name"]) print(" expected tag_date: %s" % expected_tag_date) print(" current tag_date: %s" % release_tag_date) print("") return False # Check that expected attributes are correct (e.g without typo) for attribute in [ "tag_name", "name", "body", "draft", "prerelease", "package_count", "package_pattern", "packages", "tag_date" ]: expected.pop(attribute, None) if len(expected) > 0: display_error() print("Unknown expected attributes: %s\n" % ", ".join(expected)) return False return True
def test_prerelease_mode(): """In ``prerelease`` mode, the script is expected to create a prerelease only if HEAD is not associated with a tag different from the prerelease tag""" global TEST_CASE TEST_CASE = "test_prerelease_mode" mode = "prerelease" reset() # 2017-01-01 do_commit() # 2017-01-02 do_release("1.0.0") # 2017-01-03 do_commit(version="2.0.0", push=True) # 2017-01-04 publish_github_release(mode) assert (check_releases([ # Release 1.0.0 { "tag_name": "1.0.0", "tag_date": "20170103", "draft": False, "prerelease": False, "package_pattern": (16, "*1.0.0-*.whl") }, # Prerelease { "tag_name": PRERELEASE_TAG, "tag_date": "20170104", "draft": False, "prerelease": True, "package_pattern": (16, "*2.0.0.dev*.whl") } ])) # Get 'published_at' value initial_published_at = [ release["published_at"] for release in get_releases(REPO_NAME) if release["tag_name"] == PRERELEASE_TAG ][0] do_commit(push=True) # 2017-01-05 # # Check that the correct assets are removed after # publishing packages associated with each system. # publish_github_release(mode, system="manylinux1") publish_github_release(mode, system="manylinux1", re_upload=True) assert (check_releases([ # Release 1.0.0 { "tag_name": "1.0.0", "tag_date": "20170103", "draft": False, "prerelease": False, "package_pattern": (16, "*1.0.0-*.whl") }, # Prerelease { "tag_name": PRERELEASE_TAG, "tag_date": "20170105", "draft": False, "prerelease": True, "package_pattern": [(12, "*2.0.0.dev20170104*.whl"), (4, "*2.0.0.dev20170105*.whl")] } ])) final_published_at = [ release["published_at"] for release in get_releases(REPO_NAME) if release["tag_name"] == PRERELEASE_TAG ][0] print("") print("Check that 'published_at' was updated:") print(" initial_published_at: %s" % initial_published_at) print(" final_published_at: %s" % final_published_at) assert initial_published_at != final_published_at publish_github_release(mode, system="macosx") assert (check_releases([ # Release 1.0.0 { "tag_name": "1.0.0", "tag_date": "20170103", "draft": False, "prerelease": False, "package_pattern": (16, "*1.0.0-*.whl") }, # Prerelease { "tag_name": PRERELEASE_TAG, "tag_date": "20170105", "draft": False, "prerelease": True, "package_pattern": [(8, "*2.0.0.dev20170104*.whl"), (8, "*2.0.0.dev20170105*.whl")] } ])) publish_github_release(mode, system="win") assert (check_releases([ # Release 1.0.0 { "tag_name": "1.0.0", "tag_date": "20170103", "draft": False, "prerelease": False, "package_pattern": (16, "*1.0.0-*.whl") }, # Prerelease { "tag_name": PRERELEASE_TAG, "tag_date": "20170105", "draft": False, "prerelease": True, "package_pattern": [(0, "*2.0.0.dev20170104*.whl"), (16, "*2.0.0.dev20170105*.whl")] } ])) do_commit() # 2017-01-06 do_commit(release_tag="2.0.0", push=True) # 2017-01-07 # # Check that no prerelease is created if HEAD is associated # to a tag different from PRERELEASE_TAG # publish_github_release(mode, system="manylinux1") assert (check_releases([ # Release 1.0.0 { "tag_name": "1.0.0", "tag_date": "20170103", "draft": False, "prerelease": False, "package_pattern": (16, "*1.0.0-*.whl") }, # Prerelease { "tag_name": PRERELEASE_TAG, "tag_date": "20170105", "draft": False, "prerelease": True, "package_pattern": (16, "*2.0.0.dev20170105*.whl") } ])) # # Check that re-uploading the same packages does not raise an exception # publish_github_release(mode, system="manylinux1") assert (check_releases([ # Release 1.0.0 { "tag_name": "1.0.0", "tag_date": "20170103", "draft": False, "prerelease": False, "package_pattern": (16, "*1.0.0-*.whl") }, # Prerelease { "tag_name": PRERELEASE_TAG, "tag_date": "20170105", "draft": False, "prerelease": True, "package_pattern": (16, "*2.0.0.dev20170105*.whl") } ]))
def get_latest_release(repo): releases = github_release.get_releases(repo) if not releases: raise ValueError("no releases found for {!r}".format(repo)) return releases[0]
def check_releases(expected, releases=None): # noqa: C901 """Return False if expected release data are missing or incorrect. Expected data can be either a dictionary or a list of dictionaries. Supported attributes are tag_name, name, draft, prerelease, package_count, package_pattern, packages and tag_date. * tag_name, name, and body are string * draft and prerelease are boolean * package_count is an integer * packages is a list of strings * package_pattern is either one tuple or a list of tuples of the form (expected_count, pattern). """ def display_error(): print("-" * 80 + "\nERROR:\n") if releases is None: releases = get_releases(REPO_NAME) if type(expected) is list: # Check overall count if len(releases) != len(expected): display_error() print("Numbers of releases is incorrect") print(" expected: %s" % len(expected)) print(" current: %s" % len(releases)) print("") return False # Check each release statuses = [] for _expected in expected: statuses.append(check_releases(_expected, releases)) return reduce(operator.and_, statuses) if statuses else True # Lookup release current_release = None for release in releases: if release["tag_name"] == expected["tag_name"]: current_release = release break if current_release is None: display_error() print("release [%s] not found" % expected["tag_name"]) print("") return False # Check simple attributes for attribute in ["name", "draft", "prerelease", "body"]: if attribute not in expected: continue if attribute not in release: display_error() print("Release [%s] is missing [%s] " "attributes" % (expected["tag_name"], attribute)) display_release("Expected", expected) display_release("Current", release) print("") return False if expected[attribute] != release[attribute]: display_error() print("Release [%s]: attribute [%s] is " "different" % (expected["tag_name"], attribute)) display_release("Expected", expected) display_release("Current", release) print("") return False if "package_count" in expected: current_count = len(release["assets"]) expected_count = expected["package_count"] if current_count != expected_count: display_error() print("Release [%s]: " "Number of packages does not match" % expected["tag_name"]) print(" expected: %s" % expected["package_count"]) print(" current: %s" % current_count) print("") display_package_names(expected, release) return False if "package_pattern" in expected: if "packages" in expected: display_error() print("Release [%s]: attributes 'package_pattern' and 'packages' " "are exclusive. Use only one." % expected["tag_name"]) print("") return False if "package_count" in expected: display_error() print("Release [%s]: attributes 'package_pattern' and " "'package_count' are exclusive. " "Use only one." % expected["tag_name"]) print("") return False patterns = expected["package_pattern"] if type(patterns) is not list: patterns = [patterns] for expected_package_count, pattern in patterns: current_package_count = 0 for package in get_release_packages(release): if fnmatch.fnmatch(package, pattern): current_package_count += 1 continue if expected_package_count != current_package_count: display_error() print("Release [%s]: " "Number of packages associated with pattern [%s] " "does not match" % (expected["tag_name"], pattern)) print(" expected: %s" % expected_package_count) print(" current: %s" % current_package_count) print("") display_package_names(expected, release) print("") return False if "packages" in expected: diff = set(expected["packages"]) & set(get_release_packages(release)) if diff: display_error() print("Release [%s]: " "List of packages names are different" % expected["tag_name"]) display_package_names(expected, release) print("") return False if "tag_date" in expected and release["draft"] is not True: expected_tag_date = expected["tag_date"] release_tag_date = get_commit_date(release["tag_name"]) if expected_tag_date != release_tag_date: display_error() print("Release [%s]: tag dates do not match" % expected["tag_name"]) print(" expected tag_date: %s" % expected_tag_date) print(" current tag_date: %s" % release_tag_date) print("") return False # Check that expected attributes are correct (e.g without typo) for attribute in [ "tag_name", "name", "body", "draft", "prerelease", "package_count", "package_pattern", "packages", "tag_date"]: expected.pop(attribute, None) if len(expected) > 0: display_error() print("Unknown expected attributes: %s\n" % ", ".join(expected)) return False return True