def main(argv):
pars = get_parser()
opts = argparse.Namespace()
pars_config(opts, 'setup.cfg')
opts = pars.parse_args(argv, namespace=opts)
logger.info(f'Final namespace: {opts}')
tag = os.getenv(opts.tagvar, '')
if not tag:
logger.error(f'No tag found for {opts.tagvar}')
return 1
logger.info(f'envar {opts.tagvar} resolved to {tag}')
nicetag = tag
pkgname = None
if '@' in tag:
logger.info('@ delimited tag, splitting into name and tag part')
pkgname, nicetag = tag.split('@')
logger.info(f'Got pkgname={pkgname} @ nicetag={nicetag}')
m = re.search(SEMVER_RE, nicetag)
if not m:
logger.error(f'nicetag={nicetag} does not match semver regex')
return 1
is_prerelease = False
if m.groupdict().get('pre'):
is_prerelease = True
defvalu = ''
if not opts.dryrun:
envd = os.getenv('VTX_COMMON_DRYRUN', None)
if envd is not None:
logger.info(
'Setting opts.dryrun to True from environment variable')
opts.dryrun = True
if opts.dryrun:
defvalu = 'DRYRUN'
gh_token = os.getenv(opts.gittokenvar, defvalu)
if not gh_token:
logger.error('No github token found')
return 1
gh_username = os.getenv(opts.gituservar, defvalu)
if not gh_username:
logger.error('No github user found')
return 1
gh_repo = os.getenv(opts.gitrepovar, defvalu)
if not gh_repo:
logger.error('No github repo found')
return 1
extra_parts = []
pfile = opts.storm_pkg_file
mtyp = opts.storm_pkg_type
if pfile and mtyp:
if opts.storm_pkg_file_pkgname:
assert pkgname is not None
logger.info('Injecting pkgname to pkg path')
pfile = pfile.format(pkgname=pkgname)
if opts.storm_pkg_type_pkgname:
assert pkgname is not None
logger.info('Injecting pkgname to pkg mtype')
mtyp = mtyp.format(pkgname=pkgname)
logger.info(f'Getting storm package from {pfile}')
pkg = v_gpsm.yamlload(pfile)
minv_message = v_gpsm.getMessageFromPkg(pkg, mtyp)
logger.info(f'Got message: {minv_message}')
extra_parts.append(minv_message)
extra_lines = opts.extra_lines
if extra_lines:
logger.info(f'Extra lines found: {extra_lines}')
extra_parts.append(extra_lines)
# Join extra lines together
extra_lines = '\n'.join(extra_parts)
changelog_fp = opts.changelog
if opts.changelog_pkgname:
assert pkgname is not None
logger.info('Injecting pkgname to changelog path')
changelog_fp = changelog_fp.format(pkgname=pkgname)
assert os.path.isfile(changelog_fp)
raw_changelog = open(changelog_fp, 'rb').read().decode()
parsed_logs = parse_changelog(raw_changelog)
target_log = parsed_logs.get(nicetag)
if not target_log:
logger.error(f'Unable to find logs for tag [{nicetag}]')
# It's possible for pre-release tags to end up without a changelog.
# This condition should not end up failing a CI pipeline.
return 0
logger.info(f'Found changelogs for [{nicetag}] in [{changelog_fp}]')
if opts.remove_urls:
logger.info('Removing URLs')
target_log = remove_urls(target_log)
# join logs together and strip them
target_log = '\n'.join(target_log)
target_log = target_log.strip()
if extra_lines:
logger.info(f'Appending extra line data')
target_log = '\n'.join([target_log, '', extra_lines])
# remove trailing data if present in extra_lines
target_log = target_log.strip()
logger.info('Final Log:')
for line in target_log.split('\n'):
logger.debug(line)
name = nicetag
if opts.release_name:
name = f'{opts.release_name} {nicetag}'
if opts.release_name_pkgname:
logger.info(f'Injecting pkgname into [{name}]')
name = name.format(pkgname=pkgname)
logger.info(f'Name is now [{name}]')
logger.info(f'Release Name: [{name}]')
gh_repo_path = f'{gh_username}/{gh_repo}'
if opts.dryrun:
logger.info(
'Dry-run mode enabled. Not performing a Github release action.')
logger.info('Would have made release with the following information:')
logger.info(f'gh_repo_path={gh_repo_path}')
logger.info(f'tag={tag}')
logger.info(f'name={name}')
logger.info(f'prerelease={is_prerelease}')
logger.info(f'message={target_log}')
return 0
gh = github.Github(gh_token)
logger.info(f'Getting github repo for {gh_repo_path}')
repo = gh.get_repo(gh_repo_path)
logger.info('Making github release')
release = repo.create_git_release(
tag=tag,
name=name,
draft=False,
message=target_log,
prerelease=is_prerelease,
)
logger.info(f'Made github release {release}')
return 0
def pr_detailed_comment(org_name, repo_name, pr_owner, pr_repo, pr_branch,
pr_num, comment):
is_staged_recipes = (repo_name == "staged-extensions")
if not (repo_name.endswith("-feedstock") or is_staged_recipes):
return
if not is_staged_recipes and UPDATE_CIRCLECI_KEY_MSG.search(comment):
update_circle(org_name, repo_name)
gh = github.Github(os.environ['GH_TOKEN'])
repo = gh.get_repo("{}/{}".format(org_name, repo_name))
pull = repo.get_pull(int(pr_num))
message = textwrap.dedent("""
Hi! This is the friendly automated nwb-extensions-webservice.
I just wanted to let you know that I updated the circle-ci deploy key and followed the project.
""")
pull.create_issue_comment(message)
pr_commands = [LINT_MSG]
if not is_staged_recipes:
pr_commands += [RERENDER_MSG]
if not any(command.search(comment) for command in pr_commands):
return
with tmp_directory() as tmp_dir:
print(tmp_dir, repo_name)
feedstock_dir = os.path.join(tmp_dir, repo_name)
repo_url = "https://{}@github.com/{}/{}.git".format(
os.environ['GH_TOKEN'], pr_owner, pr_repo)
repo = Repo.clone_from(repo_url, feedstock_dir, branch=pr_branch)
if LINT_MSG.search(comment):
relint(org_name, repo_name, pr_num)
changed_anything = False
rerender_error = False
expected_changes = []
if not is_staged_recipes:
do_rerender = False
if RERENDER_MSG.search(comment):
do_rerender = True
expected_changes.append('re-render')
if do_rerender:
try:
changed_anything |= rerender(repo)
except RuntimeError:
rerender_error = True
if expected_changes:
if len(expected_changes) > 1:
expected_changes[-1] = 'and ' + expected_changes[-1]
joiner = ", " if len(expected_changes) > 2 else " "
changes_str = joiner.join(expected_changes)
gh = github.Github(os.environ['GH_TOKEN'])
gh_repo = gh.get_repo("{}/{}".format(org_name, repo_name))
pull = gh_repo.get_pull(int(pr_num))
if changed_anything:
try:
repo.remotes.origin.push()
except GitCommandError:
message = textwrap.dedent("""
Hi! This is the friendly automated nwb-extensions-webservice.
I tried to {} for you, but it looks like I wasn't able to push to the {} branch of {}/{}. Did you check the "Allow edits from maintainers" box?
""").format(pr_branch, pr_owner, pr_repo,
changes_str) # noqa: E501
pull.create_issue_comment(message)
else:
if rerender_error:
message = textwrap.dedent("""
Hi! This is the friendly automated nwb-extensionse-webservice.
I tried to {} for you but ran into some issues, please ping nwb-extensions/core for further assistance.
""").format(changes_str) # noqa: E501
else:
message = textwrap.dedent("""
Hi! This is the friendly automated nwb-extensions-webservice.
I tried to {} for you, but it looks like there was nothing to do.
""").format(changes_str)
pull.create_issue_comment(message)
def pr_detailed_comment(org_name, repo_name, pr_owner, pr_repo, pr_branch, pr_num, comment):
is_staged_recipes = (repo_name == "staged-recipes")
if not (repo_name.endswith("-feedstock") or is_staged_recipes):
return
if not is_staged_recipes and UPDATE_CIRCLECI_KEY_MSG.search(comment):
update_circle(org_name, repo_name)
gh = github.Github(os.environ['GH_TOKEN'])
repo = gh.get_repo("{}/{}".format(org_name, repo_name))
pull = repo.get_pull(int(pr_num))
message = textwrap.dedent("""
Hi! This is the friendly automated conda-forge-webservice.
I just wanted to let you know that I updated the circle-ci deploy key and followed the project.
""")
pull.create_issue_comment(message)
pr_commands = [LINT_MSG]
if not is_staged_recipes:
pr_commands += [ADD_NOARCH_MSG, RERENDER_MSG, UPDATE_CB3_MSG]
if not any(command.search(comment) for command in pr_commands):
return
with tmp_directory() as tmp_dir:
feedstock_dir = os.path.join(tmp_dir, repo_name)
repo_url = "https://{}@github.com/{}/{}.git".format(
os.environ['GH_TOKEN'], pr_owner, pr_repo)
repo = Repo.clone_from(repo_url, feedstock_dir, branch=pr_branch)
if LINT_MSG.search(comment):
relint(org_name, repo_name, pr_num)
changed_anything = False
expected_changes = []
extra_msg = ''
if not is_staged_recipes:
do_noarch = do_cb3 = do_rerender = False
if ADD_NOARCH_MSG.search(comment):
do_noarch = do_rerender = True
expected_changes.append('add noarch')
if UPDATE_CB3_MSG.search(comment):
do_cb3 = do_rerender = True
expected_changes.append('update for conda-build 3')
if RERENDER_MSG.search(comment):
do_rerender = True
expected_changes.append('re-render')
if do_noarch:
changed_anything |= make_noarch(repo)
if do_cb3:
c, cb3_changes = update_cb3(repo)
changed_anything |= c
extra_msg += '\n\n' + cb3_changes
if do_rerender:
changed_anything |= rerender(repo)
if expected_changes:
if len(expected_changes) > 1:
expected_changes[-1] = 'and ' + expected_changes[-1]
joiner = ", " if len(expected_changes) > 2 else " "
changes_str = joiner.join(expected_changes)
gh = github.Github(os.environ['GH_TOKEN'])
gh_repo = gh.get_repo("{}/{}".format(org_name, repo_name))
pull = gh_repo.get_pull(int(pr_num))
if changed_anything:
try:
repo.remotes.origin.push()
except GitCommandError:
message = textwrap.dedent("""
Hi! This is the friendly automated conda-forge-webservice.
I tried to {} for you, but it looks like I wasn't able to push to the {} branch of {}/{}. Did you check the "Allow edits from maintainers" box?
""").format(pr_branch, pr_owner, pr_repo, changes_str)
pull.create_issue_comment(message)
else:
message = textwrap.dedent("""
Hi! This is the friendly automated conda-forge-webservice.
I tried to {} for you, but it looks like there was nothing to do.
""").format(changes_str)
pull.create_issue_comment(message)
parser.add_argument(
'--source-ref',
required=True,
help=
'The git version for the source. Used to retry failed imports without bumping versions.'
)
parser.add_argument('--dest',
required=True,
help='The destination rosdistro name')
parser.add_argument('--release-org',
required=True,
help='The organization containing release repositories')
args = parser.parse_args()
gclient = github.Github(os.environ['GITHUB_TOKEN'])
release_org = gclient.get_organization(args.release_org)
org_release_repos = [r.name for r in release_org.get_repos() if r.name]
if not os.path.isfile('index-v4.yaml'):
raise RuntimeError(
'This script must be run from a rosdistro index directory.')
rosdistro_dir = os.path.abspath(os.getcwd())
rosdistro_index_url = f'file://{rosdistro_dir}/index-v4.yaml'
index = get_index(rosdistro_index_url)
index_yaml = yaml.safe_load(open('index-v4.yaml', 'r'))
if len(index_yaml['distributions'][args.source]['distribution']) != 1 or \
len(index_yaml['distributions'][args.dest]['distribution']) != 1:
raise RuntimeError(
def testGetOAuthScopesFromHeader(self):
g = github.Github(self.oauth_token)
self.assertEqual(g.oauth_scopes, None)
g.get_user().name
self.assertEqual(g.oauth_scopes, ['repo', 'user', 'gist'])
def api_request_loop(state):
g = github.Github(ACCESS_TOKEN)
print("Collecting Github Search API data...")
try:
repositories = g.search_code(state.query)
state.totalCount = repositories.totalCount
# Hack to backfill PaginatedList with garbage to avoid ratelimiting on
# restore, library fetches in 30 counts
repositories.__dict__["_PaginatedListBase__elements"] = [None] * (
state.index // 30) * 30
state.lastInitIndex = state.index
print(bcolors.CLEAR)
i = state.index
stepBack = False
while i < state.totalCount:
while True:
try:
state.index = i
# Manually fill Paginator to avoid ratelimiting on restore
repositories = pagination_hack(repositories, state)
repo = repositories[i]
log_buf = "https://github.com/" + \
bcolors.OKGREEN + repo.repository.owner.login + "/" + \
bcolors.OKBLUE + repo.repository.name + "/blob" + \
bcolors.ENDC + \
os.path.dirname(repo.html_url.split('blob')[1]) + \
"/" + bcolors.WARNING + repo.name + bcolors.ENDC
print(log_buf)
log_buf = "\n" + log_buf + "\n"
if should_parse(repo, state) or stepBack:
stepBack = False
log_buf += regex_search(state.checks, repo)
ui_loop(repo, repositories, log_buf, state)
if state.index < i:
i = state.index
stepBack = True
print(bcolors.CLEAR)
else:
print("Skipping...")
i += 1
break
except github.RateLimitExceededException:
try:
print(bcolors.FAIL + "RateLimitException: "
"Please wait about 30 seconds before you "
"try again, or exit (CTRL-C).\n " + bcolors.ENDC)
save_state("ratelimited", repositories, state)
input("Press enter to try again...")
except KeyboardInterrupt:
sys.exit(1)
except github.RateLimitExceededException:
print(bcolors.FAIL + "RateLimitException: "
"Please wait about 30 seconds before you try again.\n" +
bcolors.ENDC)
save_state("ratelimited", repositories, state)
sys.exit(-1)
# The master toctree document.
master_doc = 'index'
# General information about the project.
project = 'Parselmouth'
copyright = '2017-2020, Yannick Jadoul'
author = 'Yannick Jadoul'
if on_rtd:
rtd_version = os.environ.get('READTHEDOCS_VERSION')
branch = 'master' if rtd_version == 'latest' else rtd_version
github_token = os.environ['GITHUB_TOKEN']
head_sha = git.Repo(search_parent_directories=True).head.commit.hexsha
g = github.Github()
runs = g.get_repo('YannickJadoul/Parselmouth').get_workflow(
"wheels.yml").get_runs(branch=branch)
artifacts_url = next(r for r in runs
if r.head_sha == head_sha).artifacts_url
archive_download_url = next(
artifact
for artifact in requests.get(artifacts_url).json()['artifacts']
if artifact['name'] == 'rtd-wheel')['archive_download_url']
artifact_bin = io.BytesIO(
requests.get(archive_download_url,
headers={
'Authorization': f'token {github_token}'
},
stream=True).content)
def _database():
gh = github.Github(flask.current_app.config['GITHUB_TOKEN'])
return githubdb.GithubDB(gh)
merge_method))
post_parameters["merge_method"] = merge_method
## End custom code ##
headers, data = self._requester.requestJsonAndCheck(
"PUT", self.url + "/merge", input=post_parameters)
return github.PullRequestMergeStatus.PullRequestMergeStatus(
self._requester, headers, data, completed=True)
github.PullRequest.PullRequest = ExtendedPullRequest
# Initialize GitHub client. For documentation,
# see http://pygithub.github.io/PyGithub/v1/reference.html.
github_access_token = os.environ['GITHUB_ACCESS_TOKEN']
edx = github.Github(github_access_token).get_organization('edx')
class Repo:
# Make 18 attempts to merge the PR (the original attempt plus 17 retries), sleeping for 5 minutes between each
# attempt. This should result in a total 90 minutes of sleep time. Status checks should almost always
# complete in this period.
MAX_MERGE_RETRIES = 17
"""Utility representing a Git repo."""
def __init__(self,
clone_url,
repo_owner,
branch_name,
message,
merge_method=DEFAULT_MERGE_METHOD):
# See https://github.com/blog/1270-easier-builds-and-deployments-using-git-over-https-and-oauth.
def get_github_instance():
g = github.Github(settings.GITHUB_ACCESS_TOKEN)
return g
def setUp(self):
super(FunctionalTestBase, self).setUp()
self.existing_labels = []
self.pr_counter = 0
self.git_counter = 0
self.cassette_library_dir = os.path.join(CASSETTE_LIBRARY_DIR_BASE,
self.__class__.__name__,
self._testMethodName)
# Recording stuffs
if RECORD:
if os.path.exists(self.cassette_library_dir):
shutil.rmtree(self.cassette_library_dir)
os.makedirs(self.cassette_library_dir)
self.recorder = vcr.VCR(
cassette_library_dir=self.cassette_library_dir,
record_mode="all" if RECORD else "none",
match_on=["method", "uri"],
filter_headers=[
("Authorization", "<TOKEN>"),
("X-Hub-Signature", "<SIGNATURE>"),
("User-Agent", None),
("Accept-Encoding", None),
("Connection", None),
],
before_record_response=self.response_filter,
custom_patches=((pygithub.MainClass, "HTTPSConnection",
vcr.stubs.VCRHTTPSConnection), ),
)
if RECORD:
github.CachedToken.STORAGE = {}
else:
# Never expire token during replay
mock.patch.object(github_app,
"get_or_create_jwt",
return_value="<TOKEN>").start()
mock.patch.object(
github.GithubAppInstallationAuth,
"get_access_token",
return_value="<TOKEN>",
).start()
# NOTE(sileht): httpx pyvcr stubs does not replay auth_flow as it directly patch client.send()
# So anything occurring during auth_flow have to be mocked during replay
def get_auth(owner=None, auth=None):
if auth is None:
auth = github.get_auth(owner)
auth.installation = {
"id": config.INSTALLATION_ID,
}
auth.permissions_need_to_be_updated = False
auth.owner_id = config.TESTING_ORGANIZATION_ID
return auth
async def github_aclient(owner=None, auth=None):
return github.AsyncGithubInstallationClient(
get_auth(owner, auth))
def github_client(owner=None, auth=None):
return github.GithubInstallationClient(get_auth(owner, auth))
mock.patch.object(github, "get_client", github_client).start()
mock.patch.object(github, "aget_client", github_aclient).start()
with open(engine.mergify_rule_path, "r") as f:
engine.MERGIFY_RULE = yaml.safe_load(f.read().replace(
"mergify[bot]", "mergify-test[bot]"))
mock.patch.object(branch_updater.utils, "Gitter",
self.get_gitter).start()
mock.patch.object(duplicate_pull.utils, "Gitter",
self.get_gitter).start()
if not RECORD:
# NOTE(sileht): Don't wait exponentialy during replay
mock.patch.object(context.Context._ensure_complete.retry, "wait",
None).start()
# Web authentification always pass
mock.patch("hmac.compare_digest", return_value=True).start()
branch_prefix_path = os.path.join(self.cassette_library_dir,
"branch_prefix")
if RECORD:
self.BRANCH_PREFIX = datetime.datetime.utcnow().strftime(
"%Y%m%d%H%M%S")
with open(branch_prefix_path, "w") as f:
f.write(self.BRANCH_PREFIX)
else:
with open(branch_prefix_path, "r") as f:
self.BRANCH_PREFIX = f.read()
self.master_branch_name = self.get_full_branch_name("master")
self.git = self.get_gitter(LOG)
self.addCleanup(self.git.cleanup)
self.loop = asyncio.get_event_loop()
self.loop.run_until_complete(web.startup())
self.app = testclient.TestClient(web.app)
# NOTE(sileht): Prepare a fresh redis
self.redis = utils.get_redis_for_cache()
self.redis.flushall()
self.subscription = subscription.Subscription(
config.INSTALLATION_ID,
self.SUBSCRIPTION_ACTIVE,
"You're not nice",
{"mergify-test1": config.ORG_ADMIN_GITHUB_APP_OAUTH_TOKEN},
frozenset(
getattr(subscription.Features, f)
for f in subscription.Features.__members__)
if self.SUBSCRIPTION_ACTIVE else frozenset(),
)
self.loop.run_until_complete(
self.subscription.save_subscription_to_cache())
# Let's start recording
cassette = self.recorder.use_cassette("http.json")
cassette.__enter__()
self.addCleanup(cassette.__exit__)
integration = pygithub.GithubIntegration(config.INTEGRATION_ID,
config.PRIVATE_KEY)
self.installation_token = integration.get_access_token(
config.INSTALLATION_ID).token
base_url = config.GITHUB_API_URL
self.g_integration = pygithub.Github(self.installation_token,
base_url=base_url)
self.g_admin = pygithub.Github(config.ORG_ADMIN_PERSONAL_TOKEN,
base_url=base_url)
self.g_fork = pygithub.Github(self.FORK_PERSONAL_TOKEN,
base_url=base_url)
self.o_admin = self.g_admin.get_organization(
config.TESTING_ORGANIZATION)
self.o_integration = self.g_integration.get_organization(
config.TESTING_ORGANIZATION)
self.u_fork = self.g_fork.get_user()
assert self.o_admin.login == "mergifyio-testing"
assert self.o_integration.login == "mergifyio-testing"
assert self.u_fork.login in ["mergify-test2", "mergify-test3"]
self.r_o_admin = self.o_admin.get_repo(self.REPO_NAME)
self.r_o_integration = self.o_integration.get_repo(self.REPO_NAME)
self.r_fork = self.u_fork.get_repo(self.REPO_NAME)
self.url_main = f"{config.GITHUB_URL}/{self.r_o_integration.full_name}"
self.url_fork = (
f"{config.GITHUB_URL}/{self.u_fork.login}/{self.r_o_integration.name}"
)
self.cli_integration = github.get_client(config.TESTING_ORGANIZATION, )
real_get_subscription = subscription.Subscription.get_subscription
async def fake_retrieve_subscription_from_db(owner_id):
if owner_id == config.TESTING_ORGANIZATION_ID:
return self.subscription
return subscription.Subscription(
owner_id,
False,
"We're just testing",
{},
set(),
)
async def fake_subscription(owner_id):
if owner_id == config.TESTING_ORGANIZATION_ID:
return await real_get_subscription(owner_id)
return subscription.Subscription(
owner_id,
False,
"We're just testing",
{},
set(),
)
mock.patch(
"mergify_engine.subscription.Subscription._retrieve_subscription_from_db",
side_effect=fake_retrieve_subscription_from_db,
).start()
mock.patch(
"mergify_engine.subscription.Subscription.get_subscription",
side_effect=fake_subscription,
).start()
mock.patch(
"github.MainClass.Installation.Installation.get_repos",
return_value=[self.r_o_integration],
).start()
self._event_reader = EventReader(self.app)
self._event_reader.drain()
def __init__(self, name, access_token):
self.github = github.Github(access_token)
self.repo = self.github.get_repo(name)
self.name = name
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
"lbry_part", help="part of lbry version to bump",
choices=LBRY_PARTS
)
parser.add_argument(
"--skip-lbryum", help="skip bumping lbryum, even if there are changes",
action="store_true",
)
parser.add_argument(
"--lbryum-part", help="part of lbryum version to bump",
choices=LBRYUM_PARTS
)
parser.add_argument(
"--last-release",
help=("manually set the last release version. The default is to query and parse the"
" value from the release page.")
)
parser.add_argument(
"--skip-sanity-checks", action="store_true")
parser.add_argument(
"--require-changelog", action="store_true",
help=("Set this flag to raise an exception if a submodules has changes without a"
" corresponding changelog entry. The default is to log a warning")
)
parser.add_argument(
"--skip-push", action="store_true",
help="Set to not push changes to remote repo"
)
args = parser.parse_args()
base = git.Repo(os.getcwd())
branch = 'master'
if not args.skip_sanity_checks:
run_sanity_checks(base, branch)
base_repo = Repo('lbry-app', args.lbry_part, os.getcwd())
base_repo.assert_new_tag_is_absent()
last_release = args.last_release or base_repo.get_last_tag()
logging.info('Last release: %s', last_release)
gh_token = get_gh_token()
auth = github.Github(gh_token)
github_repo = auth.get_repo('lbryio/lbry-app')
names = ['lbryum', 'lbry']
repos = {name: Repo(name, get_part(args, name)) for name in names}
changelogs = {}
for repo in repos.values():
logging.info('Processing repo: %s', repo.name)
repo.checkout(branch)
last_submodule_hash = base_repo.get_submodule_hash(last_release, repo.name)
if repo.has_changes_from_revision(last_submodule_hash):
if repo.name == 'lbryum':
if args.skip_lbryum:
continue
if not repo.part:
repo.part = get_lbryum_part()
entry = repo.get_changelog_entry()
if entry:
changelogs[repo.name] = entry.strip()
repo.add_changelog()
else:
msg = 'Changelog entry is missing for {}'.format(repo.name)
if args.require_changelog:
raise Exception(msg)
else:
logging.warning(msg)
else:
logging.warning('Submodule %s has no changes.', repo.name)
if repo.name == 'lbryum':
# The other repos have their version track each other so need to bump
# them even if there aren't any changes, but lbryum should only be
# bumped if it has changes
continue
# bumpversion will fail if there is already the tag we want in the repo
repo.assert_new_tag_is_absent()
repo.bumpversion()
release_msg = get_release_msg(changelogs, names)
for name in names:
base.git.add(name)
base_repo.bumpversion()
current_tag = base.git.describe()
is_rc = re.match('\drc\d+$', current_tag) is not None
github_repo.create_git_release(current_tag, current_tag, release_msg, draft=True,
prerelease=is_rc)
no_change_msg = ('No change since the last release. This release is simply a placeholder'
' so that LBRY and LBRY App track the same version')
lbrynet_daemon_release_msg = changelogs.get('lbry', no_change_msg)
auth.get_repo('lbryio/lbry').create_git_release(
current_tag, current_tag, lbrynet_daemon_release_msg, draft=True)
if not args.skip_push:
for repo in repos.values():
repo.git.push(follow_tags=True)
base.git.push(follow_tags=True, recurse_submodules='check')
else:
logging.info('Skipping push; you will have to reset and delete tags if '
'you want to run this script again. Take a look at reset.sh; '
'it probably does what you want.')
feedstocks_dir = os.path.expanduser(args.feedstocks_dir)
change_limit = args.limit
feedstocks.clone_all('conda-forge', feedstocks_dir)
feedstocks.fetch_feedstocks(feedstocks_dir)
regexp = re.compile(args.regexp)
randomised_feedstocks = [
feedstock for feedstock in feedstocks.cloned_feedstocks(feedstocks_dir)
if regexp.match(feedstock.package)
]
# Shuffle is in-place. :(
random.shuffle(randomised_feedstocks)
gh_token = conda_smithy.github.gh_token()
gh = github.Github(gh_token)
gh_me = gh.get_user()
if gh_me.login != 'conda-forge-admin':
raise ValueError("The github token isn't that of conda-forge-admin (it's "
"for {}), I'm going to have to bail.".format(gh_me.login))
gh_forge = gh.get_organization('conda-forge')
def my_repos(gh_user):
"""
List all of my repos.
See https://github.com/PyGithub/PyGithub/issues/390 for rationale.
def testBadUserAgent(self):
self.assertRaises(
github.BadUserAgentException, lambda: github.Github(
self.login, self.password, user_agent="").get_user().name)
def make_github_agent(user=None, password=None):
""" Create github agent to auth """
if not user:
return github.Github()
else:
return github.Github(user, password)
def main():
"""Collects necessary info and dumps it to disk."""
branch_name = common.get_current_branch_name()
if not common.is_current_branch_a_release_branch():
raise Exception(
'This script should only be run from the latest release branch.')
parsed_args = _PARSER.parse_args()
if parsed_args.github_username is None:
raise Exception(
'No GitHub username provided. Please re-run the '
'script specifying a username using '
'--github_username=<Your username>')
github_username = parsed_args.github_username
personal_access_token = common.get_personal_access_token()
g = github.Github(personal_access_token)
repo = g.get_organization('oppia').get_repo('oppia')
repo_fork = g.get_repo('%s/oppia' % github_username)
common.check_blocking_bug_issue_count(repo)
common.check_prs_for_current_release_are_released(repo)
python_utils.PRINT('Generating release summary...')
generate_release_info.main(personal_access_token)
if not os.path.exists(release_constants.RELEASE_SUMMARY_FILEPATH):
raise Exception(
'Release summary file %s is missing. Please re-run '
'this script.' % release_constants.RELEASE_SUMMARY_FILEPATH)
current_release_version_number = common.get_current_release_version_number(
branch_name)
target_branch = 'update-changelog-for-releasev%s' % (
current_release_version_number)
remove_updates_and_delete_branch(repo_fork, target_branch)
# Opens Credit Form.
python_utils.PRINT(
'Note: Make following changes directly to %s and make sure to '
'save the file after making these changes.' % (
release_constants.RELEASE_SUMMARY_FILEPATH))
common.ask_user_to_confirm(
'Check emails and names for new authors and new contributors in the '
'file: %s and verify that the emails are '
'correct through welcome emails sent from [email protected] '
'(confirm with Sean in case of doubt). Please ensure that you correct '
'the emails of the form: %s.' % (
release_constants.RELEASE_SUMMARY_FILEPATH,
release_constants.INVALID_EMAIL_SUFFIX))
common.open_new_tab_in_browser_if_possible(
release_constants.CREDITS_FORM_URL)
common.ask_user_to_confirm(
'Check the credits form and add any additional contributors '
'to the contributor list in the file: %s.' % (
release_constants.RELEASE_SUMMARY_FILEPATH))
common.ask_user_to_confirm(
'Categorize the PR titles in the Uncategorized section of the '
'changelog in the file: %s, and arrange the changelog '
'to have user-facing categories on top.' % (
release_constants.RELEASE_SUMMARY_FILEPATH))
common.ask_user_to_confirm(
'Verify each item is in the correct section in the '
'file: %s and remove trivial changes like "Fix lint errors" '
'from the changelog.' % (
release_constants.RELEASE_SUMMARY_FILEPATH))
common.ask_user_to_confirm(
'Ensure that all items in changelog in the file: %s '
'start with a verb in simple present tense.' % (
release_constants.RELEASE_SUMMARY_FILEPATH))
common.ask_user_to_confirm(
'Please save the file: %s with all the changes that '
'you have made.' % (
release_constants.RELEASE_SUMMARY_FILEPATH))
release_summary_lines = get_release_summary_lines()
update_changelog(
branch_name, release_summary_lines, current_release_version_number)
update_authors(release_summary_lines)
update_contributors(release_summary_lines)
update_developer_names(release_summary_lines)
message = (
'Please check the changes and make updates if required in the '
'following files:\n1. %s\n2. %s\n3. %s\n4. %s\n' % (
CHANGELOG_FILEPATH, AUTHORS_FILEPATH, CONTRIBUTORS_FILEPATH,
ABOUT_PAGE_CONSTANTS_FILEPATH))
common.ask_user_to_confirm(message)
create_branch(
repo, repo_fork, target_branch, github_username,
current_release_version_number)
def __init__(self, github_organizations):
self.github_organizations = github_organizations
self.github_client = github.Github()
def main(summary_path, repos_xml_path):
GITHUB_USERNAME = os.environ['GITHUB_USERNAME']
GITHUB_TOKEN_FILE = os.environ['GITHUB_TOKEN_FILE']
GITHUB_ORG = os.environ['GITHUB_ORG']
with open(summary_path) as f:
repos = json.load(f)
with open(GITHUB_TOKEN_FILE) as f:
token = f.read().strip()
g = github.Github(GITHUB_USERNAME, token, per_page=50)
gu = g.get_organization(GITHUB_ORG)
gh_repos = set()
# check repo states
for data in repos.values():
# 1. we don't add repos with broken metadata but we also don't
# remove existing ones -- we hope maintainers will fix them,
# or overlays team will remove them
#
# 2. remove repos with unsupported VCS -- this means that
# upstream has switched, and there's no point in keeping
# an outdated mirror
#
# 3. we can't update repos which are broken to the point of
# being implicitly removed
data['x-can-create'] = data['x-state'] in ('GOOD', 'BAD_CACHE')
data['x-can-update'] = data['x-can-create']
data['x-should-remove'] = data['x-state'] in ('REMOVED', 'UNSUPPORTED')
# 0. scan all repos
to_remove = []
to_update = []
for i, r in enumerate(gu.get_repos()):
sys.stderr.write('\r@ scanning [%-3d/%-3d]' % (i + 1, gu.public_repos))
if r.name not in repos or repos[r.name]['x-should-remove']:
to_remove.append(r)
else:
gh_repos.add(r.name)
if repos[r.name]['x-can-update']:
to_update.append(r)
repos[r.name]['x-mirror-sources'] = gh_sources(r)
sys.stderr.write('\n')
# 1. delete stale repos
for r in to_remove:
sys.stderr.write('* removing %s\n' % r.name)
r.delete()
# 2. now create new repos :)
for r, data in sorted(repos.items()):
if r not in gh_repos and data['x-can-create']:
sys.stderr.write('* adding %s\n' % r)
gr = gu.create_repo(
r,
description=' '.join(
data.get('description', {}).get('en').split())
or github.GithubObject.NotSet,
homepage=data.get('homepage') or github.GithubObject.NotSet,
has_issues=False,
has_wiki=False)
repos[r]['x-mirror-sources'] = gh_sources(gr)
to_update.append(gr)
# 3. write a new repositories.xml for them
root = et.Element('repositories')
root.set('version', '1.0')
for r, data in sorted(repos.items()):
if 'x-mirror-sources' not in data:
continue
rel = et.Element('repo')
for attr, val in sorted(data.items(), key=dtd_sort_key):
if attr.startswith('x-'):
continue
elif attr == 'source': # replace
for t, url in data['x-mirror-sources']:
subel = et.Element('source')
subel.set('type', t)
subel.text = url
rel.append(subel)
elif attr in ('quality', 'status'): # attributes
rel.set(attr, val)
elif attr in ('name', 'homepage'): # single-value
subel = et.Element(attr)
subel.text = val
rel.append(subel)
elif attr in ('description', 'longdescription'): # lang-dict
for l, v in val.items():
subel = et.Element(attr)
subel.set('lang', l)
subel.text = v
rel.append(subel)
elif attr in ('owner', 'feed'): # lists
for v in val:
subel = et.Element(attr)
if attr == 'owner':
for k, subval in v.items():
if k == 'type':
subel.set(k, subval)
else:
subsubel = et.Element(k)
subsubel.text = subval
subel.append(subsubel)
else:
subel.text = v
rel.append(subel)
root.append(rel)
xml = et.ElementTree(root)
with open(repos_xml_path, 'wb') as f:
f.write(b'<?xml version="1.0" encoding="UTF-8"?>\n')
f.write(
b'<!DOCTYPE repositories SYSTEM "http://www.gentoo.org/dtd/repositories.dtd">\n'
)
xml.write(f, 'UTF-8', False)
print('DELETED_REPOS = %s' % ' '.join(r.name for r in to_remove))
print('REPOS = %s' % ' '.join(r.name for r in to_update))
def collect_metrics():
redis = utils.get_redis_for_cache()
integration = github.GithubIntegration(config.INTEGRATION_ID,
config.PRIVATE_KEY)
installations = collections.defaultdict(int)
repositories_per_installation = collections.defaultdict(int)
users_per_installation = collections.defaultdict(int)
LOG.info("GitHub Polling started")
redis.delete("badges.tmp")
for installation in utils.get_installations(integration):
try:
_id = installation["id"]
target_type = installation["target_type"]
account = installation["account"]["login"]
LOG.info("Get subscription", account=account)
subscribed = sub_utils.get_subscription(redis,
_id)["subscription_active"]
installations[(subscribed, target_type)] += 1
token = integration.get_access_token(_id).token
g = github.Github(token,
base_url="https://api.%s" % config.GITHUB_DOMAIN)
if installation["target_type"] == "Organization":
LOG.info("Get members",
install=installation["account"]["login"])
org = g.get_organization(installation["account"]["login"])
value = len(list(org.get_members()))
users_per_installation[(subscribed, target_type,
account)] = value
else:
users_per_installation[(subscribed, target_type, account)] = 1
LOG.info("Get repos", account=account)
repositories = sorted(g.get_installation(_id).get_repos(),
key=operator.attrgetter("private"))
for private, repos in itertools.groupby(
repositories, key=operator.attrgetter("private")):
configured_repos = 0
unconfigured_repos = 0
for repo in repos:
try:
repo.get_contents(".mergify.yml")
configured_repos += 1
redis.sadd("badges.tmp", repo.full_name)
except github.GithubException as e:
if e.status >= 500: # pragma: no cover
raise
unconfigured_repos += 1
repositories_per_installation[(subscribed, target_type,
account, private,
True)] = configured_repos
repositories_per_installation[(subscribed, target_type,
account, private,
False)] = unconfigured_repos
except github.GithubException as e: # pragma: no cover
# Ignore rate limit/abuse
if e.status != 403:
raise
LOG.info("GitHub Polling finished")
# NOTE(sileht): Prometheus can scrape data during our loop. So make it fast
# to ensure we always have the good values.
# Also we can't known which labels we should delete from the Gauge,
# that's why we delete all of them to re-add them.
# And prometheus_client doesn't provide API to that, so we just
# override _metrics
set_gauges(INSTALLATIONS, installations)
set_gauges(USERS_PER_INSTALLATION, users_per_installation)
set_gauges(REPOSITORIES_PER_INSTALLATION, repositories_per_installation)
if redis.exists("badges.tmp"):
redis.rename("badges.tmp", "badges")
LOG.info("Gauges and badges cache updated")
def __init__(self, config):
token = self.get_token(config)
self.g = github.Github(token)
self._repos = config['repos']
self.monitoring_db = monitoring.GetDatabase('spinbot')
self.logging = logging.getLogger('github_client_wrapper')
def test_maintainer_exists(self):
lints, hints = linter.lintify(
{'extra': {
'recipe-maintainers': ['support']
}}, conda_forge=True)
expected_message = ('Recipe maintainer "support" does not exist')
self.assertIn(expected_message, lints)
lints = linter.lintify({'extra': {
'recipe-maintainers': ['isuruf']
}},
conda_forge=True)
expected_message = ('Recipe maintainer "isuruf" does not exist')
self.assertNotIn(expected_message, lints)
expected_message = 'Feedstock with the same name exists in conda-forge'
# Check that feedstock exists if staged_recipes
lints = linter.lintify({'package': {
'name': 'python'
}},
recipe_dir='python',
conda_forge=True)
self.assertIn(expected_message, lints)
lints = linter.lintify({'package': {
'name': 'python'
}},
recipe_dir='python',
conda_forge=False)
self.assertNotIn(expected_message, lints)
# No lint if in a feedstock
lints = linter.lintify({'package': {
'name': 'python'
}},
recipe_dir='recipe',
conda_forge=True)
self.assertNotIn(expected_message, lints)
lints = linter.lintify({'package': {
'name': 'python'
}},
recipe_dir='recipe',
conda_forge=False)
self.assertNotIn(expected_message, lints)
# Make sure there's no feedstock named python1 before proceeding
gh = github.Github(os.environ['GH_TOKEN'])
cf = gh.get_user('conda-forge')
try:
cf.get_repo('python1-feedstock')
feedstock_exists = True
except github.UnknownObjectException as e:
feedstock_exists = False
if feedstock_exists:
warnings.warn(
"There's a feedstock named python1, but tests assume that there isn't"
)
else:
lints = linter.lintify({'package': {
'name': 'python1'
}},
recipe_dir="python",
conda_forge=True)
self.assertNotIn(expected_message, lints)
# Test bioconda recipe checking
expected_message = (
"Recipe with the same name exists in bioconda: "
"please discuss with @conda-forge/bioconda-recipes.")
bio = gh.get_user('bioconda').get_repo('bioconda-recipes')
r = 'samtools'
try:
bio.get_dir_contents('recipe/{}'.format(r))
except github.UnknownObjectException as e:
warnings.warn(
"There's no bioconda recipe named {}, but tests assume that there is"
.format(r))
else:
# Check that feedstock exists if staged_recipes
lints = linter.lintify({'package': {
'name': r
}},
recipe_dir=r,
conda_forge=True)
self.assertIn(expected_message, lints)
lints = linter.lintify({'package': {
'name': r
}},
recipe_dir=r,
conda_forge=False)
self.assertNotIn(expected_message, lints)
# No lint if in a feedstock
lints = linter.lintify({'package': {
'name': r
}},
recipe_dir='recipe',
conda_forge=True)
self.assertNotIn(expected_message, lints)
lints = linter.lintify({'package': {
'name': r
}},
recipe_dir='recipe',
conda_forge=False)
self.assertNotIn(expected_message, lints)
r = 'this-will-never-exist'
try:
bio.get_dir_contents('recipes/{}'.format(r))
except github.UnknownObjectException as e:
lints = linter.lintify({'package': {
'name': r
}},
recipe_dir=r,
conda_forge=True)
self.assertNotIn(expected_message, lints)
else:
warnings.warn(
"There's a bioconda recipe named {}, but tests assume that there isn't"
.format(r))
def testGetAuthorizationsSucceedsWhenAutenticatedThroughLoginPassword(
self):
g = github.Github(self.login, self.password)
self.assertListKeyEqual(
g.get_user().get_authorizations(), lambda a: a.note,
[None, None, 'cligh', None, None, 'GitHub Android App'])
def __repo(self) -> github.Repository.Repository:
return github.Github(self.token).get_repo(self.repo_name)
# get inputs
workflow_name = os.environ["GITHUB_WORKFLOW"]
polling_interval = _parse_int("POLL_INTERVAL", 60)
continue_after = _parse_int("CONTINUE_AFTER", None)
abort_after = _parse_int("ABORT_AFTER", None)
if "GITHUB_HEAD_REF" in os.environ and len(os.environ["GITHUB_HEAD_REF"]) > 0:
branch = os.environ["GITHUB_HEAD_REF"]
elif "GITHUB_REF" in os.environ and len(os.environ["GITHUB_REF"]) > 0:
branch = os.environ["GITHUB_REF"][11:]
else:
# TODO: should this be main?
branch = "master"
print("computed branch '%s' for workflow" % branch, flush=True)
gh = github.Github(os.environ["GITHUB_TOKEN"])
repo = gh.get_repo(os.environ["GITHUB_REPOSITORY"])
wf = None
limit = 100
done = 0
for _wf in repo.get_workflows():
if _wf.name == workflow_name:
wf = _wf
break
done += 1
if done == limit:
break
if wf is None:
parser.add_argument("auth_token")
parser.add_argument("version", type=version_type)
subparsers = parser.add_subparsers(dest="subparser_name",
help="Subcommands: check, move_milestone, rn, release")
subparsers.required = True
check_parser = subparsers.add_parser("check")
check_parser.set_defaults(func=check_release_exists)
milestone_parser = subparsers.add_parser("move_milestone")
milestone_parser.set_defaults(func=move_milestone)
milestone_parser.add_argument("next_version", type=version_type)
rn_parser = subparsers.add_parser("rn")
rn_parser.set_defaults(func=generate_release_notes)
release = subparsers.add_parser("release")
release.set_defaults(func=create_release)
release.add_argument("commit")
return parser.parse_args()
if __name__ == "__main__":
parser = create_parser()
gh = github.Github(parser.auth_token)
repo = get_repo(gh, parser.owner, parser.repo)
parser.func(repo, parser)
def issue_comment(org_name, repo_name, issue_num, title, comment):
if not repo_name.endswith("-feedstock"):
return
text = comment + title
issue_commands = [UPDATE_TEAM_MSG, ADD_NOARCH_MSG, UPDATE_CIRCLECI_KEY_MSG,
RERENDER_MSG, UPDATE_CB3_MSG]
send_pr_commands = [ADD_NOARCH_MSG, RERENDER_MSG, UPDATE_CB3_MSG]
if not any(command.search(text) for command in issue_commands):
return
gh = github.Github(os.environ['GH_TOKEN'])
repo = gh.get_repo("{}/{}".format(org_name, repo_name))
issue = repo.get_issue(int(issue_num))
if UPDATE_TEAM_MSG.search(text):
update_team(org_name, repo_name)
if UPDATE_TEAM_MSG.search(title):
issue.edit(state="closed")
message = textwrap.dedent("""
Hi! This is the friendly automated conda-forge-webservice.
I just wanted to let you know that I updated the team with maintainers from master.
""")
issue.create_comment(message)
if UPDATE_CIRCLECI_KEY_MSG.search(text):
update_circle(org_name, repo_name)
if UPDATE_CIRCLECI_KEY_MSG.search(title):
issue.edit(state="closed")
message = textwrap.dedent("""
Hi! This is the friendly automated conda-forge-webservice.
I just wanted to let you know that I updated the circle-ci deploy key and followed the project.
""")
issue.create_comment(message)
if any(command.search(text) for command in send_pr_commands):
forked_user = gh.get_user().login
forked_repo = gh.get_user().create_fork(repo)
with tmp_directory() as tmp_dir:
feedstock_dir = os.path.join(tmp_dir, repo_name)
repo_url = "https://{}@github.com/{}/{}.git".format(
os.environ['GH_TOKEN'], forked_user, repo_name)
upstream_repo_url = "https://{}@github.com/{}/{}.git".format(
os.environ['GH_TOKEN'], org_name, repo_name)
git_repo = Repo.clone_from(repo_url, feedstock_dir)
forked_repo_branch = 'conda_forge_admin_{}'.format(issue_num)
upstream = git_repo.create_remote('upstream', upstream_repo_url)
upstream.fetch()
new_branch = git_repo.create_head(forked_repo_branch, upstream.refs.master)
new_branch.checkout()
changed_anything = False
extra_msg = ""
if UPDATE_CB3_MSG.search(text):
pr_title = "MNT: Update for conda-build 3"
comment_msg = "updated the recipe for conda-build 3"
to_close = UPDATE_CB3_MSG.search(title)
if ADD_NOARCH_MSG.search(text):
changed_anything |= make_noarch(git_repo)
pr_title += ' and add noarch: python'
comment_msg += ' and added `noarch: python`'
c, cb3_changes = update_cb3(git_repo)
changed_anything |= c
if not c:
cb3_changes = "There weren't any changes to make for conda-build 3."
extra_msg = '\n\n' + cb3_changes
changed_anything |= rerender(git_repo)
elif ADD_NOARCH_MSG.search(text):
pr_title = "MNT: Add noarch: python"
comment_msg = "made the recipe `noarch: python`"
to_close = ADD_NOARCH_MSG.search(title)
changed_anything |= make_noarch(git_repo)
changed_anything |= rerender(git_repo)
elif RERENDER_MSG.search(text):
pr_title = "MNT: rerender"
comment_msg = "rerendered the recipe"
to_close = RERENDER_MSG.search(title)
changed_anything |= rerender(git_repo)
if changed_anything:
git_repo.git.push("origin", forked_repo_branch)
pr_message = textwrap.dedent("""
Hi! This is the friendly automated conda-forge-webservice.
I've {} as instructed in #{}.{}
Here's a checklist to do before merging.
- [ ] Bump the build number if needed.
""").format(comment_msg, issue_num, extra_msg)
if to_close:
pr_message += "\nFixes #{}".format(issue_num)
pr = repo.create_pull(
pr_title, pr_message,
"master", "{}:{}".format(forked_user, forked_repo_branch))
message = textwrap.dedent("""
Hi! This is the friendly automated conda-forge-webservice.
I just wanted to let you know that I {} in {}/{}#{}.
""").format(comment_msg, org_name, repo_name, pr.number)
issue.create_comment(message)
else:
message = textwrap.dedent("""
Hi! This is the friendly automated conda-forge-webservice.
I've {} as requested, but nothing actually changed.
""").format(comment_msg)
issue.create_comment(message)
if to_close:
issue.edit(state="closed")
def testBadCredentials(self):
self.assertRaises(
github.BadCredentialsException,
lambda: github.Github("BadUser", "BadPassword").get_user().login)
def testNoAuthentication( self ):
g = github.Github()
self.assertEqual( g.get_user( "jacquev6" ).name, "Vincent Jacques" )
def __init__(self):
self.git_token = get_env('GIT_TOKEN')
self.git_instance = github.Github(self.git_token)
