def update_pr(gh_token, repo_id, pr_number):
from github import Github
con = Github(gh_token)
repo = con.get_repo(repo_id)
sdk_pr = repo.get_pull(pr_number)
# "get_files" of Github only download the first 300 files. Might not be enough.
package_names = {f.filename.split('/')[0] for f in sdk_pr.get_files() if f.filename.startswith("azure")}
# Get PR branch to push
head_repo = sdk_pr.head.repo.full_name
head_branch = sdk_pr.head.ref
branched_index = "{}@{}".format(head_repo, head_branch)
with tempfile.TemporaryDirectory() as temp_dir, \
manage_git_folder(gh_token, Path(temp_dir) / Path("sdk"), branched_index) as sdk_folder:
sdk_repo = Repo(str(sdk_folder))
configure_user(gh_token, sdk_repo)
for package_name in package_names:
if package_name.endswith("nspkg"):
_LOGGER.info("Skip nspkg packages for update PR")
continue
# Rebuild packaging
build_packaging_by_package_name(package_name, sdk_folder, build_conf=True)
# Commit that
do_commit(
sdk_repo,
"Packaging update of {}".format(package_name),
head_branch,
None # Unused
)
# Push all commits at once
sdk_repo.git.push('origin', head_branch, set_upstream=True)
def update_pr(gh_token, repo_id, pr_number):
from github import Github
con = Github(gh_token)
repo = con.get_repo(repo_id)
sdk_pr = repo.get_pull(pr_number)
files = [
one_file.filename for one_file in sdk_pr.get_files()
if one_file.status not in ['removed']
]
# "get_files" of Github only download the first 300 files. Might not be enough.
package_names = {('.', f.split('/')[0])
for f in files if f.startswith("azure")}
# Handle the SDK folder as well
matches = {_SDK_FOLDER_RE.search(f) for f in files}
package_names.update(
{match.groups()
for match in matches if match is not None})
# Get PR branch to push
head_repo = sdk_pr.head.repo.full_name
head_branch = sdk_pr.head.ref
branched_index = "{}@{}".format(head_repo, head_branch)
_LOGGER.info("Checkout %s", branched_index)
with tempfile.TemporaryDirectory() as temp_dir, \
manage_git_folder(gh_token, Path(temp_dir) / Path("sdk"), branched_index) as sdk_repo_root:
sdk_repo = Repo(str(sdk_repo_root))
configure_user(gh_token, sdk_repo)
for base_folder, package_name in package_names:
if package_name.endswith("nspkg"):
_LOGGER.info("Skip nspkg packages for update PR")
continue
# Rebuild packaging
_LOGGER.info("Try update package %s from folder %s", package_name,
base_folder)
build_packaging_by_package_name(package_name,
sdk_repo_root / Path(base_folder),
build_conf=True)
# Commit that
do_commit(
sdk_repo,
"Packaging update of {}".format(package_name),
head_branch,
None # Unused
)
# Push all commits at once
sdk_repo.git.push('origin', head_branch, set_upstream=True)
def test_do_commit():
finished = False # Authorize PermissionError on cleanup
try:
with tempfile.TemporaryDirectory() as temp_dir:
Repo.clone_from('https://github.com/lmazuel/TestingRepo.git',
temp_dir)
repo = Repo(temp_dir)
result = do_commit(repo, 'Test {hexsha}', 'testing', 'fakehexsha')
assert not result
assert 'fakehexsha' not in repo.head.commit.message
assert repo.active_branch.name == 'master'
file_path = Path(temp_dir, 'file.txt')
file_path.write_text('Something')
result = do_commit(repo, 'Test {hexsha}', 'testing', 'fakehexsha')
assert result
assert repo.head.commit.message == 'Test fakehexsha'
assert repo.active_branch.name == 'testing'
assert 'file.txt' in repo.head.commit.stats.files
file_path.write_text('New content')
result = do_commit(repo, 'Now it is {hexsha}', 'newbranch',
'new-fakehexsha')
assert result
assert repo.head.commit.message == 'Now it is new-fakehexsha'
assert repo.active_branch.name == 'newbranch'
assert 'file.txt' in repo.head.commit.stats.files
file_path.unlink()
file_path.write_text('New content')
result = do_commit(repo, 'Now it is {hexsha}', 'fakebranch',
'hexsha_not_used')
assert not result
assert repo.head.commit.message == 'Now it is new-fakehexsha'
assert repo.active_branch.name == 'newbranch'
finished = True
except PermissionError:
if finished:
return
raise
def test_do_commit():
finished = False # Authorize PermissionError on cleanup
try:
with tempfile.TemporaryDirectory() as temp_dir:
Repo.clone_from("https://github.com/lmazuel/TestingRepo.git",
temp_dir)
repo = Repo(temp_dir)
result = do_commit(repo, "Test {hexsha}", "testing", "fakehexsha")
assert not result
assert "fakehexsha" not in repo.head.commit.message
assert repo.active_branch.name == "master"
file_path = Path(temp_dir, "file.txt")
file_path.write_text("Something")
result = do_commit(repo, "Test {hexsha}", "testing", "fakehexsha")
assert result
assert repo.head.commit.message == "Test fakehexsha"
assert repo.active_branch.name == "testing"
assert "file.txt" in repo.head.commit.stats.files
file_path.write_text("New content")
result = do_commit(repo, "Now it is {hexsha}", "newbranch",
"new-fakehexsha")
assert result
assert repo.head.commit.message == "Now it is new-fakehexsha"
assert repo.active_branch.name == "newbranch"
assert "file.txt" in repo.head.commit.stats.files
file_path.unlink()
file_path.write_text("New content")
result = do_commit(repo, "Now it is {hexsha}", "fakebranch",
"hexsha_not_used")
assert not result
assert repo.head.commit.message == "Now it is new-fakehexsha"
assert repo.active_branch.name == "newbranch"
finished = True
except PermissionError:
if finished:
return
raise
def rebuild(self, issue, project_pattern):
if not issue.pull_request:
return "Rebuild is just supported in PR for now"
sdkid = issue.repository.full_name
pr = issue.repository.get_pull(issue.number)
new_comment = issue.create_comment("Working on generating {} for you!!!".format(project_pattern))
config_path = CONFIG_FILE
message = "Rebuild by "+issue.html_url
autorest_bin = None
branch_name = pr.head.ref
branched_sdk_id = pr.head.repo.full_name+'@'+branch_name
if project_pattern.startswith("https://"):
link = GithubLink.from_string(project_pattern)
link = link.as_raw_link() # Ensure this is a raw link.
rest_api_id = link.gitid
rest_api_branch = link.branch_or_commit
token = link.token if link.token else self.gh_token
path = link.path
else:
rest_api_id = "Azure/azure-rest-api-specs"
rest_api_branch = "master"
token = self.gh_token
path = None # Not such notion of path here, since it's inside SwaggerToSdk conf
branched_rest_api_id = rest_api_id + "@" + rest_api_branch
config = read_config_from_github(pr.head.repo.full_name, branch_name, token)
with tempfile.TemporaryDirectory() as temp_dir, \
manage_git_folder(token, Path(temp_dir) / Path("rest"), branched_rest_api_id) as restapi_git_folder, \
manage_git_folder(self.gh_token, Path(temp_dir) / Path("sdk"), branched_sdk_id) as sdk_folder:
sdk_repo = Repo(str(sdk_folder))
configure_user(self.gh_token, sdk_repo)
if path: # Assume this is a Readme path
config["projects"] = {} # Wipe out everything
build_swaggertosdk_conf_from_json_readme(path, sdkid, config, base_folder=restapi_git_folder)
skip_callback = lambda x, y: False
else:
def skip_callback(project, local_conf):
del local_conf # Unused
if not project.startswith(project_pattern):
return True
return False
from swaggertosdk import SwaggerToSdkNewCLI
SwaggerToSdkNewCLI.build_libraries(config, skip_callback, restapi_git_folder,
sdk_repo, temp_dir, autorest_bin)
new_comment.edit("End of generation, doing commit")
commit_sha = do_commit(sdk_repo, message, branch_name, "")
if commit_sha:
new_comment.edit("Pushing")
sdk_repo.git.push('origin', branch_name, set_upstream=True)
new_comment.delete()
else:
new_comment.delete()
return "Nothing to rebuild, this PR is up to date"
_LOGGER.info("Build SDK finished and cleaned")
return "Build SDK finished and cleaned"
def generate_sdk_from_git_object(git_object, branch_name, restapi_git_id, sdk_git_id, base_branch_names, *, fallback_base_branch_name="master", sdk_tag=None):
"""Generate SDK from a commit or a PR object.
git_object is the initial commit/PR from the RestAPI repo. If git_object is a PR, prefer to checkout Github PR "merge_commit_sha"
restapi_git_id explains where to clone the repo.
sdk_git_id explains where to push the commit.
sdk_tag explains what is the tag used in the Readme for the swagger-to-sdk section. If not provided, use sdk_git_id.
branch_name is the expected branch name in the SDK repo.
- If this branch exists, use it.
- If not, use the base branch to create that branch (base branch is where I intend to do my PR)
- If base_branch_names is not provided, use fallback_base_branch_name as base
- If this base branch is provided and does not exists, create this base branch first using fallback_base_branch_name (this one is required to exist)
WARNING:
This method might push to "branch_name" and "base_branch_name". No push will be made to "fallback_base_branch_name"
"""
gh_token = os.environ["GH_TOKEN"]
message_template = DEFAULT_COMMIT_MESSAGE
autorest_bin = None
if sdk_tag is None:
sdk_tag = sdk_git_id
try: # Checkout the sha if commit obj
branched_rest_api_id = restapi_git_id+'@'+git_object.sha
pr_number = None
except (AttributeError, TypeError): # This is a PR, don't clone the fork but "base" repo and PR magic commit
if git_object.merge_commit_sha:
branched_rest_api_id = git_object.base.repo.full_name+'@'+git_object.merge_commit_sha
else:
branched_rest_api_id = git_object.base.repo.full_name
pr_number = git_object.number
# Always clone SDK from fallback branch that is required to exist
branched_sdk_git_id = sdk_git_id+'@'+fallback_base_branch_name
# I don't know if the destination branch exists, try until it works
config = None
branch_list = base_branch_names + [branch_name] + [fallback_base_branch_name]
for branch in branch_list:
try:
config = read_config_from_github(sdk_git_id, branch, gh_token)
except Exception:
pass
else:
break
if config is None:
raise ValueError("Unable to locate configuration in {}".format(branch_list))
global_conf = config["meta"]
# If PR is only about a language that this conf can't handle, skip fast
if not this_conf_will_generate_for_this_pr(git_object, global_conf):
_LOGGER.info("Skipping this job based on conf not impacted by Git object")
return
with tempfile.TemporaryDirectory() as temp_dir:
clone_dir = Path(temp_dir) / Path(global_conf.get("advanced_options", {}).get("clone_dir", "sdk"))
_LOGGER.info("Clone dir will be: %s", clone_dir)
with manage_git_folder(gh_token, Path(temp_dir) / Path("rest"), branched_rest_api_id, pr_number=pr_number) as restapi_git_folder, \
manage_git_folder(gh_token, clone_dir, branched_sdk_git_id) as sdk_folder:
readme_files_infered = get_readme_files_from_git_object(git_object, restapi_git_folder)
_LOGGER.info("Readmes files infered from PR: %s ", readme_files_infered)
if not readme_files_infered:
_LOGGER.info("No Readme in PR, quit")
return
# SDK part
sdk_repo = Repo(str(sdk_folder))
for base_branch in base_branch_names:
_LOGGER.info('Checkout and create %s', base_branch)
checkout_and_create_branch(sdk_repo, base_branch)
_LOGGER.info('Try to checkout destination branch %s', branch_name)
try:
sdk_repo.git.checkout(branch_name)
_LOGGER.info('The branch exists.')
except GitCommandError:
_LOGGER.info('Destination branch does not exists')
# Will be created by do_commit
configure_user(gh_token, sdk_repo)
# Look for configuration in Readme
_LOGGER.info('Extract conf from Readmes for target: %s', sdk_git_id)
extract_conf_from_readmes(readme_files_infered, restapi_git_folder, sdk_tag, config)
_LOGGER.info('End of extraction')
def skip_callback(project, local_conf):
# We know "project" is based on Path in "readme_files_infered"
if Path(project) in readme_files_infered:
return False
# Might be a regular project
markdown_relative_path, optional_relative_paths = get_input_paths(global_conf, local_conf)
if not (
markdown_relative_path in readme_files_infered or
any(input_file in readme_files_infered for input_file in optional_relative_paths)):
_LOGGER.info(f"In project {project} no files involved in this commit")
return True
return False
build_libraries(config, skip_callback, restapi_git_folder,
sdk_repo, temp_dir, autorest_bin)
try:
commit_for_sha = git_object.commit # Commit
except AttributeError:
commit_for_sha = list(git_object.get_commits())[-1].commit # PR
message = message_template + "\n\n" + commit_for_sha.message
commit_sha = do_commit(sdk_repo, message, branch_name, commit_for_sha.sha)
if commit_sha:
for base_branch in base_branch_names:
sdk_repo.git.push('origin', base_branch, set_upstream=True)
sdk_repo.git.push('origin', branch_name, set_upstream=True)
return "https://github.com/{}/commit/{}".format(sdk_git_id, commit_sha)
def generate_sdk_from_git_object(git_object,
branch_name,
restapi_git_id,
sdk_git_id,
base_branch_names,
*,
fallback_base_branch_name="master",
sdk_tag=None):
"""Generate SDK from a commit or a PR object.
git_object is the initial commit/PR from the RestAPI repo. If git_object is a PR, prefer to checkout Github PR "merge_commit_sha"
restapi_git_id explains where to clone the repo.
sdk_git_id explains where to push the commit.
sdk_tag explains what is the tag used in the Readme for the swagger-to-sdk section. If not provided, use sdk_git_id.
branch_name is the expected branch name in the SDK repo.
- If this branch exists, use it.
- If not, use the base branch to create that branch (base branch is where I intend to do my PR)
- If base_branch_names is not provided, use fallback_base_branch_name as base
- If this base branch is provided and does not exists, create this base branch first using fallback_base_branch_name (this one is required to exist)
WARNING:
This method might push to "branch_name" and "base_branch_name". No push will be made to "fallback_base_branch_name"
"""
gh_token = os.environ["GH_TOKEN"]
message_template = DEFAULT_COMMIT_MESSAGE
autorest_bin = None
if sdk_tag is None:
sdk_tag = sdk_git_id
try: # Checkout the sha if commit obj
branched_rest_api_id = restapi_git_id + '@' + git_object.sha
pr_number = None
except (
AttributeError, TypeError
): # This is a PR, don't clone the fork but "base" repo and PR magic commit
if git_object.merge_commit_sha:
branched_rest_api_id = git_object.base.repo.full_name + '@' + git_object.merge_commit_sha
else:
branched_rest_api_id = git_object.base.repo.full_name
pr_number = git_object.number
# Always clone SDK from fallback branch that is required to exist
branched_sdk_git_id = sdk_git_id + '@' + fallback_base_branch_name
# I don't know if the destination branch exists, try until it works
config = None
branch_list = base_branch_names + [branch_name
] + [fallback_base_branch_name]
for branch in branch_list:
try:
config = read_config_from_github(sdk_git_id, branch, gh_token)
except Exception:
pass
else:
break
if config is None:
raise ValueError(
"Unable to locate configuration in {}".format(branch_list))
global_conf = config["meta"]
# If PR is only about a language that this conf can't handle, skip fast
if not this_conf_will_generate_for_this_pr(git_object, global_conf):
_LOGGER.info(
"Skipping this job based on conf not impacted by Git object")
return
with tempfile.TemporaryDirectory() as temp_dir:
clone_dir = Path(temp_dir) / Path(
global_conf.get("advanced_options", {}).get("clone_dir", "sdk"))
_LOGGER.info("Clone dir will be: %s", clone_dir)
with manage_git_folder(gh_token, Path(temp_dir) / Path("rest"), branched_rest_api_id, pr_number=pr_number) as restapi_git_folder, \
manage_git_folder(gh_token, clone_dir, branched_sdk_git_id) as sdk_folder:
readme_files_infered = get_readme_files_from_git_object(
git_object, restapi_git_folder)
_LOGGER.info("Readmes files infered from PR: %s ",
readme_files_infered)
if not readme_files_infered:
_LOGGER.info("No Readme in PR, quit")
return
# SDK part
sdk_repo = Repo(str(sdk_folder))
for base_branch in base_branch_names:
_LOGGER.info('Checkout and create %s', base_branch)
checkout_and_create_branch(sdk_repo, base_branch)
_LOGGER.info('Try to checkout destination branch %s', branch_name)
try:
sdk_repo.git.checkout(branch_name)
_LOGGER.info('The branch exists.')
except GitCommandError:
_LOGGER.info('Destination branch does not exists')
# Will be created by do_commit
configure_user(gh_token, sdk_repo)
# Look for configuration in Readme
_LOGGER.info('Extract conf from Readmes for target: %s',
sdk_git_id)
extract_conf_from_readmes(readme_files_infered, restapi_git_folder,
sdk_tag, config)
_LOGGER.info('End of extraction')
def skip_callback(project, local_conf):
# We know "project" is based on Path in "readme_files_infered"
if Path(project) in readme_files_infered:
return False
# Might be a regular project
markdown_relative_path, optional_relative_paths = get_input_paths(
global_conf, local_conf)
if not (markdown_relative_path in readme_files_infered
or any(input_file in readme_files_infered
for input_file in optional_relative_paths)):
_LOGGER.info(
f"In project {project} no files involved in this commit"
)
return True
return False
build_libraries(config, skip_callback, restapi_git_folder,
sdk_repo, temp_dir, autorest_bin)
try:
commit_for_sha = git_object.commit # Commit
except AttributeError:
commit_for_sha = list(
git_object.get_commits())[-1].commit # PR
message = message_template + "\n\n" + commit_for_sha.message
commit_sha = do_commit(sdk_repo, message, branch_name,
commit_for_sha.sha)
if commit_sha:
for base_branch in base_branch_names:
sdk_repo.git.push('origin', base_branch, set_upstream=True)
sdk_repo.git.push('origin', branch_name, set_upstream=True)
return "https://github.com/{}/commit/{}".format(
sdk_git_id, commit_sha)
def rebuild(self, issue, project_pattern):
if not issue.pull_request:
return "Rebuild is just supported in PR for now"
sdkid = issue.repository.full_name
pr = issue.repository.get_pull(issue.number)
new_comment = issue.create_comment(
"Working on generating {} for you!!!".format(project_pattern))
config_path = CONFIG_FILE
message = "Rebuild by " + issue.html_url
autorest_bin = None
branch_name = pr.head.ref
branched_sdk_id = pr.head.repo.full_name + '@' + branch_name
if project_pattern.startswith("https://"):
link = GithubLink.from_string(project_pattern)
link = link.as_raw_link() # Ensure this is a raw link.
rest_api_id = link.gitid
rest_api_branch = link.branch_or_commit
token = link.token if link.token else self.gh_token
path = link.path
else:
rest_api_id = "Azure/azure-rest-api-specs"
rest_api_branch = "master"
token = self.gh_token
path = None # Not such notion of path here, since it's inside SwaggerToSdk conf
branched_rest_api_id = rest_api_id + "@" + rest_api_branch
config = read_config_from_github(pr.head.repo.full_name, branch_name,
token)
with tempfile.TemporaryDirectory() as temp_dir, \
manage_git_folder(token, Path(temp_dir) / Path("rest"), branched_rest_api_id) as restapi_git_folder, \
manage_git_folder(self.gh_token, Path(temp_dir) / Path("sdk"), branched_sdk_id) as sdk_folder:
sdk_repo = Repo(str(sdk_folder))
configure_user(self.gh_token, sdk_repo)
if path: # Assume this is a Readme path
config["projects"] = {} # Wipe out everything
build_swaggertosdk_conf_from_json_readme(
path, sdkid, config, base_folder=restapi_git_folder)
skip_callback = lambda x, y: False
else:
def skip_callback(project, local_conf):
del local_conf # Unused
if not project.startswith(project_pattern):
return True
return False
from swaggertosdk import SwaggerToSdkNewCLI
SwaggerToSdkNewCLI.build_libraries(config, skip_callback,
restapi_git_folder, sdk_repo,
temp_dir, autorest_bin)
new_comment.edit("End of generation, doing commit")
commit_sha = do_commit(sdk_repo, message, branch_name, "")
if commit_sha:
new_comment.edit("Pushing")
sdk_repo.git.push('origin', branch_name, set_upstream=True)
new_comment.delete()
else:
new_comment.delete()
return "Nothing to rebuild, this PR is up to date"
_LOGGER.info("Build SDK finished and cleaned")
return "Build SDK finished and cleaned"