def request(self, method: str, credentials: Dict, url: str, json: Dict):
try:
response = requests.request(
method,
url,
headers={
"Authorization": "token {}".format(credentials["token"]),
},
json=json,
)
except OSError as error:
report_error(cause="request")
raise RepositoryException(0, str(error))
self.add_response_breadcrumb(response)
try:
data = response.json()
except JSONDecodeError as error:
report_error(cause="request json decoding")
response.raise_for_status()
raise RepositoryException(0, str(error))
# Log and parse all errors.
error_message = ""
if "message" in data:
error_message = data["message"]
self.log(data["message"], level=logging.INFO)
return data, error_message
def create_pull_request(self, credentials: Dict, origin_branch: str,
fork_remote: str, fork_branch: str):
"""Create pull request.
Use to merge branch in forked repository into branch of remote repository.
"""
if credentials["owner"]:
pr_list_url = "{url}/{owner}/{slug}/pull-requests".format(
**credentials)
pr_create_url = "{url}/{owner}/{slug}/pull-request/new".format(
**credentials)
else:
pr_list_url = "{url}/{slug}/pull-requests".format(**credentials)
pr_create_url = "{url}/{slug}/pull-request/new".format(
**credentials)
# List existing pull requests
response, error_message = self.request(
"get",
credentials,
pr_list_url,
params={"author": credentials["username"]})
if error_message:
raise RepositoryException(
0, f"Pull request listing failed: {error_message}")
if response["total_requests"] > 0:
# Open pull request from us is already there
return
title, description = self.get_merge_message()
request = {
"branch_from": fork_branch,
"branch_to": origin_branch,
"title": title,
"initial_comment": description,
}
if fork_remote != "origin":
request["repo_from"] = credentials["slug"]
request["repo_from_username"] = credentials["username"]
response, error_message = self.request("post",
credentials,
pr_create_url,
data=request)
if "id" not in response:
raise RepositoryException(0,
f"Pull request failed: {error_message}")
def disable_fork_features(self, forked_url):
"""Disable features in fork.
Gitlab initializes a lot of the features in the fork
that are not desirable, such as merge requests, issues, etc.
This function is intended to disable all such features by
editing the forked repo.
"""
access_level_dict = {
"issues_access_level": "disabled",
"forking_access_level": "disabled",
"builds_access_level": "disabled",
"wiki_access_level": "disabled",
"snippets_access_level": "disabled",
"pages_access_level": "disabled",
}
r = requests.put(
forked_url,
headers={
"Authorization": "Bearer {}".format(settings.GITLAB_TOKEN)
},
json=access_level_dict,
)
if "web_url" not in r.json():
raise RepositoryException(0, r.json()["error"])
def create_pull_request(self, credentials: Dict, origin_branch: str,
fork_remote: str, fork_branch: str):
"""Create pull request.
Use to merge branch in forked repository into branch of remote repository.
"""
target_project_id = None
pr_url = "{}/merge_requests".format(credentials["url"])
if fork_remote != "origin":
# Gitlab MR works a little different from Github. The MR needs
# to be sent with the fork's API URL along with a parameter mentioning
# the target project id
target_project_id = self.get_target_project_id(credentials)
pr_url = "{}/merge_requests".format(
self.get_forked_url(credentials))
title, description = self.get_merge_message()
request = {
"source_branch": fork_branch,
"target_branch": origin_branch,
"title": title,
"description": description,
"target_project_id": target_project_id,
}
response, error = self.request("post", credentials, pr_url, request)
if ("web_url" not in response
and "open merge request already exists" not in error):
raise RepositoryException(-1, error
or "Failed to create pull request")
def get_credentials(self) -> Dict:
url, owner, slug = self.get_api_url()
hostname = urllib.parse.urlparse(url).hostname.lower()
credentials = getattr(settings,
f"{self.identifier.upper()}_CREDENTIALS")
if hostname in credentials:
username = credentials[hostname]["username"]
token = credentials[hostname]["token"]
else:
username = getattr(settings, f"{self.identifier.upper()}_USERNAME")
token = getattr(settings, f"{self.identifier.upper()}_TOKEN")
if not username or not token:
raise RepositoryException(
0,
f"{self.name} API access for {hostname} is not configured")
return {
"url": url,
"owner": owner,
"slug": slug,
"hostname": hostname,
"username": username,
"token": token,
}
def create_fork(self, credentials: Dict):
fork_url = "{}/fork".format(credentials["url"])
base_params = {
"repo": credentials["slug"],
"wait": True,
}
if credentials["owner"]:
# We have no information whether the URL part is namespace
# or username, try both
params = [
{
"namespace": credentials["owner"]
},
{
"username": credentials["owner"]
},
]
else:
params = [{}]
for param in params:
param.update(base_params)
response, error = self.request("post", credentials, fork_url,
param)
if '" cloned to "' in error or "already exists" in error:
break
if '" cloned to "' not in error and "already exists" not in error:
raise RepositoryException(0, error or "Failed to create fork")
url = "ssh://git@{hostname}/forks/{username}/{slug}.git".format(
**credentials)
self.configure_fork_remote(url, credentials["username"])
def create_pull_request(self, credentials: Dict, origin_branch: str,
fork_remote: str, fork_branch: str):
"""Create pull request.
Use to merge branch in forked repository into branch of remote repository.
"""
if fork_remote == "origin":
head = fork_branch
else:
head = "{0}:{1}".format(fork_remote, fork_branch)
pr_url = "{}/pulls".format(credentials["url"])
title, description = self.get_merge_message()
request = {
"head": head,
"base": origin_branch,
"title": title,
"body": description,
}
response, error_message = self.request("post", credentials, pr_url,
request)
# Check for an error. If the error has a message saying A pull request already
# exists, then we ignore that, else raise an error. Currently, since the API
# doesn't return any other separate indication for a pull request existing
# compared to other errors, checking message seems to be the only option
if "url" not in response:
# Gracefully handle pull request already exists or nothing to merge cases
if ("A pull request already exists" in error_message
or "No commits between " in error_message):
return
raise RepositoryException(0, error_message
or "Pull request failed")
def create_pull_request(self, credentials: Dict, origin_branch: str,
fork_remote: str, fork_branch: str):
"""Create pull request.
Use to merge branch in forked repository into branch of remote repository.
"""
if fork_remote == "origin":
if credentials["owner"]:
pr_url = "{url}/{owner}/{slug}/pull-request/new".format(
**credentials)
else:
pr_url = "{url}/{slug}/pull-request/new".format(**credentials)
else:
pr_url = "{url}/fork/{username}/{slug}/pull-request/new".format(
**credentials)
title, description = self.get_merge_message()
request = {
"branch_from": fork_branch,
"branch_to": origin_branch,
"title": title,
"initial_comment": description,
}
response, error_message = self.request("post", credentials, pr_url,
request)
if "id" not in response:
raise RepositoryException(0, error_message
or "Pull request failed")
def request(self, method: str, credentials: Dict, url: str, json: Dict):
response = requests.request(
method,
url,
headers={
"Accept": "application/vnd.github.v3+json",
"Authorization": "token {}".format(credentials["token"]),
},
json=json,
)
try:
data = response.json()
except JSONDecodeError as error:
response.raise_for_status()
raise RepositoryException(0, str(error))
# Log and parase all errors. Sometimes GitHub returns the error
# messages in an errors list instead of the message. Sometimes, there
# is no errors list. Hence the different logics
error_message = ""
if "message" in data:
error_message = data["message"]
self.log(data["message"], level=logging.INFO)
if "errors" in data:
messages = []
for error in data["errors"]:
line = error.get("message", str(error))
messages.append(line)
self.log(line, level=logging.WARNING)
if error_message:
error_message += ": "
error_message += ", ".join(messages)
return data, error_message
def create_pull_request(
self, credentials: Dict, origin_branch: str, fork_remote: str, fork_branch: str
):
"""Create pull request.
Use to merge branch in forked repository into branch of remote repository.
"""
if fork_remote == "origin":
head = fork_branch
else:
head = "{0}:{1}".format(fork_remote, fork_branch)
pr_url = "{}/pulls".format(credentials["url"])
title, description = self.get_merge_message()
request = {
"head": head,
"base": origin_branch,
"title": title,
"body": description,
}
response = requests.post(
pr_url,
headers={
"Accept": "application/vnd.github.v3+json",
"Authorization": "token {}".format(credentials["token"]),
},
json=request,
).json()
# Log all errors
if "message" in response:
self.log(response["message"], level=logging.INFO)
if "errors" in response:
for error in response["errors"]:
self.log(error.get("message", str(error)), level=logging.WARNING)
# Check for an error. If the error has a message saying A pull request already
# exists, then we ignore that, else raise an error. Currently, since the API
# doesn't return any other separate indication for a pull request existing
# compared to other errors, checking message seems to be the only option
if "url" not in response:
# Gracefully handle pull request already exists case
if (
"errors" in response
and "A pull request already exists" in response["errors"][0]["message"]
):
return
# Sometimes GitHub returns the error messages in an errors list
# instead of the message. Sometimes, there is no errors list.
# Hence the different logics
error_message = "Pull request failed"
if "errors" in response:
error_message = "{}: {}".format(
response["message"], response["errors"][0]["message"]
)
elif "message" in response:
error_message = response["message"]
raise RepositoryException(0, error_message)
def create_fork(self, credentials: Dict):
fork_url = "{}/forks".format(credentials["url"])
# GitHub API returns the entire data of the fork, in case the fork
# already exists. Hence this is perfectly handled, if the fork already
# exists in the remote side.
response, error = self.request("post", credentials, fork_url, {})
if "ssh_url" not in response:
raise RepositoryException(0, f"Fork creation failed: {error}")
self.configure_fork_remote(response["ssh_url"], credentials["username"])
def create_fork(self, credentials: Dict):
fork_url = "{}/forks".format(credentials["url"])
response, error = self.request("post", credentials, fork_url, {})
if "message" in response and "repository is already forked by user" in error:
# we have to get the repository again if it is already forked
response, error = self.request("get", credentials, credentials["url"], {})
if "ssh_url" not in response:
raise RepositoryException(0, f"Fork creation failed: {error}")
self.configure_fork_remote(response["ssh_url"], credentials["username"])
def get_remote_branch(cls, repo: str):
if not repo:
return super().get_remote_branch(repo)
result = cls._popen(["ls-remote", "--symref", repo, "HEAD"])
for line in result.splitlines():
if not line.startswith("ref: "):
continue
# Parses 'ref: refs/heads/master\tHEAD'
return line.split("\t")[0].split("refs/heads/")[1]
raise RepositoryException(0, "Failed to figure out remote branch")
def create_pull_request(
self,
credentials: Dict,
origin_branch: str,
fork_remote: str,
fork_branch: str,
retry_fork: bool = True,
):
"""Create pull request.
Use to merge branch in forked repository into branch of remote repository.
"""
if fork_remote == "origin":
head = fork_branch
else:
head = f"{fork_remote}:{fork_branch}"
pr_url = "{}/pulls".format(credentials["url"])
title, description = self.get_merge_message()
request = {
"head": head,
"base": origin_branch,
"title": title,
"body": description,
}
response, error_message = self.request("post", credentials, pr_url, request)
# Check for an error. If the error has a message saying A pull request already
# exists, then we ignore that, else raise an error. Currently, since the API
# doesn't return any other separate indication for a pull request existing
# compared to other errors, checking message seems to be the only option
if "url" not in response:
# Gracefully handle pull request already exists or nothing to merge cases
if (
"A pull request already exists" in error_message
or "No commits between " in error_message
):
return
if "Validation Failed" in error_message:
for error in response["errors"]:
if error.get("field") == "head" and retry_fork:
# This most likely indicates that Weblate repository has moved
# and we should create a fresh fork.
self.create_fork(credentials)
self.create_pull_request(
credentials,
origin_branch,
fork_remote,
fork_branch,
retry_fork=False,
)
return
raise RepositoryException(0, f"Pull request failed: {error_message}")
def create_pull_request(
self, credentials: Dict, origin_branch: str, fork_remote: str, fork_branch: str
):
"""Create pull request.
Use to merge branch in forked repository into branch of remote repository.
"""
target_project_id = None
pr_url = "{}/merge_requests".format(credentials["url"])
if fork_remote != "origin":
# Gitlab MR works a little different from Github. The MR needs
# to be sent with the fork's API URL along with a parameter mentioning
# the target project id
target_project_id = self.get_target_project_id(credentials)
pr_url = "{}/merge_requests".format(self.get_forked_url(credentials))
title, description = self.get_merge_message()
request = {
"source_branch": fork_branch,
"target_branch": origin_branch,
"title": title,
"description": description,
"target_project_id": target_project_id,
}
response = requests.post(
pr_url,
headers={"Authorization": "Bearer {}".format(credentials["token"])},
json=request,
).json()
# Extract messages
messages = response.get("message", [])
if not isinstance(messages, list):
messages = [messages]
# Log messages
for message in messages:
self.log(message, level=logging.INFO)
if (
"web_url" not in response
and "open merge request already exists" not in messages[0]
):
raise RepositoryException(-1, ", ".join(messages))
def configure_remote(self, pull_url, push_url, branch):
"""Initialize the git-svn repository.
This does not support switching remote as it's quite complex:
https://git.wiki.kernel.org/index.php/GitSvnSwitch
The git svn init errors in case the URL is not matching.
"""
try:
existing = self.get_config("svn-remote.svn.url")
except RepositoryException:
existing = None
if existing:
# The URL is root of the repository, while we get full path
if not pull_url.startswith(existing):
raise RepositoryException(-1, "Can not switch subversion URL")
return
args, self._fetch_revision = self.get_remote_args(pull_url, self.path)
self.execute(["svn", "init"] + args)
def disable_fork_features(self, credentials: Dict, forked_url: str):
"""Disable features in fork.
Gitlab initializes a lot of the features in the fork
that are not desirable, such as merge requests, issues, etc.
This function is intended to disable all such features by
editing the forked repo.
"""
access_level_dict = {
"issues_access_level": "disabled",
"forking_access_level": "disabled",
"builds_access_level": "disabled",
"wiki_access_level": "disabled",
"snippets_access_level": "disabled",
"pages_access_level": "disabled",
}
response, error = self.request("put", credentials, forked_url,
access_level_dict)
if "web_url" not in response:
raise RepositoryException(0, error or "Failed to modify fork")
def create_fork(self, credentials: Dict):
get_fork_url = "{}/forks?owned=True".format(credentials["url"])
fork_url = "{}/fork".format(credentials["url"])
forked_repo = None
# Check if Fork already exists owned by current user. If the
# fork already exists, set that fork as remote.
# Else, create a new fork
response, error = self.request("get", credentials, get_fork_url)
for fork in response:
# Since owned=True returns forks from both the user's repo and the forks
# in all the groups owned by the user, hence we need the below logic
# to find the fork within the user repo and not the groups
if "owner" in fork and fork["owner"]["username"] == credentials[
"username"]:
forked_repo = fork
if forked_repo is None:
forked_repo, error = self.request("post", credentials, fork_url)
# If a repo with the name of the fork already exist, append numeric
# as suffix to name and path to use that as repo name and path.
if "ssh_url_to_repo" not in response and "has already been taken" in error:
fork_name = "{}-{}".format(credentials["url"].split("%2F")[-1],
random.randint(1000, 9999))
forked_repo, error = self.request(
"post",
credentials,
fork_url,
{
"name": fork_name,
"path": fork_name
},
)
if "ssh_url_to_repo" not in forked_repo:
raise RepositoryException(0, f"Failed to create fork: {error}")
self.configure_fork_features(credentials,
forked_repo["_links"]["self"])
self.configure_fork_remote(forked_repo["ssh_url_to_repo"],
credentials["username"])
def create_pull_request(self, origin_branch, fork_remote, fork_branch):
"""Create pull request.
Use to merge branch in forked repository into branch of remote repository.
"""
target_project_id = None
pr_url = "{}/merge_requests".format(self.api_url())
if fork_remote != "origin":
# Gitlab MR works a little different from Github. The MR needs
# to be sent with the fork's API URL along with a parameter mentioning
# the target project id
target_project_id = self.get_target_project_id()
pr_url = "{}/merge_requests".format(self.get_forked_url())
title, description = self.get_merge_message()
r = requests.post(
pr_url,
headers={
"Authorization": "Bearer {}".format(settings.GITLAB_TOKEN)
},
json={
"source_branch": fork_branch,
"target_branch": origin_branch,
"title": title,
"description": description,
"target_project_id": target_project_id,
},
)
response = r.json()
# Log messages
if "message" in response and isinstance(response["message"], list):
for message in response["message"]:
self.log(message, level=logging.INFO)
if "web_url" not in response and (not isinstance(
response["message"], list) or response["message"][0].find(
"open merge request already exists") == -1):
raise RepositoryException(-1, ", ".join(response["message"]))
def get_target_project_id(self, credentials: Dict):
response, error = self.request("get", credentials, credentials["url"])
if "id" not in response:
raise RepositoryException(0, error or "Failed to get project")
return response["id"]
