def create_or_update_comment(ghrequest, comment,
ONLY_UPDATE_COMMENT_BUT_NOT_CREATE):
query = "/repos/{}/issues/{}/comments"
query = query.format(ghrequest.repository, str(ghrequest.pr_number))
comments = utils._request(query).json()
# Get the last comment id by the bot
last_comment_id = None
for old_comment in comments:
if old_comment["user"]["id"] == 24736507: # ID of @pep8speaks
last_comment_id = old_comment["id"]
break
if last_comment_id is None and not ONLY_UPDATE_COMMENT_BUT_NOT_CREATE: # Create a new comment
response = utils._request(query=query,
method='POST',
json={"body": comment})
ghrequest.comment_response = response.json()
else: # Update the last comment
utc_time = datetime.datetime.utcnow()
time_now = utc_time.strftime("%B %d, %Y at %H:%M Hours UTC")
comment += "\n\n##### Comment last updated on {}"
comment = comment.format(time_now)
query = "/repos/{}/issues/comments/{}"
query = query.format(ghrequest.repository, str(last_comment_id))
response = utils._request(query,
method='PATCH',
json={"body": comment})
return response
def delete_if_forked(ghrequest):
FORKED = False
query = "/user/repos"
r = utils._request(query)
for repo in r.json():
if repo["description"]:
if ghrequest.target_repo_fullname in repo["description"]:
FORKED = True
url = "/repos/{}"
url = url.format(repo["full_name"])
utils._request(url, method='DELETE')
return FORKED
def autopep8(ghrequest, config):
# Run pycodestyle
r = utils._request(ghrequest.diff_url)
## All the python files with additions
patch = unidiff.PatchSet(r.content.splitlines(), encoding=r.encoding)
# A dictionary with filename paired with list of new line numbers
py_files = {}
for patchset in patch:
if patchset.target_file[-3:] == '.py':
py_file = patchset.target_file[1:]
py_files[py_file] = []
for hunk in patchset:
for line in hunk.target_lines():
if line.is_added:
py_files[py_file].append(line.target_line_no)
# Ignore errors and warnings specified in the config file
to_ignore = ",".join(config["pycodestyle"]["ignore"])
arg_to_ignore = ""
if len(to_ignore) > 0:
arg_to_ignore = "--ignore " + to_ignore
for file in py_files:
filename = file[1:]
url = "https://raw.githubusercontent.com/{}/{}/{}"
url = url.format(ghrequest.repository, ghrequest.sha, file)
r = utils._request(url)
with open("file_to_fix.py", 'w+', encoding=r.encoding) as file_to_fix:
file_to_fix.write(r.text)
cmd = 'autopep8 file_to_fix.py --diff {arg_to_ignore}'.format(
arg_to_ignore=arg_to_ignore)
proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
stdout, _ = proc.communicate()
ghrequest.diff[filename] = stdout.decode(r.encoding)
# Fix the errors
ghrequest.diff[filename] = ghrequest.diff[filename].replace(
"file_to_check.py", filename)
ghrequest.diff[filename] = ghrequest.diff[filename].replace(
"\\", "\\\\")
## Store the link to the file
url = "https://github.com/{}/blob/{}{}"
ghrequest.links = {}
ghrequest.links[filename + "_link"] = url.format(
ghrequest.repository, ghrequest.sha, file)
os.remove("file_to_fix.py")
def test_request(self, mocker, query, method, json, data, headers, params):
mock_func = mock.MagicMock(return_value=True)
mocker.patch('requests.request', mock_func)
_request(query, method, json, data, headers, params)
assert mock_func.call_count == 1
assert mock_func.call_args[0][0] == method
assert mock_func.call_args[1]['headers'] == headers
assert mock_func.call_args[1]['auth'] == ('', '')
assert mock_func.call_args[1]['params'] == params
assert mock_func.call_args[1]['json'] == json
if query[0] == "/":
assert mock_func.call_args[0][1] == BASE_URL + query
else:
assert mock_func.call_args[0][1] == query
def comment_permission_check(ghrequest):
"""
Check for quite and resume status or duplicate comments
"""
repository = ghrequest.repository
# Check for duplicate comment
url = "/repos/{}/issues/{}/comments"
url = url.format(repository, str(ghrequest.pr_number))
comments = utils._request(url).json()
"""
# Get the last comment by the bot
last_comment = ""
for old_comment in reversed(comments):
if old_comment["user"]["id"] == 24736507: # ID of @pep8speaks
last_comment = old_comment["body"]
break
# Disabling this because only a single comment is made per PR
text1 = ''.join(BeautifulSoup(markdown(comment)).findAll(text=True))
text2 = ''.join(BeautifulSoup(markdown(last_comment)).findAll(text=True))
if text1 == text2.replace("submitting", "updating"):
PERMITTED_TO_COMMENT = False
"""
# Check if the bot is asked to keep quiet
for old_comment in reversed(comments):
if '@pep8speaks' in old_comment['body']:
if 'resume' in old_comment['body'].lower():
break
elif 'quiet' in old_comment['body'].lower():
return False
# Check for [skip pep8]
## In commits
commits = utils._request(ghrequest.commits_url).json()
for commit in commits:
if any(m in commit["commit"]["message"].lower()
for m in ["[skip pep8]", "[pep8 skip]"]):
return False
## PR title
if any(m in ghrequest.pr_title.lower()
for m in ["[skip pep8]", "[pep8 skip]"]):
return False
## PR description
if any(m in ghrequest.pr_desc.lower()
for m in ["[skip pep8]", "[pep8 skip]"]):
return False
return True
def _create_diff(ghrequest, config):
# Dictionary with filename matched with a string of diff
ghrequest.diff = {}
# Process the files and prepare the diff for the gist
helpers.autopep8(ghrequest, config)
# Create the gist
helpers.create_gist(ghrequest, config)
comment = "Here you go with [the gist]({}) !\n\n" + \
"> You can ask me to create a PR against this branch " + \
"with those fixes. Simply comment " + \
"`@pep8speaks pep8ify`.\n\n"
if ghrequest.reviewer == ghrequest.author: # Both are the same person
comment += "@{} "
comment = comment.format(ghrequest.gist_url, ghrequest.reviewer)
else:
comment += "@{} @{} "
comment = comment.format(ghrequest.gist_url, ghrequest.reviewer,
ghrequest.author)
query = "/repos/{}/issues/{}/comments"
query = query.format(ghrequest.repository, str(ghrequest.pr_number))
response = utils._request(query, method='POST', json={"body": comment})
ghrequest.comment_response = response.json()
if ghrequest.error:
return utils.Response(ghrequest, status=400)
return utils.Response(ghrequest)
def follow_user(user):
"""Follow the user of the service"""
headers = {
"Content-Length": "0",
}
query = "/user/following/{}".format(user)
return utils._request(query=query, method='PUT', headers=headers)
def commit(ghrequest):
fullname = ghrequest.fork_fullname
for file, new_file in ghrequest.results.items():
query = "/repos/{}/contents/{}"
query = query.format(fullname, file)
params = {"ref": ghrequest.new_branch}
r = utils._request(query, params=params)
sha_blob = r.json().get("sha")
params["path"] = file
content_code = base64.b64encode(new_file.encode()).decode("utf-8")
request_json = {
"path": file,
"message": "Fix pep8 errors in {}".format(file),
"content": content_code,
"sha": sha_blob,
"branch": ghrequest.new_branch,
}
r = utils._request(query, method='PUT', json=request_json)
def create_new_branch(ghrequest):
query = "/repos/{}/git/refs/heads"
query = query.format(ghrequest.fork_fullname)
sha = None
r = utils._request(query)
for ref in r.json():
if ref["ref"].split("/")[-1] == ghrequest.target_repo_branch:
sha = ref["object"]["sha"]
query = "/repos/{}/git/refs"
query = query.format(ghrequest.fork_fullname)
ghrequest.new_branch = "{}-pep8-patch".format(ghrequest.target_repo_branch)
request_json = {
"ref": "refs/heads/{}".format(ghrequest.new_branch),
"sha": sha,
}
r = utils._request(query, method='POST', json=request_json)
if r.status_code > 299:
ghrequest.error = "Could not create new branch in the fork"
def fork_for_pr(ghrequest):
query = "/repos/{}/forks"
query = query.format(ghrequest.target_repo_fullname)
r = utils._request(query, method='POST')
if r.status_code == 202:
ghrequest.fork_fullname = r.json()["full_name"]
return True
ghrequest.error = "Unable to fork"
return False
def update_fork_desc(ghrequest):
# Check if forked (takes time)
query = "/repos/{}".format(ghrequest.fork_fullname)
r = utils._request(query)
ATTEMPT = 0
while (r.status_code != 200):
time.sleep(5)
r = utils._request(query)
ATTEMPT += 1
if ATTEMPT > 10:
ghrequest.error = "Forking is taking more than usual time"
break
full_name = ghrequest.target_repo_fullname
author, name = full_name.split("/")
request_json = {
"name": name,
"description": "Forked from @{}'s {}".format(author, full_name)
}
r = utils._request(query, method='PATCH', data=json.dumps(request_json))
if r.status_code != 200:
ghrequest.error = "Could not update description of the fork"
def run_pycodestyle(ghrequest, config):
"""
Runs the pycodestyle cli tool on the files and update ghrequest
"""
repo = ghrequest.repository
pr_number = ghrequest.pr_number
commit = ghrequest.after_commit_hash
# Run pycodestyle
## All the python files with additions
# A dictionary with filename paired with list of new line numbers
files_to_exclude = config["pycodestyle"]["exclude"]
py_files = get_py_files_in_pr(repo, pr_number, files_to_exclude)
for file in py_files:
filename = file[1:]
query = "https://raw.githubusercontent.com/{}/{}/{}"
query = query.format(repo, commit, file)
r = utils._request(query)
with open("file_to_check.py", 'w+',
encoding=r.encoding) as file_to_check:
file_to_check.write(r.text)
# Use the command line here
cmd = 'pycodestyle {config[pycodestyle_cmd_config]} file_to_check.py'.format(
config=config)
proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
stdout, _ = proc.communicate()
ghrequest.extra_results[filename] = stdout.decode(
r.encoding).splitlines()
# Put only relevant errors in the ghrequest.results dictionary
ghrequest.results[filename] = []
for error in list(ghrequest.extra_results[filename]):
if re.search("^file_to_check.py:\d+:\d+:\s[WE]\d+\s.*", error):
ghrequest.results[filename].append(
error.replace("file_to_check.py", filename))
ghrequest.extra_results[filename].remove(error)
## Remove errors in case of diff_only = True
## which are caused in the whole file
for error in list(ghrequest.results[filename]):
if config["scanner"]["diff_only"]:
if not int(error.split(":")[1]) in py_files[file]:
ghrequest.results[filename].remove(error)
## Store the link to the file
url = "https://github.com/{}/blob/{}{}"
ghrequest.links = {} # UI Link of each updated file in the PR
ghrequest.links[filename + "_link"] = url.format(repo, commit, file)
os.remove("file_to_check.py")
def create_pr(ghrequest):
query = "/repos/{}/pulls"
query = query.format(ghrequest.target_repo_fullname)
request_json = {
"title": "Fix pep8 errors",
"head": "pep8speaks:{}".format(ghrequest.new_branch),
"base": ghrequest.target_repo_branch,
"body": "The changes are suggested by autopep8",
}
r = utils._request(query, method='POST', json=request_json)
if r.status_code == 201:
ghrequest.pr_url = r.json()["html_url"]
else:
ghrequest.error = "Pull request could not be created"
def _get_pull_request(self, request, event):
"""
Get data about the pull request created
"""
if not self.OK:
return None
if event == "issue_comment":
pr_url = request.json['issue']['pull_request']['url']
pull_request = utils._request(pr_url).json()
elif event in ("pull_request", "pull_request_review"):
pull_request = request.json['pull_request']
else:
return None
return pull_request
def create_gist(ghrequest, config):
"""Create gists for diff files"""
request_json = {}
request_json["public"] = True
request_json["files"] = {}
request_json["description"] = "In response to @{0}'s comment : {1}".format(
ghrequest.reviewer, ghrequest.review_url)
for file, diffs in ghrequest.diff.items():
if len(diffs) != 0:
request_json["files"][file.split("/")[-1] + ".diff"] = {
"content": diffs
}
# Call github api to create the gist
query = "/gists"
response = utils._request(query, method='POST', json=request_json).json()
ghrequest.gist_response = response
ghrequest.gist_url = response["html_url"]
def get_files_involved_in_pr(repo, pr_number):
"""
Return a list of file names modified/added in the PR
"""
headers = {"Accept": "application/vnd.github.VERSION.diff"}
query = "/repos/{}/pulls/{}"
query = query.format(repo, pr_number)
r = utils._request(query, headers=headers)
patch = unidiff.PatchSet(r.content.splitlines(), encoding=r.encoding)
files = {}
for patchset in patch:
file = patchset.target_file[1:]
files[file] = []
for hunk in patchset:
for line in hunk.target_lines():
if line.is_added:
files[file].append(line.target_line_no)
return files
def _pep8ify(ghrequest, config):
ghrequest.target_repo_fullname = ghrequest.pull_request["head"]["repo"][
"full_name"]
ghrequest.target_repo_branch = ghrequest.pull_request["head"]["ref"]
ghrequest.results = {}
# Check if the fork of the target repo exists
# If yes, then delete it
helpers.delete_if_forked(ghrequest)
# Fork the target repository
helpers.fork_for_pr(ghrequest)
# Update the fork description. This helps in fast deleting it
helpers.update_fork_desc(ghrequest)
# Create a new branch for the PR
helpers.create_new_branch(ghrequest)
# Fix the errors in the files
helpers.autopep8ify(ghrequest, config)
# Commit each change onto the branch
helpers.commit(ghrequest)
# Create a PR from the branch to the target repository
helpers.create_pr(ghrequest)
comment = "Here you go with [the Pull Request]({}) ! The fixes are " \
"suggested by [autopep8](https://github.com/hhatto/autopep8).\n\n"
if ghrequest.reviewer == ghrequest.author: # Both are the same person
comment += "@{} "
comment = comment.format(ghrequest.pr_url, ghrequest.reviewer)
else:
comment += "@{} @{} "
comment = comment.format(ghrequest.pr_url, ghrequest.reviewer,
ghrequest.author)
query = "/repos/{}/issues/{}/comments"
query = query.format(ghrequest.repository, str(ghrequest.pr_number))
response = utils._request(query, method='POST', json={"body": comment})
ghrequest.comment_response = response.json()
return utils.Response(ghrequest)
def get_config(repo, base_branch):
"""
Get .pep8speaks.yml config file from the repository and return
the config dictionary
"""
# Default configuration parameters
config = {
"message": {
"opened": {
"header": "",
"footer": ""
},
"updated": {
"header": "",
"footer": ""
},
"no_errors":
"Cheers ! There are no PEP8 issues in this Pull Request. :beers: ",
},
"scanner": {
"diff_only": False
},
"pycodestyle": {
"ignore": [],
"max-line-length": 79,
"count": False,
"first": False,
"show-pep8": False,
"filename": [],
"exclude": [],
"select": [],
"show-source": False,
"statistics": False,
"hang-closing": False,
},
"no_blank_comment": True,
"only_mention_files_with_errors": True,
"descending_issues_order": False,
}
# Configuration file
query = "https://raw.githubusercontent.com/{}/{}/.pep8speaks.yml"
query = query.format(repo, base_branch)
r = utils._request(query)
if r.status_code == 200:
try:
new_config = yaml.load(r.text)
# overloading the default configuration with the one specified
config = utils.update_dict(config, new_config)
except yaml.YAMLError: # Bad YAML file
pass
# Create pycodestyle command line arguments
arguments = []
confs = config["pycodestyle"]
for key, value in confs.items():
if value: # Non empty
if isinstance(value, int):
if isinstance(value, bool):
arguments.append("--{}".format(key))
else:
arguments.append("--{}={}".format(key, value))
elif isinstance(value, list):
arguments.append("--{}={}".format(key, ','.join(value)))
config["pycodestyle_cmd_config"] = ' {arguments}'.format(
arguments=' '.join(arguments))
# pycodestyle is case-sensitive
config["pycodestyle"]["ignore"] = [
e.upper() for e in list(config["pycodestyle"]["ignore"])
]
return config