def test_add_to_tags_str(self):
T.assert_equal("A", add_to_tags_str("", "A"))
T.assert_equal("A", add_to_tags_str("A", "A"))
T.assert_equal("A,B", add_to_tags_str("A", "B"))
T.assert_equal("A,B,C", add_to_tags_str("A,B", "C"))
T.assert_equal("A,B,C", add_to_tags_str("A,B,C", "C"))
T.assert_equal("A,B,C", add_to_tags_str("A", "A,B,C"))
T.assert_equal("A,B", add_to_tags_str("A", "B"))
T.assert_equal("A,B,C", add_to_tags_str("B,A", "C"))
T.assert_equal("A,B,C", add_to_tags_str("A,C,B", "C"))
T.assert_equal("A,B,C", add_to_tags_str("A", "B,A,C"))
def update_request(cls, request_id):
req = cls._get_request(request_id)
if not req:
# Just log this and return. We won't be able to get more
# data out of the request.
error_msg = "Git queue worker received a job for non-existent request id %s" % request_id
logging.error(error_msg)
return
if cls.request_is_excluded_from_git_verification(req):
return
if not req['branch']:
error_msg = "Git queue worker received a job for request with no branch (id %s)" % request_id
return cls.update_request_failure(req, error_msg)
sha = cls._get_branch_sha_from_repo(req)
if sha is None:
error_msg = "Git queue worker could not get the revision from request branch (id %s)" % request_id
return cls.update_request_failure(req, error_msg)
duplicate_req = cls._get_request_with_sha(sha)
if duplicate_req and duplicate_req.has_key(
'state') and not duplicate_req['state'] == "discarded":
error_msg = "Git queue worker found another request with the same revision sha (ids %s and %s)" % (
duplicate_req['id'], request_id)
return cls.update_request_failure(req, error_msg)
updated_tags = add_to_tags_str(req['tags'], 'git-ok')
updated_tags = del_from_tags_str(updated_tags, 'git-error')
updated_values = {'revision': sha, 'tags': updated_tags}
updated_request = cls._update_request(req, updated_values)
if updated_request:
cls.update_request_successful(updated_request)
def _test_pickme_conflict_master(
cls, worker_id, req, target_branch,
repo_path, pushmanager_url, requeue):
"""Test whether the pickme given by req can be successfully merged onto
master.
If the pickme was merged successfully, it calls
_test_pickme_conflict_pickme to check the pickme against others in the
same push.
:param req: Details of pickme request to test
:param target_branch: The name of the test branch to use for testing
:param repo_path: The location of the repository we are working in
"""
# Ensure we have a copy of the pickme branch
cls.create_or_update_local_repo(
worker_id,
req['repo'],
branch=req['branch'],
fetch=True,
checkout=False
)
# Create a test branch following master
with git_branch_context_manager(target_branch, repo_path):
# Merge the pickme we are testing onto the test branch
# If this fails, that means pickme conflicts with master
try:
with git_merge_context_manager(target_branch, repo_path):
# Try to merge the pickme onto master
cls.git_merge_pickme(worker_id, req, repo_path)
# Check for conflicts with other pickmes
return cls._test_pickme_conflict_pickme(
worker_id,
req,
target_branch,
repo_path,
pushmanager_url,
requeue
)
except GitException, e:
updated_tags = add_to_tags_str(req['tags'], 'conflict-master')
updated_tags = del_from_tags_str(updated_tags, 'no-conflicts')
conflict_details = "<strong>Conflict with master:</strong><br/> %s <br/> %s" % (e.gitout, e.giterr)
updated_values = {
'tags': updated_tags,
'conflicts': conflict_details
}
updated_request = cls._update_request(req, updated_values)
if not updated_request:
raise Exception("Failed to update pickme")
else:
return True, updated_request
def convert_tag_callback(oldtag, newtag, success, db_results):
check_db_results(success, db_results)
requests = db_results[0].fetchall()
update_queries = []
for request in requests:
if tags_contain(request['tags'], [oldtag]):
updated_tags = del_from_tags_str(request['tags'], oldtag)
updated_tags = add_to_tags_str(updated_tags, newtag)
update_query = db.push_requests.update().where(
db.push_requests.c.id == request.id).values(
{'tags': updated_tags})
update_queries.append(update_query)
db.execute_transaction_cb(update_queries, check_db_results)
def convert_tag_callback(oldtag, newtag, success, db_results):
check_db_results(success, db_results)
requests = db_results[0].fetchall()
update_queries = []
for request in requests:
if tags_contain(request['tags'], [oldtag]):
updated_tags = del_from_tags_str(request['tags'], oldtag)
updated_tags = add_to_tags_str(updated_tags, newtag)
update_query = db.push_requests.update().where(
db.push_requests.c.id == request.id
).values({'tags': updated_tags})
update_queries.append(update_query)
db.execute_transaction_cb(update_queries, check_db_results)
def update_request_failure(cls, request, failure_msg):
logging.error(failure_msg)
updated_tags = add_to_tags_str(request['tags'], 'git-error')
updated_tags = del_from_tags_str(updated_tags, 'git-ok')
updated_values = {'tags': updated_tags}
cls._update_request(request, updated_values)
msg = ("""
<p>
<em>PushManager could <strong>not</strong> verify the branch for your request.</em>
</p>
<p>
<strong>%(user)s - %(title)s</strong><br />
<em>%(repo)s/%(branch)s</em><br />
<a href="https://%(pushmanager_servername)s/request?id=%(id)s">https://%(pushmanager_servername)s/request?id=%(id)s</a>
</p>
<p>
<strong>Error message</strong>:<br />
%(failure_msg)s
</p>
<p>
Review # (if specified): <a href="https://%(reviewboard_servername)s/r/%(reviewid)s">%(reviewid)s</a>
</p>
<p>
Verified revision: <code>%(revision)s</code><br/>
<em>(If this is <strong>not</strong> the revision you expected,
make sure you've pushed your latest version to the correct repo!)</em>
</p>
<p>
Regards,<br/>
PushManager
</p>
""")
request.update({
'failure_msg':
failure_msg,
'pushmanager_servername':
Settings['main_app']['servername'],
'reviewboard_servername':
Settings['reviewboard']['servername']
})
msg %= EscapedDict(request)
subject = '[push] %s - %s' % (request['user'], request['title'])
user_to_notify = request['user']
MailQueue.enqueue_user_email([user_to_notify], msg, subject)
def verify_branch_failure(cls, request, failure_msg, pushmanager_url):
logging.error(failure_msg)
updated_tags = add_to_tags_str(request['tags'], 'git-error')
updated_tags = del_from_tags_str(updated_tags, 'git-ok')
updated_values = {'tags': updated_tags}
cls._update_request(request, updated_values)
msg = (
"""
<p>
<em>PushManager could <strong>not</strong> verify the branch for your request.</em>
</p>
<p>
<strong>%(user)s - %(title)s</strong><br />
<em>%(repo)s/%(branch)s</em><br />
<a href="%(pushmanager_url)s/request?id=%(id)s">%(pushmanager_url)s/request?id=%(id)s</a>
</p>
<p>
<strong>Error message</strong>:<br />
%(failure_msg)s
</p>
<p>
Review # (if specified): <a href="https://%(reviewboard_servername)s/r/%(reviewid)s">%(reviewid)s</a>
</p>
<p>
Verified revision: <code>%(revision)s</code><br/>
<em>(If this is <strong>not</strong> the revision you expected,
make sure you've pushed your latest version to the correct repo!)</em>
</p>
<p>
Regards,<br/>
PushManager
</p>
"""
)
request.update({
'failure_msg': failure_msg,
'pushmanager_url' : pushmanager_url,
'reviewboard_servername': Settings['reviewboard']['servername']
})
msg %= EscapedDict(request)
subject = '[push] %s - %s' % (request['user'], request['title'])
user_to_notify = request['user']
MailQueue.enqueue_user_email([user_to_notify], msg, subject)
def verify_branch(cls, request_id, pushmanager_url):
req = cls._get_request(request_id)
if not req:
# Just log this and return. We won't be able to get more
# data out of the request.
error_msg = "Git queue worker received a job for non-existent request id %s" % request_id
logging.error(error_msg)
return
if cls.request_is_excluded_from_git_verification(req):
return
if not req['branch']:
error_msg = "Git queue worker received a job for request with no branch (id %s)" % request_id
return cls.verify_branch_failure(req, error_msg, pushmanager_url)
sha = cls._get_branch_sha_from_repo(req)
if sha is None:
error_msg = "Git queue worker could not get the revision from request branch (id %s)" % request_id
return cls.verify_branch_failure(req, error_msg, pushmanager_url)
duplicate_req = cls._get_request_with_sha(sha)
if (
duplicate_req and 'state' in duplicate_req
and not duplicate_req['state'] == "discarded"
and duplicate_req['id'] is not request_id
):
error_msg = "Git queue worker found another request with the same revision sha (ids %s and %s)" % (
duplicate_req['id'],
request_id
)
return cls.verify_branch_failure(req, error_msg, pushmanager_url)
updated_tags = add_to_tags_str(req['tags'], 'git-ok')
updated_tags = del_from_tags_str(updated_tags, 'git-error')
updated_values = {'revision': sha, 'tags': updated_tags}
updated_request = cls._update_request(req, updated_values)
if updated_request:
cls.verify_branch_successful(updated_request, pushmanager_url)
def test_pickme_conflicts(
cls,
worker_id,
request_id,
pushmanager_url,
requeue=True):
"""
Tests for conflicts between a pickme and both master and other pickmes
in the same push.
:param request_id: ID number of the pickme to be tested
:param requeue: Whether or not pickmes that this pickme conflicts with
should be added back into the GitQueue as a test conflict task.
"""
req = cls._get_request(request_id)
if not req:
logging.error(
"Tried to test conflicts for invalid request id %s",
request_id
)
return
if 'state' not in req or req['state'] not in ('pickme', 'added'):
return
push = cls._get_push_for_request(request_id)
if not push:
logging.error(
"Request %s (%s) doesn't seem to be part of a push",
request_id,
req['title']
)
return
push_id = push['push']
#### Set up the environment as though we are preparing a deploy push
## Create a branch pickme_test_PUSHID_PICKMEID
# Ensure that the local copy of master is up-to-date
cls.create_or_update_local_repo(
worker_id,
Settings['git']['main_repository'],
branch="master",
fetch=True
)
# Get base paths and names for the relevant repos
repo_path = cls._get_local_repository_uri(
Settings['git']['main_repository'],
worker_id
)
target_branch = "pickme_test_{push_id}_{pickme_id}".format(
push_id=push_id,
pickme_id=request_id
)
# Check that the branch is still reachable
sha = cls._get_branch_sha_from_repo(req)
if sha is None:
return
# Check if the pickme has already been merged into master
if cls._sha_exists_in_master(worker_id, sha):
return
# Clear the pickme's conflict info
cls._clear_pickme_conflict_details(req)
# Check for conflicts with master
conflict, updated_pickme = cls._test_pickme_conflict_master(
worker_id,
req,
target_branch,
repo_path,
pushmanager_url,
requeue
)
if conflict:
if updated_pickme is None:
raise Exception(
"Encountered merge conflict but was not passed details"
)
cls.pickme_conflict_detected(updated_pickme, requeue, pushmanager_url)
else:
# If the request does not conflict here or anywhere else, mark it as
# no-conflicts
req = cls._get_request(request_id)
if 'conflict' in req['tags']:
return
updated_tags = add_to_tags_str(req['tags'], 'no-conflicts')
updated_values = {
'tags': updated_tags,
}
updated_request = cls._update_request(req, updated_values)
if not updated_request:
raise Exception("Failed to update pickme")
# conflict. Pass on that it was requeued automatically and to
# NOT requeue things in that run, otherwise two tickets will
# requeue each other forever.
if requeue and pickme_details['state'] != 'added':
GitQueue.enqueue_request(
GitTaskAction.TEST_PICKME_CONFLICT,
pickme,
pushmanager_url=pushmanager_url,
requeue=False
)
# If there were no conflicts, don't update the request
if not conflict_pickmes:
return False, None
updated_tags = add_to_tags_str(req['tags'], 'conflict-pickme')
updated_tags = del_from_tags_str(updated_tags, 'no-conflicts')
formatted_conflicts = ""
for broken_pickme, git_out, git_err in conflict_pickmes:
pickme_details = cls._get_request(broken_pickme)
formatted_pickme_err = (
"""<strong>Conflict with <a href=\"/request?id={pickme_id}\">
{pickme_name}</a>: </strong><br/>{pickme_out}<br/>{pickme_err}
<br/><br/>"""
).format(
pickme_id=broken_pickme,
pickme_err=xhtml_escape(git_err),
pickme_out=xhtml_escape(git_out),
pickme_name=xhtml_escape(pickme_details['title'])
)
formatted_conflicts += formatted_pickme_err
