def test_add_to_tags_str(self): T.assert_equal("A", add_to_tags_str("", "A")) T.assert_equal("A", add_to_tags_str("A", "A")) T.assert_equal("A,B", add_to_tags_str("A", "B")) T.assert_equal("A,B,C", add_to_tags_str("A,B", "C")) T.assert_equal("A,B,C", add_to_tags_str("A,B,C", "C")) T.assert_equal("A,B,C", add_to_tags_str("A", "A,B,C")) T.assert_equal("A,B", add_to_tags_str("A", "B")) T.assert_equal("A,B,C", add_to_tags_str("B,A", "C")) T.assert_equal("A,B,C", add_to_tags_str("A,C,B", "C")) T.assert_equal("A,B,C", add_to_tags_str("A", "B,A,C"))
def update_request(cls, request_id): req = cls._get_request(request_id) if not req: # Just log this and return. We won't be able to get more # data out of the request. error_msg = "Git queue worker received a job for non-existent request id %s" % request_id logging.error(error_msg) return if cls.request_is_excluded_from_git_verification(req): return if not req['branch']: error_msg = "Git queue worker received a job for request with no branch (id %s)" % request_id return cls.update_request_failure(req, error_msg) sha = cls._get_branch_sha_from_repo(req) if sha is None: error_msg = "Git queue worker could not get the revision from request branch (id %s)" % request_id return cls.update_request_failure(req, error_msg) duplicate_req = cls._get_request_with_sha(sha) if duplicate_req and duplicate_req.has_key( 'state') and not duplicate_req['state'] == "discarded": error_msg = "Git queue worker found another request with the same revision sha (ids %s and %s)" % ( duplicate_req['id'], request_id) return cls.update_request_failure(req, error_msg) updated_tags = add_to_tags_str(req['tags'], 'git-ok') updated_tags = del_from_tags_str(updated_tags, 'git-error') updated_values = {'revision': sha, 'tags': updated_tags} updated_request = cls._update_request(req, updated_values) if updated_request: cls.update_request_successful(updated_request)
def _test_pickme_conflict_master( cls, worker_id, req, target_branch, repo_path, pushmanager_url, requeue): """Test whether the pickme given by req can be successfully merged onto master. If the pickme was merged successfully, it calls _test_pickme_conflict_pickme to check the pickme against others in the same push. :param req: Details of pickme request to test :param target_branch: The name of the test branch to use for testing :param repo_path: The location of the repository we are working in """ # Ensure we have a copy of the pickme branch cls.create_or_update_local_repo( worker_id, req['repo'], branch=req['branch'], fetch=True, checkout=False ) # Create a test branch following master with git_branch_context_manager(target_branch, repo_path): # Merge the pickme we are testing onto the test branch # If this fails, that means pickme conflicts with master try: with git_merge_context_manager(target_branch, repo_path): # Try to merge the pickme onto master cls.git_merge_pickme(worker_id, req, repo_path) # Check for conflicts with other pickmes return cls._test_pickme_conflict_pickme( worker_id, req, target_branch, repo_path, pushmanager_url, requeue ) except GitException, e: updated_tags = add_to_tags_str(req['tags'], 'conflict-master') updated_tags = del_from_tags_str(updated_tags, 'no-conflicts') conflict_details = "<strong>Conflict with master:</strong><br/> %s <br/> %s" % (e.gitout, e.giterr) updated_values = { 'tags': updated_tags, 'conflicts': conflict_details } updated_request = cls._update_request(req, updated_values) if not updated_request: raise Exception("Failed to update pickme") else: return True, updated_request
def convert_tag_callback(oldtag, newtag, success, db_results): check_db_results(success, db_results) requests = db_results[0].fetchall() update_queries = [] for request in requests: if tags_contain(request['tags'], [oldtag]): updated_tags = del_from_tags_str(request['tags'], oldtag) updated_tags = add_to_tags_str(updated_tags, newtag) update_query = db.push_requests.update().where( db.push_requests.c.id == request.id).values( {'tags': updated_tags}) update_queries.append(update_query) db.execute_transaction_cb(update_queries, check_db_results)
def convert_tag_callback(oldtag, newtag, success, db_results): check_db_results(success, db_results) requests = db_results[0].fetchall() update_queries = [] for request in requests: if tags_contain(request['tags'], [oldtag]): updated_tags = del_from_tags_str(request['tags'], oldtag) updated_tags = add_to_tags_str(updated_tags, newtag) update_query = db.push_requests.update().where( db.push_requests.c.id == request.id ).values({'tags': updated_tags}) update_queries.append(update_query) db.execute_transaction_cb(update_queries, check_db_results)
def update_request_failure(cls, request, failure_msg): logging.error(failure_msg) updated_tags = add_to_tags_str(request['tags'], 'git-error') updated_tags = del_from_tags_str(updated_tags, 'git-ok') updated_values = {'tags': updated_tags} cls._update_request(request, updated_values) msg = (""" <p> <em>PushManager could <strong>not</strong> verify the branch for your request.</em> </p> <p> <strong>%(user)s - %(title)s</strong><br /> <em>%(repo)s/%(branch)s</em><br /> <a href="https://%(pushmanager_servername)s/request?id=%(id)s">https://%(pushmanager_servername)s/request?id=%(id)s</a> </p> <p> <strong>Error message</strong>:<br /> %(failure_msg)s </p> <p> Review # (if specified): <a href="https://%(reviewboard_servername)s/r/%(reviewid)s">%(reviewid)s</a> </p> <p> Verified revision: <code>%(revision)s</code><br/> <em>(If this is <strong>not</strong> the revision you expected, make sure you've pushed your latest version to the correct repo!)</em> </p> <p> Regards,<br/> PushManager </p> """) request.update({ 'failure_msg': failure_msg, 'pushmanager_servername': Settings['main_app']['servername'], 'reviewboard_servername': Settings['reviewboard']['servername'] }) msg %= EscapedDict(request) subject = '[push] %s - %s' % (request['user'], request['title']) user_to_notify = request['user'] MailQueue.enqueue_user_email([user_to_notify], msg, subject)
def verify_branch_failure(cls, request, failure_msg, pushmanager_url): logging.error(failure_msg) updated_tags = add_to_tags_str(request['tags'], 'git-error') updated_tags = del_from_tags_str(updated_tags, 'git-ok') updated_values = {'tags': updated_tags} cls._update_request(request, updated_values) msg = ( """ <p> <em>PushManager could <strong>not</strong> verify the branch for your request.</em> </p> <p> <strong>%(user)s - %(title)s</strong><br /> <em>%(repo)s/%(branch)s</em><br /> <a href="%(pushmanager_url)s/request?id=%(id)s">%(pushmanager_url)s/request?id=%(id)s</a> </p> <p> <strong>Error message</strong>:<br /> %(failure_msg)s </p> <p> Review # (if specified): <a href="https://%(reviewboard_servername)s/r/%(reviewid)s">%(reviewid)s</a> </p> <p> Verified revision: <code>%(revision)s</code><br/> <em>(If this is <strong>not</strong> the revision you expected, make sure you've pushed your latest version to the correct repo!)</em> </p> <p> Regards,<br/> PushManager </p> """ ) request.update({ 'failure_msg': failure_msg, 'pushmanager_url' : pushmanager_url, 'reviewboard_servername': Settings['reviewboard']['servername'] }) msg %= EscapedDict(request) subject = '[push] %s - %s' % (request['user'], request['title']) user_to_notify = request['user'] MailQueue.enqueue_user_email([user_to_notify], msg, subject)
def verify_branch(cls, request_id, pushmanager_url): req = cls._get_request(request_id) if not req: # Just log this and return. We won't be able to get more # data out of the request. error_msg = "Git queue worker received a job for non-existent request id %s" % request_id logging.error(error_msg) return if cls.request_is_excluded_from_git_verification(req): return if not req['branch']: error_msg = "Git queue worker received a job for request with no branch (id %s)" % request_id return cls.verify_branch_failure(req, error_msg, pushmanager_url) sha = cls._get_branch_sha_from_repo(req) if sha is None: error_msg = "Git queue worker could not get the revision from request branch (id %s)" % request_id return cls.verify_branch_failure(req, error_msg, pushmanager_url) duplicate_req = cls._get_request_with_sha(sha) if ( duplicate_req and 'state' in duplicate_req and not duplicate_req['state'] == "discarded" and duplicate_req['id'] is not request_id ): error_msg = "Git queue worker found another request with the same revision sha (ids %s and %s)" % ( duplicate_req['id'], request_id ) return cls.verify_branch_failure(req, error_msg, pushmanager_url) updated_tags = add_to_tags_str(req['tags'], 'git-ok') updated_tags = del_from_tags_str(updated_tags, 'git-error') updated_values = {'revision': sha, 'tags': updated_tags} updated_request = cls._update_request(req, updated_values) if updated_request: cls.verify_branch_successful(updated_request, pushmanager_url)
def test_pickme_conflicts( cls, worker_id, request_id, pushmanager_url, requeue=True): """ Tests for conflicts between a pickme and both master and other pickmes in the same push. :param request_id: ID number of the pickme to be tested :param requeue: Whether or not pickmes that this pickme conflicts with should be added back into the GitQueue as a test conflict task. """ req = cls._get_request(request_id) if not req: logging.error( "Tried to test conflicts for invalid request id %s", request_id ) return if 'state' not in req or req['state'] not in ('pickme', 'added'): return push = cls._get_push_for_request(request_id) if not push: logging.error( "Request %s (%s) doesn't seem to be part of a push", request_id, req['title'] ) return push_id = push['push'] #### Set up the environment as though we are preparing a deploy push ## Create a branch pickme_test_PUSHID_PICKMEID # Ensure that the local copy of master is up-to-date cls.create_or_update_local_repo( worker_id, Settings['git']['main_repository'], branch="master", fetch=True ) # Get base paths and names for the relevant repos repo_path = cls._get_local_repository_uri( Settings['git']['main_repository'], worker_id ) target_branch = "pickme_test_{push_id}_{pickme_id}".format( push_id=push_id, pickme_id=request_id ) # Check that the branch is still reachable sha = cls._get_branch_sha_from_repo(req) if sha is None: return # Check if the pickme has already been merged into master if cls._sha_exists_in_master(worker_id, sha): return # Clear the pickme's conflict info cls._clear_pickme_conflict_details(req) # Check for conflicts with master conflict, updated_pickme = cls._test_pickme_conflict_master( worker_id, req, target_branch, repo_path, pushmanager_url, requeue ) if conflict: if updated_pickme is None: raise Exception( "Encountered merge conflict but was not passed details" ) cls.pickme_conflict_detected(updated_pickme, requeue, pushmanager_url) else: # If the request does not conflict here or anywhere else, mark it as # no-conflicts req = cls._get_request(request_id) if 'conflict' in req['tags']: return updated_tags = add_to_tags_str(req['tags'], 'no-conflicts') updated_values = { 'tags': updated_tags, } updated_request = cls._update_request(req, updated_values) if not updated_request: raise Exception("Failed to update pickme")
# conflict. Pass on that it was requeued automatically and to # NOT requeue things in that run, otherwise two tickets will # requeue each other forever. if requeue and pickme_details['state'] != 'added': GitQueue.enqueue_request( GitTaskAction.TEST_PICKME_CONFLICT, pickme, pushmanager_url=pushmanager_url, requeue=False ) # If there were no conflicts, don't update the request if not conflict_pickmes: return False, None updated_tags = add_to_tags_str(req['tags'], 'conflict-pickme') updated_tags = del_from_tags_str(updated_tags, 'no-conflicts') formatted_conflicts = "" for broken_pickme, git_out, git_err in conflict_pickmes: pickme_details = cls._get_request(broken_pickme) formatted_pickme_err = ( """<strong>Conflict with <a href=\"/request?id={pickme_id}\"> {pickme_name}</a>: </strong><br/>{pickme_out}<br/>{pickme_err} <br/><br/>""" ).format( pickme_id=broken_pickme, pickme_err=xhtml_escape(git_err), pickme_out=xhtml_escape(git_out), pickme_name=xhtml_escape(pickme_details['title']) ) formatted_conflicts += formatted_pickme_err