def test_transplant_using_at_bookmark(self):
inbound_repo_path = tempfile.mkdtemp()
local_repo_path = tempfile.mkdtemp()
mozreview_repo_path = tempfile.mkdtemp()
upstream_repo_path = tempfile.mkdtemp()
try:
# Configure 'upstream' repo
cmds = [['hg', 'init']]
for cmd in cmds:
subprocess.check_call(cmd, cwd=upstream_repo_path)
# Configure 'mozreview' repo
shutil.copy(os.path.join(HERE, 'test-data', 'initial.patch'),
mozreview_repo_path)
cmds = [['hg', 'init'],
['hg', 'import', 'initial.patch']]
for cmd in cmds:
# Use check_output to suppress the output from hg import
subprocess.check_output(cmd, stderr=subprocess.STDOUT,
cwd=mozreview_repo_path)
with open(os.path.join(mozreview_repo_path, '.hg', 'hgrc'), 'w') as f:
f.write('[phases]\npublish = False\n')
# Configure 'inbound' repo
cmds = [['hg', 'init']]
for cmd in cmds:
subprocess.check_call(cmd, cwd=inbound_repo_path)
# Configure 'local' repo
cmds = [['hg', 'init'],
['hg', 'bookmark', 'upstream']]
for cmd in cmds:
subprocess.check_call(cmd, cwd=local_repo_path)
with open(os.path.join(local_repo_path, '.hg', 'hgrc'), 'w') as f:
f.write('[paths]\nversion-control-tools = ')
f.write(inbound_repo_path)
f.write('\nmozreview = ')
f.write(mozreview_repo_path)
f.write('\nupstream= ')
f.write(mozreview_repo_path)
f.write('\n')
def get_repo_path(tree):
return local_repo_path
transplant.get_repo_path = get_repo_path
landed, result = transplant.transplant('mozreview',
'version-control-tools',
'0', '', '@')
self.assertEqual(landed, True)
self.assertEqual(len(result), 12)
self.assertIsNotNone(re.match('([a-f0-9]+)', result))
finally:
shutil.rmtree(local_repo_path)
shutil.rmtree(mozreview_repo_path)
shutil.rmtree(inbound_repo_path)
shutil.rmtree(upstream_repo_path)
def handle_pending_transplants(logger, dbconn):
cursor = dbconn.cursor()
query = """
select tree,revision,bugid from Autoland
where can_be_landed is true and landed is null
"""
cursor.execute(query)
landed = []
for row in cursor.fetchall():
tree, rev, bugid = row
pushlog = mercurial.get_pushlog(tree, rev)
if not pushlog:
logger.debug('could not get pushlog for tree: %s rev %s' %
(tree, rev))
return
changesets = []
for key in pushlog:
for changeset in pushlog[key]['changesets']:
# we assume by convention head revision is empty and should
# not be landed
if changeset != rev:
changesets.append(changeset)
# TODO: allow for transplant to other trees than 'mozilla-inbound'
result = transplant.transplant(tree, 'mozilla-inbound', changesets)
if not result:
logger.debug("""could not connect to transplant server:
tree: %s rev %s""" % (tree, rev))
continue
if 'error' in result:
succeeded = False
logger.info('transplant failed: tree: %s rev: %s error: %s' %
(tree, rev, json.dumps(result)))
comment = 'Autoland request failed: could not transplant: %s'
add_bugzilla_comment(dbconn, bugid, comment % result['error'])
else:
succeeded = True
comment = 'Autoland request succeeded: mozilla-inbound tip: %s'
add_bugzilla_comment(dbconn, bugid, comment % result['tip'])
landed.append([succeeded, json.dumps(result), datetime.datetime.now(),
tree, rev])
if landed:
query = """
update Autoland set landed=%s,transplant_result=%s,last_updated=%s
where tree=%s and revision=%s
"""
cursor.executemany(query, landed)
dbconn.commit()
def test_rewrite_descriptions(self):
inbound_repo_path = tempfile.mkdtemp()
local_repo_path = tempfile.mkdtemp()
mozreview_repo_path = tempfile.mkdtemp()
upstream_repo_path = tempfile.mkdtemp()
try:
# Configure 'upstream' repo
cmds = [['hg', 'init']]
for cmd in cmds:
subprocess.check_call(cmd, cwd=upstream_repo_path)
# Configure 'mozreview' repo
for f in ['initial.patch', 'added-blah.patch']:
shutil.copy(os.path.join(HERE, 'test-data', f),
mozreview_repo_path)
cmds = [['hg', 'init'],
['hg', 'import', 'initial.patch'],
['hg', 'import', 'added-blah.patch'],
['hg', 'log', '-r', '.', '--template', '{node|short}']]
for cmd in cmds:
# Use check_output to suppress the output from hg import
rev = subprocess.check_output(cmd, stderr=subprocess.STDOUT,
cwd=mozreview_repo_path)
with open(os.path.join(mozreview_repo_path, '.hg', 'hgrc'), 'w') as f:
f.write('[phases]\npublish = False\n')
# Configure 'inbound' repo
shutil.copy(os.path.join(HERE, 'test-data', 'initial.patch'),
inbound_repo_path)
cmds = [['hg', 'init'],
['hg', 'import', 'initial.patch']]
for cmd in cmds:
subprocess.check_call(cmd, cwd=inbound_repo_path)
# Configure 'local' repo
cmds = [['hg', 'init'],
['hg', 'bookmark', 'upstream']]
for cmd in cmds:
subprocess.check_call(cmd, cwd=local_repo_path)
with open(os.path.join(local_repo_path, '.hg', 'hgrc'), 'w') as f:
f.write('[paths]\ninbound = ')
f.write(inbound_repo_path)
f.write('\nmozreview = ')
f.write(mozreview_repo_path)
f.write('\nupstream= ')
f.write(upstream_repo_path)
f.write('\n')
ext = os.path.join(os.path.split(HERE)[0], 'hgext', 'rewritecommitdescriptions.py')
f.write('[extensions]\n')
f.write('rewritecommitdescriptions = %s\n\n' % ext)
def get_repo_path(tree):
return local_repo_path
transplant.get_repo_path = get_repo_path
commit_descriptions = {'1cf8d88f5d98': 'please rewrite me!'}
landed, result = transplant.transplant('mozreview', 'inbound',
'1cf8d88f5d98',
commit_descriptions=commit_descriptions)
self.assertEqual(landed, True)
self.assertEqual(len(result), 12)
self.assertIsNotNone(re.match('([a-f0-9]+)', result))
# Check that we rewrote as expected
cmds = [['hg', 'log', '-r', 'tip']]
for cmd in cmds:
result = subprocess.check_output(cmd, cwd=inbound_repo_path)
self.assertTrue('please rewrite me!' in result)
finally:
shutil.rmtree(inbound_repo_path)
shutil.rmtree(local_repo_path)
shutil.rmtree(mozreview_repo_path)
shutil.rmtree(upstream_repo_path)
def handle_pending_transplants(logger, dbconn):
cursor = dbconn.cursor()
now = datetime.datetime.now()
query = """
select id, destination, request
from Transplant
where landed is null and (last_updated is null
or last_updated<=%(time)s)
"""
transplant_retry_delay = TRANSPLANT_RETRY_DELAY
if config.testing():
transplant_retry_delay = datetime.timedelta(seconds=1)
cursor.execute(query, ({'time': now - transplant_retry_delay}))
current_treestatus = {}
finished_revisions = []
mozreview_updates = []
retry_revisions = []
def handle_treeclosed(transplant_id, tree, rev, destination, trysyntax,
pingback_url):
retry_revisions.append((now, transplant_id))
data = {
'request_id': transplant_id,
'tree': tree,
'rev': rev,
'destination': destination,
'trysyntax': trysyntax,
'landed': False,
'error_msg': '',
'result': 'Tree %s is closed - retrying later.' % tree
}
mozreview_updates.append([transplant_id, json.dumps(data)])
# This code is a bit messy because we have to deal with the fact that the
# the tree could close between the call to tree_is_open and when we
# actually attempt the revision.
#
# We keep a list of revisions to retry called retry_revisions which we
# append to whenever we detect a closed tree. These revisions have their
# last_updated field updated so we will retry them after a suitable delay.
#
# The other list we keep is for transplant attempts that either succeeded
# or failed due to a reason other than a closed tree, which is called
# finished_revisions. Successful or not, we're finished with them, they
# will not be retried.
for row in cursor.fetchall():
transplant_id, destination, request = row
requester = request['ldap_username']
tree = request['tree']
rev = request['rev']
trysyntax = request.get('trysyntax', '')
push_bookmark = request.get('push_bookmark', '')
pingback_url = request.get('pingback_url', '')
commit_descriptions = request.get('commit_descriptions')
tree_open = current_treestatus.setdefault(destination,
treestatus.tree_is_open(destination))
if not tree_open:
handle_treeclosed(transplant_id, tree, rev, destination,
trysyntax, pingback_url)
continue
attempts = 0
logger.info('initiating transplant from tree: %s rev: %s '
'to destination: %s' % (tree, rev, destination))
started = datetime.datetime.now()
while attempts < MAX_TRANSPLANT_ATTEMPTS:
# TODO: We should break the transplant call into two steps, one
# to pull down the commits to transplant, and another
# one to rebase it and attempt to push so we don't
# duplicate work unnecessarily if we have to rebase more
# than once.
os.environ['AUTOLAND_REQUEST_USER'] = requester
landed, result = transplant.transplant(logger, tree,
destination, rev,
trysyntax, push_bookmark,
commit_descriptions)
del os.environ['AUTOLAND_REQUEST_USER']
if landed or 'abort: push creates new remote head' not in result:
break
attempts += 1
if landed:
logger.info('transplant successful - new revision: %s' % result)
else:
if 'is CLOSED!' in result:
logger.info('transplant failed: tree: %s is closed - '
' retrying later.' % tree)
current_treestatus[destination] = False
handle_treeclosed(transplant_id, tree, rev, destination,
trysyntax, pingback_url)
continue
elif 'abort: push creates new remote head' in result:
logger.info('transplant failed: we lost a push race')
retry_revisions.append((now, transplant_id))
continue
elif 'unresolved conflicts (see hg resolve' in result:
logger.info('transplant failed - manual rebase required: '
'tree: %s rev: %s destination: %s error: %s' %
(tree, rev, destination, result))
# This is the only autoland error for which we expect the
# user to take action. We should make things nicer than the
# raw mercurial error.
# TODO: sad trombone sound
header = ('We\'re sorry, Autoland could not rebase your '
'commits for you automatically. Please manually '
'rebase your commits and try again.\n\n')
result = header + result
else:
logger.info('transplant failed: tree: %s rev: %s '
'destination: %s error: %s' %
(tree, rev, destination, result))
completed = datetime.datetime.now()
logger.info('elapsed transplant time: %s' % (completed - started))
# set up data to be posted back to mozreview
data = {
'request_id': transplant_id,
'tree': tree,
'rev': rev,
'destination': destination,
'trysyntax': trysyntax,
'landed': landed,
'error_msg': '',
'result': ''
}
if landed:
data['result'] = result
else:
data['error_msg'] = result
mozreview_updates.append([transplant_id, json.dumps(data)])
finished_revisions.append([landed, result, transplant_id])
if retry_revisions:
query = """
update Transplant set last_updated=%s
where id=%s
"""
cursor.executemany(query, retry_revisions)
dbconn.commit()
if finished_revisions:
query = """
update Transplant set landed=%s,result=%s
where id=%s
"""
cursor.executemany(query, finished_revisions)
dbconn.commit()
if mozreview_updates:
query = """
insert into MozreviewUpdate(transplant_id,data)
values(%s,%s)
"""
cursor.executemany(query, mozreview_updates)
dbconn.commit()
def handle_pending_transplants(logger, dbconn):
cursor = dbconn.cursor()
now = datetime.datetime.now()
query = """
SELECT id, destination, request
FROM Transplant
WHERE landed IS NULL
AND (last_updated IS NULL OR last_updated<=%(time)s)
ORDER BY created
"""
transplant_retry_delay = TRANSPLANT_RETRY_DELAY
if config.testing():
transplant_retry_delay = datetime.timedelta(seconds=1)
cursor.execute(query, ({'time': now - transplant_retry_delay}))
current_treestatus = {}
finished_revisions = []
mozreview_updates = []
retry_revisions = []
def handle_treeclosed(transplant_id, tree, rev, destination, trysyntax,
pingback_url):
retry_revisions.append((now, transplant_id))
data = {
'request_id': transplant_id,
'tree': tree,
'rev': rev,
'destination': destination,
'trysyntax': trysyntax,
'landed': False,
'error_msg': '',
'result': 'Tree %s is closed - retrying later.' % tree
}
mozreview_updates.append([transplant_id, json.dumps(data)])
# This code is a bit messy because we have to deal with the fact that the
# the tree could close between the call to tree_is_open and when we
# actually attempt the revision.
#
# We keep a list of revisions to retry called retry_revisions which we
# append to whenever we detect a closed tree. These revisions have their
# last_updated field updated so we will retry them after a suitable delay.
#
# The other list we keep is for transplant attempts that either succeeded
# or failed due to a reason other than a closed tree, which is called
# finished_revisions. Successful or not, we're finished with them, they
# will not be retried.
for row in cursor.fetchall():
transplant_id, destination, request = row
# Many of these values are used as command arguments. So convert
# to binary because command arguments aren't unicode.
destination = destination.encode('ascii')
requester = request['ldap_username']
tree = request['tree'].encode('ascii')
rev = request['rev'].encode('ascii')
trysyntax = request.get('trysyntax', '')
push_bookmark = request.get('push_bookmark', '').encode('ascii')
pingback_url = request.get('pingback_url', '').encode('ascii')
commit_descriptions = request.get('commit_descriptions')
tree_open = current_treestatus.setdefault(
destination, treestatus.tree_is_open(destination))
if not tree_open:
handle_treeclosed(transplant_id, tree, rev, destination,
trysyntax, pingback_url)
continue
attempts = 0
started = datetime.datetime.now()
while attempts < MAX_TRANSPLANT_ATTEMPTS:
logger.info('initiating transplant from tree: %s rev: %s '
'to destination: %s, attempt %s' % (
tree, rev, destination, attempts + 1))
# TODO: We should break the transplant call into two steps, one
# to pull down the commits to transplant, and another
# one to rebase it and attempt to push so we don't
# duplicate work unnecessarily if we have to rebase more
# than once.
os.environ['AUTOLAND_REQUEST_USER'] = requester
landed, result = transplant.transplant(logger, tree,
destination, rev,
trysyntax, push_bookmark,
commit_descriptions)
del os.environ['AUTOLAND_REQUEST_USER']
logger.info('transplant from tree: %s rev: %s attempt: %s: %s' % (
tree, rev, attempts + 1, result))
if landed or 'abort: push creates new remote head' not in result:
break
attempts += 1
if landed:
logger.info('transplant successful - new revision: %s' % result)
else:
if 'is CLOSED!' in result:
logger.info('transplant failed: tree: %s is closed - '
' retrying later.' % tree)
current_treestatus[destination] = False
handle_treeclosed(transplant_id, tree, rev, destination,
trysyntax, pingback_url)
continue
elif 'abort: push creates new remote head' in result:
logger.info('transplant failed: we lost a push race')
retry_revisions.append((now, transplant_id))
continue
elif 'unresolved conflicts (see hg resolve' in result:
logger.info('transplant failed - manual rebase required: '
'tree: %s rev: %s destination: %s error: %s' %
(tree, rev, destination, result))
# This is the only autoland error for which we expect the
# user to take action. We should make things nicer than the
# raw mercurial error.
# TODO: sad trombone sound
header = ('We\'re sorry, Autoland could not rebase your '
'commits for you automatically. Please manually '
'rebase your commits and try again.\n\n')
result = header + result
else:
logger.info('transplant failed: tree: %s rev: %s '
'destination: %s error: %s' %
(tree, rev, destination, result))
completed = datetime.datetime.now()
logger.info('elapsed transplant time: %s' % (completed - started))
# set up data to be posted back to mozreview
data = {
'request_id': transplant_id,
'tree': tree,
'rev': rev,
'destination': destination,
'trysyntax': trysyntax,
'landed': landed,
'error_msg': '',
'result': ''
}
if landed:
data['result'] = result
else:
data['error_msg'] = result
mozreview_updates.append([transplant_id, json.dumps(data)])
finished_revisions.append([landed, result, transplant_id])
if retry_revisions:
query = """
update Transplant set last_updated=%s
where id=%s
"""
cursor.executemany(query, retry_revisions)
dbconn.commit()
if finished_revisions:
query = """
update Transplant set landed=%s,result=%s
where id=%s
"""
cursor.executemany(query, finished_revisions)
dbconn.commit()
if mozreview_updates:
query = """
insert into MozreviewUpdate(transplant_id,data)
values(%s,%s)
"""
cursor.executemany(query, mozreview_updates)
dbconn.commit()
def handle_pending_transplants(logger, dbconn):
cursor = dbconn.cursor()
query = """
select id,tree,rev,destination,trysyntax,push_bookmark,pingback_url
from Transplant
where landed is null
"""
cursor.execute(query)
landed_revisions = []
mozreview_updates = []
for row in cursor.fetchall():
(transplant_id, tree, rev, destination, trysyntax, push_bookmark,
pingback_url) = row
if not treestatus.tree_is_open(destination):
continue
landed, result = transplant.transplant(tree, destination, rev,
trysyntax, push_bookmark)
if landed:
logger.info(('transplanted from tree: %s rev: %s'
' to destination: %s new revision: %s') %
(tree, rev, destination, result))
else:
if 'is CLOSED!' in result:
logger.info('transplant failed: tree: %s is closed - '
' retrying later.' % tree)
# continuing here will skip updating the autoland request
# so we will attempt to land it again later.
continue
else:
logger.info('transplant failed: tree: %s rev: %s '
'destination: %s error: %s' %
(tree, rev, destination, result))
# set up data to be posted back to mozreview
data = {
'request_id': transplant_id,
'tree': tree,
'rev': rev,
'destination': destination,
'trysyntax': trysyntax,
'landed': landed,
'error_msg': '',
'result': ''
}
if landed:
data['result'] = result
else:
data['error_msg'] = result
mozreview_updates.append([transplant_id, pingback_url, json.dumps(data)])
landed_revisions.append([landed, result, transplant_id])
if landed_revisions:
query = """
update Transplant set landed=%s,result=%s
where id=%s
"""
cursor.executemany(query, landed_revisions)
dbconn.commit()
if mozreview_updates:
query = """
insert into MozreviewUpdate(request_id,pingback_url,data)
values(%s,%s,%s)
"""
cursor.executemany(query, mozreview_updates)
dbconn.commit()