def test_no_exception(self):
foo = common.once_or_more("Foo", False, self.increment, 1)
eq_(1, self._counter)
eq_(1, foo)
foo = common.once_or_more("Foo", True, self.increment, 2)
eq_(3, self._counter)
eq_(3, foo)
def iter_svn_log_entries(svn_url, first_rev, last_rev, retry):
"""
Iterate over SVN log entries between first_rev and last_rev.
This function features chunked log fetching so that it isn't too nasty
to the SVN server if many entries are requested.
"""
cur_rev = first_rev
chunk_length = log_min_chunk_length
first_run = True
while last_rev == "HEAD" or cur_rev <= last_rev:
start_t = time.time()
stop_rev = min(last_rev, cur_rev + chunk_length)
ui.status("Fetching %s SVN log entries starting from revision %d...",
chunk_length,
cur_rev,
level=ui.VERBOSE)
entries = once_or_more("Fetching SVN log", retry, run_svn_log, svn_url,
cur_rev, "HEAD", chunk_length)
duration = time.time() - start_t
if not first_run:
# skip first revision on subsequent runs, as it is overlapped
entries.pop(0)
first_run = False
if not entries:
break
for e in entries:
if e['revision'] > last_rev:
break
yield e
if e['revision'] >= last_rev:
break
cur_rev = e['revision']
# Adapt chunk length based on measured request duration
if duration < log_duration_threshold:
chunk_length = int(chunk_length * 2.0)
elif duration > log_duration_threshold * 2:
chunk_length = max(log_min_chunk_length, int(chunk_length / 2.0))
def iter_svn_log_entries(svn_url, first_rev, last_rev, retry):
"""
Iterate over SVN log entries between first_rev and last_rev.
This function features chunked log fetching so that it isn't too nasty
to the SVN server if many entries are requested.
"""
cur_rev = first_rev
chunk_length = log_min_chunk_length
first_run = True
while last_rev == "HEAD" or cur_rev <= last_rev:
start_t = time.time()
stop_rev = min(last_rev, cur_rev + chunk_length)
ui.status("Fetching %s SVN log entries starting from revision %d...",
chunk_length, cur_rev, level=ui.VERBOSE)
entries = once_or_more("Fetching SVN log", retry, run_svn_log, svn_url,
cur_rev, "HEAD", chunk_length)
duration = time.time() - start_t
if not first_run:
# skip first revision on subsequent runs, as it is overlapped
entries.pop(0)
first_run = False
if not entries:
break
for e in entries:
if e['revision'] > last_rev:
break
yield e
if e['revision'] >= last_rev:
break
cur_rev = e['revision']
# Adapt chunk length based on measured request duration
if duration < log_duration_threshold:
chunk_length = int(chunk_length * 2.0)
elif duration > log_duration_threshold * 2:
chunk_length = max(log_min_chunk_length, int(chunk_length / 2.0))
def pull_svn_rev(log_entry, current_rev, svn_wc, wc_url, wc_base, retry):
"""
Pull SVN changes from the given log entry.
Returns the new SVN revision. If an exception occurs, it will rollback
to revision 'current_rev'.
"""
svn_rev = log_entry['revision']
added_paths = []
copied_paths = []
removed_paths = []
changed_paths = []
unrelated_paths = []
replaced_paths = {}
# 1. Prepare for the `svn up` changes that are pulled in the second step
# by analyzing log_entry for the changeset
for d in log_entry['changed_paths']:
# e.g. u'/branches/xmpp-subprotocols-2178-2/twisted/words/test/test_jabberxmlstream.py'
p = d['path']
if not p.startswith(wc_base + "/"):
# Ignore changed files that are not part of this subdir
if p != wc_base:
unrelated_paths.append(p)
continue
action = d['action']
if action not in 'MARD':
raise UnsupportedSVNAction(
"In SVN rev. %d: action '%s' not supported. Please report a bug!"
% (svn_rev, action))
# e.g. u'twisted/words/test/test_jabberxmlstream.py'
p = p[len(wc_base):].strip("/")
# Record for commit
changed_paths.append(p)
# Detect special cases
old_p = d['copyfrom_path']
if old_p and old_p.startswith(wc_base + "/"):
old_p = old_p[len(wc_base):].strip("/")
# Both paths can be identical if copied from an old rev.
# We treat like it a normal change.
if old_p != p:
# Try to hint hg about file and dir copies
if not os.path.isdir(old_p):
copied_paths.append((old_p, p))
if action == 'R':
removed_paths.append(old_p)
else:
# Extract actual copied files (hg doesn't track dirs
# and will refuse "hg copy -A" with dirs)
r = run_hg(["st", "-nc"], [old_p],
output_is_locale_encoding=True)
for old_f in r.splitlines():
f = p + old_f[len(old_p):]
copied_paths.append((old_f, f))
if action == 'R':
removed_paths.append(old_f)
continue
if d['action'] == 'A':
added_paths.append(p)
elif d['action'] == 'D':
# Same as above: unfold directories into their affected files
if not os.path.isdir(p):
removed_paths.append(p)
else:
r = run_hg(["st", "-nc"], [p], output_is_locale_encoding=True)
for f in r.splitlines():
removed_paths.append(f)
elif d['action'] == 'R':
# (R)eplaced directories can have added and removed files without
# them being mentioned in the SVN log => we must detect those files
# ourselves.
# (http://svn.python.org/projects/python/branches/py3k, rev 59625)
if os.path.isdir(p):
replaced_paths[p] = get_svn_versioned_files(
os.path.join(svn_wc, p))
else:
# We never know what twisty semantics (R) can have. addremove
# is safest.
added_paths.append(p)
# 2. Update SVN + add/remove/commit hg
try:
if changed_paths:
args = ["up", "--ignore-externals"]
if get_svn_client_version() >= (1, 5):
args.extend(['--accept', 'postpone'])
ui.status('Attempting to update to revision %s...', str(svn_rev))
once_or_more("SVN update", retry, run_svn,
args + ["-r", svn_rev, svn_wc])
conflicts = []
for status_entry in get_svn_status('.'):
if status_entry['status'] == 'conflicted':
conflicts.append(status_entry['path'])
if conflicts:
ui.status('SVN updated resulted in conflicts!', level=ui.ERROR)
ui.status('Conflicted files: %s', ','.join(conflicts))
ui.status('Please report a bug.')
return None
for p, old_contents in replaced_paths.items():
old_contents = set(old_contents)
new_contents = set(
get_svn_versioned_files(os.path.join(svn_wc, p)))
added_paths.extend(p + '/' + f
for f in new_contents - old_contents)
removed_paths.extend(p + '/' + f
for f in old_contents - new_contents)
if added_paths:
# Use 'addremove' because an added SVN directory may
# overwrite a previous directory with the same name.
# XXX what about untracked files in those directories?
run_hg(["addremove"] + hg_exclude_options, added_paths)
for old, new in copied_paths:
try:
run_hg(["copy", "-A"], [old, new])
except ExternalCommandFailed:
# Maybe the "old" path is obsolete, i.e. it comes from an
# old SVN revision and was later removed.
s = run_hg(['st', '-nd'], [old],
output_is_locale_encoding=True)
if s.strip():
# The old path is known by hg, something else happened.
raise
run_hg(["add"], [new])
if removed_paths:
try:
for file_path in removed_paths:
if os.path.exists(file_path):
run_hg(["remove", "-A"], file_path)
except (ExternalCommandFailed), e:
if str(e).find("file is untracked") > 0:
ui.status(
"Ignoring warnings about untracked files: '%s'" %
str(e),
level=ui.VERBOSE)
else:
raise
hg_commit_from_svn_log_entry(log_entry)
elif unrelated_paths:
detect_overwritten_svn_branch(wc_url, svn_rev)
def pull_svn_rev(log_entry, current_rev, svn_wc, wc_url, wc_base, retry):
"""
Pull SVN changes from the given log entry.
Returns the new SVN revision. If an exception occurs, it will rollback
to revision 'current_rev'.
"""
svn_rev = log_entry['revision']
added_paths = []
copied_paths = []
removed_paths = []
changed_paths = []
unrelated_paths = []
replaced_paths = {}
# 1. Prepare for the `svn up` changes that are pulled in the second step
# by analyzing log_entry for the changeset
for d in log_entry['changed_paths']:
# e.g. u'/branches/xmpp-subprotocols-2178-2/twisted/words/test/test_jabberxmlstream.py'
p = d['path']
if not p.startswith(wc_base + "/"):
# Ignore changed files that are not part of this subdir
if p != wc_base:
unrelated_paths.append(p)
continue
action = d['action']
if action not in 'MARD':
raise UnsupportedSVNAction("In SVN rev. %d: action '%s' not supported. Please report a bug!"
% (svn_rev, action))
# e.g. u'twisted/words/test/test_jabberxmlstream.py'
p = p[len(wc_base):].strip("/")
# Record for commit
changed_paths.append(p)
# Detect special cases
old_p = d['copyfrom_path']
if old_p and old_p.startswith(wc_base + "/"):
old_p = old_p[len(wc_base):].strip("/")
# Both paths can be identical if copied from an old rev.
# We treat like it a normal change.
if old_p != p:
# Try to hint hg about file and dir copies
if not os.path.isdir(old_p):
copied_paths.append((old_p, p))
if action == 'R':
removed_paths.append(old_p)
else:
# Extract actual copied files (hg doesn't track dirs
# and will refuse "hg copy -A" with dirs)
r = run_hg(["st", "-nc"], [old_p], output_is_locale_encoding=True)
for old_f in r.splitlines():
f = p + old_f[len(old_p):]
copied_paths.append((old_f, f))
if action == 'R':
removed_paths.append(old_f)
continue
if d['action'] == 'A':
added_paths.append(p)
elif d['action'] == 'D':
# Same as above: unfold directories into their affected files
if not os.path.isdir(p):
removed_paths.append(p)
else:
r = run_hg(["st", "-nc"], [p], output_is_locale_encoding=True)
for f in r.splitlines():
removed_paths.append(f)
elif d['action'] == 'R':
# (R)eplaced directories can have added and removed files without
# them being mentioned in the SVN log => we must detect those files
# ourselves.
# (http://svn.python.org/projects/python/branches/py3k, rev 59625)
if os.path.isdir(p):
replaced_paths[p] = get_svn_versioned_files(
os.path.join(svn_wc, p))
else:
# We never know what twisty semantics (R) can have. addremove
# is safest.
added_paths.append(p)
# 2. Update SVN + add/remove/commit hg
try:
if changed_paths:
args = ["up", "--ignore-externals"]
if get_svn_client_version() >= (1, 5):
args.extend(['--accept', 'postpone'])
ui.status('Attempting to update to revision %s...', str(svn_rev))
once_or_more("SVN update", retry, run_svn, args + ["-r", svn_rev, svn_wc])
conflicts = []
for status_entry in get_svn_status('.'):
if status_entry['status'] == 'conflicted':
conflicts.append(status_entry['path'])
if conflicts:
ui.status('SVN updated resulted in conflicts!', level=ui.ERROR)
ui.status('Conflicted files: %s', ','.join(conflicts))
ui.status('Please report a bug.')
return None
for p, old_contents in replaced_paths.items():
old_contents = set(old_contents)
new_contents = set(get_svn_versioned_files(
os.path.join(svn_wc, p)))
added_paths.extend(p + '/' + f for f in new_contents - old_contents)
removed_paths.extend(p + '/' + f for f in old_contents - new_contents)
if added_paths:
# Use 'addremove' because an added SVN directory may
# overwrite a previous directory with the same name.
# XXX what about untracked files in those directories?
run_hg(["addremove"] + hg_exclude_options, added_paths)
for old, new in copied_paths:
try:
run_hg(["copy", "-A"], [old, new])
except ExternalCommandFailed:
# Maybe the "old" path is obsolete, i.e. it comes from an
# old SVN revision and was later removed.
s = run_hg(['st', '-nd'], [old], output_is_locale_encoding=True)
if s.strip():
# The old path is known by hg, something else happened.
raise
run_hg(["add"], [new])
if removed_paths:
try:
for file_path in removed_paths:
if os.path.exists(file_path):
run_hg(["remove", "-A"], file_path)
except (ExternalCommandFailed), e:
if str(e).find("file is untracked") > 0:
ui.status("Ignoring warnings about untracked files: '%s'" % str(e), level=ui.VERBOSE)
else:
raise
hg_commit_from_svn_log_entry(log_entry)
elif unrelated_paths:
detect_overwritten_svn_branch(wc_url, svn_rev)
def test_with_exception_retry(self):
eq_(0, self._counter)
common.once_or_more("Foo", True, self.increment_until_3, 1)
eq_(3, self._counter)