def pull_svn_rev(log_entry, current_rev, svn_wc, wc_url, wc_base, retry):
"""
Pull SVN changes from the given log entry.
Returns the new SVN revision. If an exception occurs, it will rollback
to revision 'current_rev'.
"""
svn_rev = log_entry['revision']
added_paths = []
copied_paths = []
removed_paths = []
changed_paths = []
unrelated_paths = []
replaced_paths = {}
# 1. Prepare for the `svn up` changes that are pulled in the second step
# by analyzing log_entry for the changeset
for d in log_entry['changed_paths']:
# e.g. u'/branches/xmpp-subprotocols-2178-2/twisted/words/test/test_jabberxmlstream.py'
p = d['path']
if not p.startswith(wc_base + "/"):
# Ignore changed files that are not part of this subdir
if p != wc_base:
unrelated_paths.append(p)
continue
action = d['action']
if action not in 'MARD':
raise UnsupportedSVNAction(
"In SVN rev. %d: action '%s' not supported. Please report a bug!"
% (svn_rev, action))
# e.g. u'twisted/words/test/test_jabberxmlstream.py'
p = p[len(wc_base):].strip("/")
# Record for commit
changed_paths.append(p)
# Detect special cases
old_p = d['copyfrom_path']
if old_p and old_p.startswith(wc_base + "/"):
old_p = old_p[len(wc_base):].strip("/")
# Both paths can be identical if copied from an old rev.
# We treat like it a normal change.
if old_p != p:
# Try to hint hg about file and dir copies
if not os.path.isdir(old_p):
copied_paths.append((old_p, p))
if action == 'R':
removed_paths.append(old_p)
else:
# Extract actual copied files (hg doesn't track dirs
# and will refuse "hg copy -A" with dirs)
r = run_hg(["st", "-nc"], [old_p],
output_is_locale_encoding=True)
for old_f in r.splitlines():
f = p + old_f[len(old_p):]
copied_paths.append((old_f, f))
if action == 'R':
removed_paths.append(old_f)
continue
if d['action'] == 'A':
added_paths.append(p)
elif d['action'] == 'D':
# Same as above: unfold directories into their affected files
if not os.path.isdir(p):
removed_paths.append(p)
else:
r = run_hg(["st", "-nc"], [p], output_is_locale_encoding=True)
for f in r.splitlines():
removed_paths.append(f)
elif d['action'] == 'R':
# (R)eplaced directories can have added and removed files without
# them being mentioned in the SVN log => we must detect those files
# ourselves.
# (http://svn.python.org/projects/python/branches/py3k, rev 59625)
if os.path.isdir(p):
replaced_paths[p] = get_svn_versioned_files(
os.path.join(svn_wc, p))
else:
# We never know what twisty semantics (R) can have. addremove
# is safest.
added_paths.append(p)
# 2. Update SVN + add/remove/commit hg
try:
if changed_paths:
args = ["up", "--ignore-externals"]
if get_svn_client_version() >= (1, 5):
args.extend(['--accept', 'postpone'])
ui.status('Attempting to update to revision %s...', str(svn_rev))
once_or_more("SVN update", retry, run_svn,
args + ["-r", svn_rev, svn_wc])
conflicts = []
for status_entry in get_svn_status('.'):
if status_entry['status'] == 'conflicted':
conflicts.append(status_entry['path'])
if conflicts:
ui.status('SVN updated resulted in conflicts!', level=ui.ERROR)
ui.status('Conflicted files: %s', ','.join(conflicts))
ui.status('Please report a bug.')
return None
for p, old_contents in replaced_paths.items():
old_contents = set(old_contents)
new_contents = set(
get_svn_versioned_files(os.path.join(svn_wc, p)))
added_paths.extend(p + '/' + f
for f in new_contents - old_contents)
removed_paths.extend(p + '/' + f
for f in old_contents - new_contents)
if added_paths:
# Use 'addremove' because an added SVN directory may
# overwrite a previous directory with the same name.
# XXX what about untracked files in those directories?
run_hg(["addremove"] + hg_exclude_options, added_paths)
for old, new in copied_paths:
try:
run_hg(["copy", "-A"], [old, new])
except ExternalCommandFailed:
# Maybe the "old" path is obsolete, i.e. it comes from an
# old SVN revision and was later removed.
s = run_hg(['st', '-nd'], [old],
output_is_locale_encoding=True)
if s.strip():
# The old path is known by hg, something else happened.
raise
run_hg(["add"], [new])
if removed_paths:
try:
for file_path in removed_paths:
if os.path.exists(file_path):
run_hg(["remove", "-A"], file_path)
except (ExternalCommandFailed), e:
if str(e).find("file is untracked") > 0:
ui.status(
"Ignoring warnings about untracked files: '%s'" %
str(e),
level=ui.VERBOSE)
else:
raise
hg_commit_from_svn_log_entry(log_entry)
elif unrelated_paths:
detect_overwritten_svn_branch(wc_url, svn_rev)
def pull_svn_rev(log_entry, current_rev, svn_wc, wc_url, wc_base, retry):
"""
Pull SVN changes from the given log entry.
Returns the new SVN revision. If an exception occurs, it will rollback
to revision 'current_rev'.
"""
svn_rev = log_entry['revision']
added_paths = []
copied_paths = []
removed_paths = []
changed_paths = []
unrelated_paths = []
replaced_paths = {}
# 1. Prepare for the `svn up` changes that are pulled in the second step
# by analyzing log_entry for the changeset
for d in log_entry['changed_paths']:
# e.g. u'/branches/xmpp-subprotocols-2178-2/twisted/words/test/test_jabberxmlstream.py'
p = d['path']
if not p.startswith(wc_base + "/"):
# Ignore changed files that are not part of this subdir
if p != wc_base:
unrelated_paths.append(p)
continue
action = d['action']
if action not in 'MARD':
raise UnsupportedSVNAction("In SVN rev. %d: action '%s' not supported. Please report a bug!"
% (svn_rev, action))
# e.g. u'twisted/words/test/test_jabberxmlstream.py'
p = p[len(wc_base):].strip("/")
# Record for commit
changed_paths.append(p)
# Detect special cases
old_p = d['copyfrom_path']
if old_p and old_p.startswith(wc_base + "/"):
old_p = old_p[len(wc_base):].strip("/")
# Both paths can be identical if copied from an old rev.
# We treat like it a normal change.
if old_p != p:
# Try to hint hg about file and dir copies
if not os.path.isdir(old_p):
copied_paths.append((old_p, p))
if action == 'R':
removed_paths.append(old_p)
else:
# Extract actual copied files (hg doesn't track dirs
# and will refuse "hg copy -A" with dirs)
r = run_hg(["st", "-nc"], [old_p], output_is_locale_encoding=True)
for old_f in r.splitlines():
f = p + old_f[len(old_p):]
copied_paths.append((old_f, f))
if action == 'R':
removed_paths.append(old_f)
continue
if d['action'] == 'A':
added_paths.append(p)
elif d['action'] == 'D':
# Same as above: unfold directories into their affected files
if not os.path.isdir(p):
removed_paths.append(p)
else:
r = run_hg(["st", "-nc"], [p], output_is_locale_encoding=True)
for f in r.splitlines():
removed_paths.append(f)
elif d['action'] == 'R':
# (R)eplaced directories can have added and removed files without
# them being mentioned in the SVN log => we must detect those files
# ourselves.
# (http://svn.python.org/projects/python/branches/py3k, rev 59625)
if os.path.isdir(p):
replaced_paths[p] = get_svn_versioned_files(
os.path.join(svn_wc, p))
else:
# We never know what twisty semantics (R) can have. addremove
# is safest.
added_paths.append(p)
# 2. Update SVN + add/remove/commit hg
try:
if changed_paths:
args = ["up", "--ignore-externals"]
if get_svn_client_version() >= (1, 5):
args.extend(['--accept', 'postpone'])
ui.status('Attempting to update to revision %s...', str(svn_rev))
once_or_more("SVN update", retry, run_svn, args + ["-r", svn_rev, svn_wc])
conflicts = []
for status_entry in get_svn_status('.'):
if status_entry['status'] == 'conflicted':
conflicts.append(status_entry['path'])
if conflicts:
ui.status('SVN updated resulted in conflicts!', level=ui.ERROR)
ui.status('Conflicted files: %s', ','.join(conflicts))
ui.status('Please report a bug.')
return None
for p, old_contents in replaced_paths.items():
old_contents = set(old_contents)
new_contents = set(get_svn_versioned_files(
os.path.join(svn_wc, p)))
added_paths.extend(p + '/' + f for f in new_contents - old_contents)
removed_paths.extend(p + '/' + f for f in old_contents - new_contents)
if added_paths:
# Use 'addremove' because an added SVN directory may
# overwrite a previous directory with the same name.
# XXX what about untracked files in those directories?
run_hg(["addremove"] + hg_exclude_options, added_paths)
for old, new in copied_paths:
try:
run_hg(["copy", "-A"], [old, new])
except ExternalCommandFailed:
# Maybe the "old" path is obsolete, i.e. it comes from an
# old SVN revision and was later removed.
s = run_hg(['st', '-nd'], [old], output_is_locale_encoding=True)
if s.strip():
# The old path is known by hg, something else happened.
raise
run_hg(["add"], [new])
if removed_paths:
try:
for file_path in removed_paths:
if os.path.exists(file_path):
run_hg(["remove", "-A"], file_path)
except (ExternalCommandFailed), e:
if str(e).find("file is untracked") > 0:
ui.status("Ignoring warnings about untracked files: '%s'" % str(e), level=ui.VERBOSE)
else:
raise
hg_commit_from_svn_log_entry(log_entry)
elif unrelated_paths:
detect_overwritten_svn_branch(wc_url, svn_rev)
def cleanup_svn_unversioned(files):
svn_status = get_svn_status(".")
for entry in svn_status:
if (entry['type'] == 'unversioned') and entry['path'] in files:
files.remove(entry['path'])
return files
def main():
usage = """1. %prog [-r SVN rev] [-p SVN peg rev] <SVN URL> [local checkout]
2. %prog --local-only <local checkout>"""
parser = OptionParser(usage)
parser.add_option("-r", "--svn-rev", type="int", dest="svn_rev",
help="SVN revision to checkout from")
parser.add_option("-p", "--svn-peg", type="int", dest="svn_peg",
help="SVN peg revision to locate checkout URL")
parser.add_option("--no-hgignore", dest="hgignore", action="store_false",
default=True, help="Don't autogenerate .hgignore")
parser.add_option("--local-only", action="store_true", default=False,
help="Don't use the server, just build from a local checkout")
parser.add_option("--branch", type="string", dest="svn_branch",
help="Override branch name (defaults to last path component of <SVN URL>)")
(options, args) = run_parser(parser, __doc__)
if not 1 <= len(args) <= 2:
display_parser_error(parser, "incorrect number of arguments")
target_svn_url = args.pop(0).rstrip("/")
local_repo = args and args.pop(0) or None
if options.svn_peg:
target_svn_url += "@" + str(options.svn_peg)
if options.local_only:
if options.svn_peg:
display_parser_error(parser,
"--local-only and --svn-peg are mutually exclusive")
if options.svn_rev:
display_parser_error(parser,
"--local-only and --svn-rev are mutually exclusive")
if local_repo:
display_parser_error(parser,
"--local-only does not accept a target directory")
# Get SVN info
svn_info = get_svn_info(target_svn_url, options.svn_rev)
# e.g. u'svn://svn.twistedmatrix.com/svn/Twisted'
repos_url = svn_info['repos_url']
# e.g. u'svn://svn.twistedmatrix.com/svn/Twisted/branches/xmpp-subprotocols-2178-2'
actual_svn_url = svn_info['url']
assert actual_svn_url.startswith(repos_url)
# if we are allowed to go to the server, lets use it
if options.local_only:
target_svn_url = os.path.abspath(target_svn_url)
local_repo = target_svn_url
# e.g. u'/branches/xmpp-subprotocols-2178-2'
svn_path = actual_svn_url[len(repos_url):]
# e.g. 'xmpp-subprotocols-2178-2'
svn_branch = actual_svn_url.split("/")[-1]
# if --branch was passed, override the branch name derived above
if options.svn_branch:
svn_branch = options.svn_branch
svn_greatest_rev = svn_info['last_changed_rev']
if options.svn_peg:
actual_svn_url += "@" + str(options.svn_peg)
if not local_repo:
local_repo = actual_svn_url.split("/")[-1]
if os.path.exists(local_repo):
if not os.path.isdir(local_repo):
raise ValueError("%s is not a directory" % local_repo)
is_svn_dir = os.path.isdir(os.path.join(local_repo, '.svn'))
if is_svn_dir and not options.local_only:
nlocal_repo = os.path.abspath(local_repo)
print "%s is already a SVN checkout." % nlocal_repo
sys.exit(1)
elif not is_svn_dir and options.local_only:
nlocal_repo = os.path.abspath(local_repo)
print "%s must be a SVN checkout for --local-only" % nlocal_repo
sys.exit(1)
elif options.local_only and is_svn_dir:
status = get_svn_status(local_repo, quiet=True)
if status:
print ("There are uncommitted changes. Either commit them "
"or put them aside before running hgimportsvn.")
sys.exit(1)
else:
os.mkdir(local_repo)
fixup_hgsvn_dir(local_repo)
os.chdir(local_repo)
# Get log entry for the SVN revision we will check out
svn_copyfrom_path = None
svn_copyfrom_revision = None
if options.local_only or svn_greatest_rev == 0:
# fake up a log message for the initial commit
svn_start_log = {}
svn_start_log['message'] = 'initial import by hgimportsvn'
svn_start_log['revision'] = svn_info['last_changed_rev']
svn_start_log['author'] = svn_info.get('last_changed_author')
svn_start_log['date'] = svn_info['last_changed_date']
elif options.svn_rev:
# If a specific rev was requested, get log entry just before or at rev
svn_start_log = get_last_svn_log_entry(actual_svn_url, 1, options.svn_rev)
else:
# Otherwise, get log entry of branch creation
svn_start_log = get_first_svn_log_entry(actual_svn_url, 1, svn_greatest_rev)
for p in svn_start_log['changed_paths']:
if p['path'] == svn_path:
svn_copyfrom_path = p['copyfrom_path']
svn_copyfrom_revision = p['copyfrom_revision']
break
if svn_copyfrom_path:
print "SVN branch was copied from '%s' at rev %s" % (
svn_copyfrom_path, svn_copyfrom_revision)
else:
print "SVN branch isn't a copy"
# This is the revision we will checkout from
svn_rev = svn_start_log['revision']
# Initialize hg repo
if not os.path.exists(".hg"):
run_hg(["init"])
if svn_copyfrom_path:
# Try to find an hg repo tracking the SVN branch which was copied
copyfrom_branch = svn_copyfrom_path.split("/")[-1]
hg_copyfrom = os.path.join("..", copyfrom_branch)
if (os.path.exists(os.path.join(hg_copyfrom, ".hg")) and
os.path.exists(os.path.join(hg_copyfrom, svn_private_dir))):
u = get_svn_info(hg_copyfrom)['url']
if u != repos_url + svn_copyfrom_path:
print "SVN URL %s in working copy %s doesn't match, ignoring" % (u, hg_copyfrom)
else:
# Find closest hg tag before requested SVN rev
best_tag = None
for line in run_hg(["tags", "-R", hg_copyfrom]).splitlines():
if not line.startswith("svn."):
continue
tag = line.split(None, 1)[0]
tag_num = int(tag.split(".")[1])
if tag_num <= svn_copyfrom_revision and (not best_tag or best_tag < tag_num):
best_tag = tag_num
if not best_tag:
print "No hg tag matching rev %s in %s" % (svn_rev, hg_copyfrom)
else:
# Don't use "pull -u" because it fails with hg 0.9.5
# ("branch default not found")
run_hg(["pull", "-r", "svn.%d" % best_tag, hg_copyfrom])
# Not specifying "tip" fails with hg 0.9.5
# ("branch default not found")
run_hg(["up", "tip"])
run_hg(["branch", svn_branch])
# Stay on the same filesystem so as to have fast moves
if options.local_only:
checkout_dir = target_svn_url
else:
checkout_dir = tempfile.mkdtemp(dir=".")
try:
# Get SVN manifest and checkout
if not options.local_only:
svn_checkout(target_svn_url, checkout_dir, svn_rev)
svn_manifest = get_svn_versioned_files(checkout_dir)
svn_files = set(skip_dirs(svn_manifest, checkout_dir))
svn_dirs = sorted(set(svn_manifest) - svn_files)
svn_files = list(svn_files)
# in the local case the files here already
if not options.local_only:
# All directories must exist, including empty ones
# (both for hg and for moving .svn dirs later)
for d in svn_dirs:
if not os.path.isdir(d):
if os.path.exists(d):
os.remove(d)
os.mkdir(d)
# Replace checked out files
for f in svn_files:
if os.path.exists(f):
os.remove(f)
os.rename(os.path.join(checkout_dir, f), f)
try:
# Add/remove new/old files
if svn_files:
run_hg(["addremove"] + hg_exclude_options, svn_files)
hg_commit_from_svn_log_entry(svn_start_log)
#hg_commit_from_svn_log_entry(svn_start_log, svn_files)
except:
run_hg(["revert", "--all"])
raise
# in the local case the files here already
if not options.local_only:
# Move SVN working copy here (don't forget base directory)
for d in chain([""], svn_dirs):
os.rename(os.path.join(checkout_dir, d, svn_private_dir), os.path.join(d, svn_private_dir))
finally:
# in the local case the checkout_dir is the target, so don't delete it
if not options.local_only:
rmtree(checkout_dir)
if options.hgignore:
svn_ignores = []
# we use '.' because that it the location of the hg repo that we are
# building and at this point we have already copied all of svn to the
# checkout (for both the local and non-local case)
for (path, dirs, files) in os.walk('.'):
if '.svn' in dirs:
dirs.remove('.svn')
# the [2:] strips off the initial './'
local_ignores = [os.path.join(path, s.strip())[2:]
for s in run_svn(['propget', 'svn:ignore', path],
mask_atsign=True).splitlines()
if bool(s.strip())
]
svn_ignores += local_ignores
else:
# if we are not inside an svn world
# we can just bail
del dirs[:]
# Generate .hgignore file to ignore .svn and .hgsvn directories
f = open(".hgignore", "a")
try:
f.write("\n# Automatically generated by `hgimportsvn`\n")
f.write("syntax:glob\n%s\n" %
"\n".join([svn_private_dir, hgsvn_private_dir]))
f.write("\n# These lines are suggested according to the svn:ignore property")
f.write("\n# Feel free to enable them by uncommenting them")
f.write("\nsyntax:glob\n")
f.write("".join("#%s\n" % s for s in svn_ignores))
finally:
f.close()
print "Finished! You can now pull all SVN history with 'hgpullsvn'."
def cleanup_svn_unversioned(files):
svn_status = get_svn_status(".")
for entry in svn_status:
if(entry['type'] == 'unversioned') and entry['path'] in files:
files.remove(entry['path'])
return files