def pull_svn_rev(log_entry, current_rev, svn_wc, wc_url, wc_base, retry): """ Pull SVN changes from the given log entry. Returns the new SVN revision. If an exception occurs, it will rollback to revision 'current_rev'. """ svn_rev = log_entry['revision'] added_paths = [] copied_paths = [] removed_paths = [] changed_paths = [] unrelated_paths = [] replaced_paths = {} # 1. Prepare for the `svn up` changes that are pulled in the second step # by analyzing log_entry for the changeset for d in log_entry['changed_paths']: # e.g. u'/branches/xmpp-subprotocols-2178-2/twisted/words/test/test_jabberxmlstream.py' p = d['path'] if not p.startswith(wc_base + "/"): # Ignore changed files that are not part of this subdir if p != wc_base: unrelated_paths.append(p) continue action = d['action'] if action not in 'MARD': raise UnsupportedSVNAction( "In SVN rev. %d: action '%s' not supported. Please report a bug!" % (svn_rev, action)) # e.g. u'twisted/words/test/test_jabberxmlstream.py' p = p[len(wc_base):].strip("/") # Record for commit changed_paths.append(p) # Detect special cases old_p = d['copyfrom_path'] if old_p and old_p.startswith(wc_base + "/"): old_p = old_p[len(wc_base):].strip("/") # Both paths can be identical if copied from an old rev. # We treat like it a normal change. if old_p != p: # Try to hint hg about file and dir copies if not os.path.isdir(old_p): copied_paths.append((old_p, p)) if action == 'R': removed_paths.append(old_p) else: # Extract actual copied files (hg doesn't track dirs # and will refuse "hg copy -A" with dirs) r = run_hg(["st", "-nc"], [old_p], output_is_locale_encoding=True) for old_f in r.splitlines(): f = p + old_f[len(old_p):] copied_paths.append((old_f, f)) if action == 'R': removed_paths.append(old_f) continue if d['action'] == 'A': added_paths.append(p) elif d['action'] == 'D': # Same as above: unfold directories into their affected files if not os.path.isdir(p): removed_paths.append(p) else: r = run_hg(["st", "-nc"], [p], output_is_locale_encoding=True) for f in r.splitlines(): removed_paths.append(f) elif d['action'] == 'R': # (R)eplaced directories can have added and removed files without # them being mentioned in the SVN log => we must detect those files # ourselves. # (http://svn.python.org/projects/python/branches/py3k, rev 59625) if os.path.isdir(p): replaced_paths[p] = get_svn_versioned_files( os.path.join(svn_wc, p)) else: # We never know what twisty semantics (R) can have. addremove # is safest. added_paths.append(p) # 2. Update SVN + add/remove/commit hg try: if changed_paths: args = ["up", "--ignore-externals"] if get_svn_client_version() >= (1, 5): args.extend(['--accept', 'postpone']) ui.status('Attempting to update to revision %s...', str(svn_rev)) once_or_more("SVN update", retry, run_svn, args + ["-r", svn_rev, svn_wc]) conflicts = [] for status_entry in get_svn_status('.'): if status_entry['status'] == 'conflicted': conflicts.append(status_entry['path']) if conflicts: ui.status('SVN updated resulted in conflicts!', level=ui.ERROR) ui.status('Conflicted files: %s', ','.join(conflicts)) ui.status('Please report a bug.') return None for p, old_contents in replaced_paths.items(): old_contents = set(old_contents) new_contents = set( get_svn_versioned_files(os.path.join(svn_wc, p))) added_paths.extend(p + '/' + f for f in new_contents - old_contents) removed_paths.extend(p + '/' + f for f in old_contents - new_contents) if added_paths: # Use 'addremove' because an added SVN directory may # overwrite a previous directory with the same name. # XXX what about untracked files in those directories? run_hg(["addremove"] + hg_exclude_options, added_paths) for old, new in copied_paths: try: run_hg(["copy", "-A"], [old, new]) except ExternalCommandFailed: # Maybe the "old" path is obsolete, i.e. it comes from an # old SVN revision and was later removed. s = run_hg(['st', '-nd'], [old], output_is_locale_encoding=True) if s.strip(): # The old path is known by hg, something else happened. raise run_hg(["add"], [new]) if removed_paths: try: for file_path in removed_paths: if os.path.exists(file_path): run_hg(["remove", "-A"], file_path) except (ExternalCommandFailed), e: if str(e).find("file is untracked") > 0: ui.status( "Ignoring warnings about untracked files: '%s'" % str(e), level=ui.VERBOSE) else: raise hg_commit_from_svn_log_entry(log_entry) elif unrelated_paths: detect_overwritten_svn_branch(wc_url, svn_rev)
def pull_svn_rev(log_entry, current_rev, svn_wc, wc_url, wc_base, retry): """ Pull SVN changes from the given log entry. Returns the new SVN revision. If an exception occurs, it will rollback to revision 'current_rev'. """ svn_rev = log_entry['revision'] added_paths = [] copied_paths = [] removed_paths = [] changed_paths = [] unrelated_paths = [] replaced_paths = {} # 1. Prepare for the `svn up` changes that are pulled in the second step # by analyzing log_entry for the changeset for d in log_entry['changed_paths']: # e.g. u'/branches/xmpp-subprotocols-2178-2/twisted/words/test/test_jabberxmlstream.py' p = d['path'] if not p.startswith(wc_base + "/"): # Ignore changed files that are not part of this subdir if p != wc_base: unrelated_paths.append(p) continue action = d['action'] if action not in 'MARD': raise UnsupportedSVNAction("In SVN rev. %d: action '%s' not supported. Please report a bug!" % (svn_rev, action)) # e.g. u'twisted/words/test/test_jabberxmlstream.py' p = p[len(wc_base):].strip("/") # Record for commit changed_paths.append(p) # Detect special cases old_p = d['copyfrom_path'] if old_p and old_p.startswith(wc_base + "/"): old_p = old_p[len(wc_base):].strip("/") # Both paths can be identical if copied from an old rev. # We treat like it a normal change. if old_p != p: # Try to hint hg about file and dir copies if not os.path.isdir(old_p): copied_paths.append((old_p, p)) if action == 'R': removed_paths.append(old_p) else: # Extract actual copied files (hg doesn't track dirs # and will refuse "hg copy -A" with dirs) r = run_hg(["st", "-nc"], [old_p], output_is_locale_encoding=True) for old_f in r.splitlines(): f = p + old_f[len(old_p):] copied_paths.append((old_f, f)) if action == 'R': removed_paths.append(old_f) continue if d['action'] == 'A': added_paths.append(p) elif d['action'] == 'D': # Same as above: unfold directories into their affected files if not os.path.isdir(p): removed_paths.append(p) else: r = run_hg(["st", "-nc"], [p], output_is_locale_encoding=True) for f in r.splitlines(): removed_paths.append(f) elif d['action'] == 'R': # (R)eplaced directories can have added and removed files without # them being mentioned in the SVN log => we must detect those files # ourselves. # (http://svn.python.org/projects/python/branches/py3k, rev 59625) if os.path.isdir(p): replaced_paths[p] = get_svn_versioned_files( os.path.join(svn_wc, p)) else: # We never know what twisty semantics (R) can have. addremove # is safest. added_paths.append(p) # 2. Update SVN + add/remove/commit hg try: if changed_paths: args = ["up", "--ignore-externals"] if get_svn_client_version() >= (1, 5): args.extend(['--accept', 'postpone']) ui.status('Attempting to update to revision %s...', str(svn_rev)) once_or_more("SVN update", retry, run_svn, args + ["-r", svn_rev, svn_wc]) conflicts = [] for status_entry in get_svn_status('.'): if status_entry['status'] == 'conflicted': conflicts.append(status_entry['path']) if conflicts: ui.status('SVN updated resulted in conflicts!', level=ui.ERROR) ui.status('Conflicted files: %s', ','.join(conflicts)) ui.status('Please report a bug.') return None for p, old_contents in replaced_paths.items(): old_contents = set(old_contents) new_contents = set(get_svn_versioned_files( os.path.join(svn_wc, p))) added_paths.extend(p + '/' + f for f in new_contents - old_contents) removed_paths.extend(p + '/' + f for f in old_contents - new_contents) if added_paths: # Use 'addremove' because an added SVN directory may # overwrite a previous directory with the same name. # XXX what about untracked files in those directories? run_hg(["addremove"] + hg_exclude_options, added_paths) for old, new in copied_paths: try: run_hg(["copy", "-A"], [old, new]) except ExternalCommandFailed: # Maybe the "old" path is obsolete, i.e. it comes from an # old SVN revision and was later removed. s = run_hg(['st', '-nd'], [old], output_is_locale_encoding=True) if s.strip(): # The old path is known by hg, something else happened. raise run_hg(["add"], [new]) if removed_paths: try: for file_path in removed_paths: if os.path.exists(file_path): run_hg(["remove", "-A"], file_path) except (ExternalCommandFailed), e: if str(e).find("file is untracked") > 0: ui.status("Ignoring warnings about untracked files: '%s'" % str(e), level=ui.VERBOSE) else: raise hg_commit_from_svn_log_entry(log_entry) elif unrelated_paths: detect_overwritten_svn_branch(wc_url, svn_rev)
def cleanup_svn_unversioned(files): svn_status = get_svn_status(".") for entry in svn_status: if (entry['type'] == 'unversioned') and entry['path'] in files: files.remove(entry['path']) return files
def main(): usage = """1. %prog [-r SVN rev] [-p SVN peg rev] <SVN URL> [local checkout] 2. %prog --local-only <local checkout>""" parser = OptionParser(usage) parser.add_option("-r", "--svn-rev", type="int", dest="svn_rev", help="SVN revision to checkout from") parser.add_option("-p", "--svn-peg", type="int", dest="svn_peg", help="SVN peg revision to locate checkout URL") parser.add_option("--no-hgignore", dest="hgignore", action="store_false", default=True, help="Don't autogenerate .hgignore") parser.add_option("--local-only", action="store_true", default=False, help="Don't use the server, just build from a local checkout") parser.add_option("--branch", type="string", dest="svn_branch", help="Override branch name (defaults to last path component of <SVN URL>)") (options, args) = run_parser(parser, __doc__) if not 1 <= len(args) <= 2: display_parser_error(parser, "incorrect number of arguments") target_svn_url = args.pop(0).rstrip("/") local_repo = args and args.pop(0) or None if options.svn_peg: target_svn_url += "@" + str(options.svn_peg) if options.local_only: if options.svn_peg: display_parser_error(parser, "--local-only and --svn-peg are mutually exclusive") if options.svn_rev: display_parser_error(parser, "--local-only and --svn-rev are mutually exclusive") if local_repo: display_parser_error(parser, "--local-only does not accept a target directory") # Get SVN info svn_info = get_svn_info(target_svn_url, options.svn_rev) # e.g. u'svn://svn.twistedmatrix.com/svn/Twisted' repos_url = svn_info['repos_url'] # e.g. u'svn://svn.twistedmatrix.com/svn/Twisted/branches/xmpp-subprotocols-2178-2' actual_svn_url = svn_info['url'] assert actual_svn_url.startswith(repos_url) # if we are allowed to go to the server, lets use it if options.local_only: target_svn_url = os.path.abspath(target_svn_url) local_repo = target_svn_url # e.g. u'/branches/xmpp-subprotocols-2178-2' svn_path = actual_svn_url[len(repos_url):] # e.g. 'xmpp-subprotocols-2178-2' svn_branch = actual_svn_url.split("/")[-1] # if --branch was passed, override the branch name derived above if options.svn_branch: svn_branch = options.svn_branch svn_greatest_rev = svn_info['last_changed_rev'] if options.svn_peg: actual_svn_url += "@" + str(options.svn_peg) if not local_repo: local_repo = actual_svn_url.split("/")[-1] if os.path.exists(local_repo): if not os.path.isdir(local_repo): raise ValueError("%s is not a directory" % local_repo) is_svn_dir = os.path.isdir(os.path.join(local_repo, '.svn')) if is_svn_dir and not options.local_only: nlocal_repo = os.path.abspath(local_repo) print "%s is already a SVN checkout." % nlocal_repo sys.exit(1) elif not is_svn_dir and options.local_only: nlocal_repo = os.path.abspath(local_repo) print "%s must be a SVN checkout for --local-only" % nlocal_repo sys.exit(1) elif options.local_only and is_svn_dir: status = get_svn_status(local_repo, quiet=True) if status: print ("There are uncommitted changes. Either commit them " "or put them aside before running hgimportsvn.") sys.exit(1) else: os.mkdir(local_repo) fixup_hgsvn_dir(local_repo) os.chdir(local_repo) # Get log entry for the SVN revision we will check out svn_copyfrom_path = None svn_copyfrom_revision = None if options.local_only or svn_greatest_rev == 0: # fake up a log message for the initial commit svn_start_log = {} svn_start_log['message'] = 'initial import by hgimportsvn' svn_start_log['revision'] = svn_info['last_changed_rev'] svn_start_log['author'] = svn_info.get('last_changed_author') svn_start_log['date'] = svn_info['last_changed_date'] elif options.svn_rev: # If a specific rev was requested, get log entry just before or at rev svn_start_log = get_last_svn_log_entry(actual_svn_url, 1, options.svn_rev) else: # Otherwise, get log entry of branch creation svn_start_log = get_first_svn_log_entry(actual_svn_url, 1, svn_greatest_rev) for p in svn_start_log['changed_paths']: if p['path'] == svn_path: svn_copyfrom_path = p['copyfrom_path'] svn_copyfrom_revision = p['copyfrom_revision'] break if svn_copyfrom_path: print "SVN branch was copied from '%s' at rev %s" % ( svn_copyfrom_path, svn_copyfrom_revision) else: print "SVN branch isn't a copy" # This is the revision we will checkout from svn_rev = svn_start_log['revision'] # Initialize hg repo if not os.path.exists(".hg"): run_hg(["init"]) if svn_copyfrom_path: # Try to find an hg repo tracking the SVN branch which was copied copyfrom_branch = svn_copyfrom_path.split("/")[-1] hg_copyfrom = os.path.join("..", copyfrom_branch) if (os.path.exists(os.path.join(hg_copyfrom, ".hg")) and os.path.exists(os.path.join(hg_copyfrom, svn_private_dir))): u = get_svn_info(hg_copyfrom)['url'] if u != repos_url + svn_copyfrom_path: print "SVN URL %s in working copy %s doesn't match, ignoring" % (u, hg_copyfrom) else: # Find closest hg tag before requested SVN rev best_tag = None for line in run_hg(["tags", "-R", hg_copyfrom]).splitlines(): if not line.startswith("svn."): continue tag = line.split(None, 1)[0] tag_num = int(tag.split(".")[1]) if tag_num <= svn_copyfrom_revision and (not best_tag or best_tag < tag_num): best_tag = tag_num if not best_tag: print "No hg tag matching rev %s in %s" % (svn_rev, hg_copyfrom) else: # Don't use "pull -u" because it fails with hg 0.9.5 # ("branch default not found") run_hg(["pull", "-r", "svn.%d" % best_tag, hg_copyfrom]) # Not specifying "tip" fails with hg 0.9.5 # ("branch default not found") run_hg(["up", "tip"]) run_hg(["branch", svn_branch]) # Stay on the same filesystem so as to have fast moves if options.local_only: checkout_dir = target_svn_url else: checkout_dir = tempfile.mkdtemp(dir=".") try: # Get SVN manifest and checkout if not options.local_only: svn_checkout(target_svn_url, checkout_dir, svn_rev) svn_manifest = get_svn_versioned_files(checkout_dir) svn_files = set(skip_dirs(svn_manifest, checkout_dir)) svn_dirs = sorted(set(svn_manifest) - svn_files) svn_files = list(svn_files) # in the local case the files here already if not options.local_only: # All directories must exist, including empty ones # (both for hg and for moving .svn dirs later) for d in svn_dirs: if not os.path.isdir(d): if os.path.exists(d): os.remove(d) os.mkdir(d) # Replace checked out files for f in svn_files: if os.path.exists(f): os.remove(f) os.rename(os.path.join(checkout_dir, f), f) try: # Add/remove new/old files if svn_files: run_hg(["addremove"] + hg_exclude_options, svn_files) hg_commit_from_svn_log_entry(svn_start_log) #hg_commit_from_svn_log_entry(svn_start_log, svn_files) except: run_hg(["revert", "--all"]) raise # in the local case the files here already if not options.local_only: # Move SVN working copy here (don't forget base directory) for d in chain([""], svn_dirs): os.rename(os.path.join(checkout_dir, d, svn_private_dir), os.path.join(d, svn_private_dir)) finally: # in the local case the checkout_dir is the target, so don't delete it if not options.local_only: rmtree(checkout_dir) if options.hgignore: svn_ignores = [] # we use '.' because that it the location of the hg repo that we are # building and at this point we have already copied all of svn to the # checkout (for both the local and non-local case) for (path, dirs, files) in os.walk('.'): if '.svn' in dirs: dirs.remove('.svn') # the [2:] strips off the initial './' local_ignores = [os.path.join(path, s.strip())[2:] for s in run_svn(['propget', 'svn:ignore', path], mask_atsign=True).splitlines() if bool(s.strip()) ] svn_ignores += local_ignores else: # if we are not inside an svn world # we can just bail del dirs[:] # Generate .hgignore file to ignore .svn and .hgsvn directories f = open(".hgignore", "a") try: f.write("\n# Automatically generated by `hgimportsvn`\n") f.write("syntax:glob\n%s\n" % "\n".join([svn_private_dir, hgsvn_private_dir])) f.write("\n# These lines are suggested according to the svn:ignore property") f.write("\n# Feel free to enable them by uncommenting them") f.write("\nsyntax:glob\n") f.write("".join("#%s\n" % s for s in svn_ignores)) finally: f.close() print "Finished! You can now pull all SVN history with 'hgpullsvn'."
def cleanup_svn_unversioned(files): svn_status = get_svn_status(".") for entry in svn_status: if(entry['type'] == 'unversioned') and entry['path'] in files: files.remove(entry['path']) return files