Ejemplo n.º 1
0
def pull_svn_rev(log_entry, current_rev, svn_wc, wc_url, wc_base, retry):
    """
    Pull SVN changes from the given log entry.
    Returns the new SVN revision. If an exception occurs, it will rollback
    to revision 'current_rev'.
    """
    svn_rev = log_entry['revision']

    added_paths = []
    copied_paths = []
    removed_paths = []
    changed_paths = []
    unrelated_paths = []
    replaced_paths = {}

    # 1. Prepare for the `svn up` changes that are pulled in the second step
    #    by analyzing log_entry for the changeset
    for d in log_entry['changed_paths']:
        # e.g. u'/branches/xmpp-subprotocols-2178-2/twisted/words/test/test_jabberxmlstream.py'
        p = d['path']
        if not p.startswith(wc_base + "/"):
            # Ignore changed files that are not part of this subdir
            if p != wc_base:
                unrelated_paths.append(p)
            continue
        action = d['action']
        if action not in 'MARD':
            raise UnsupportedSVNAction(
                "In SVN rev. %d: action '%s' not supported. Please report a bug!"
                % (svn_rev, action))
        # e.g. u'twisted/words/test/test_jabberxmlstream.py'
        p = p[len(wc_base):].strip("/")
        # Record for commit
        changed_paths.append(p)
        # Detect special cases
        old_p = d['copyfrom_path']
        if old_p and old_p.startswith(wc_base + "/"):
            old_p = old_p[len(wc_base):].strip("/")
            # Both paths can be identical if copied from an old rev.
            # We treat like it a normal change.
            if old_p != p:
                # Try to hint hg about file and dir copies
                if not os.path.isdir(old_p):
                    copied_paths.append((old_p, p))
                    if action == 'R':
                        removed_paths.append(old_p)
                else:
                    # Extract actual copied files (hg doesn't track dirs
                    # and will refuse "hg copy -A" with dirs)
                    r = run_hg(["st", "-nc"], [old_p],
                               output_is_locale_encoding=True)
                    for old_f in r.splitlines():
                        f = p + old_f[len(old_p):]
                        copied_paths.append((old_f, f))
                        if action == 'R':
                            removed_paths.append(old_f)
                continue
        if d['action'] == 'A':
            added_paths.append(p)
        elif d['action'] == 'D':
            # Same as above: unfold directories into their affected files
            if not os.path.isdir(p):
                removed_paths.append(p)
            else:
                r = run_hg(["st", "-nc"], [p], output_is_locale_encoding=True)
                for f in r.splitlines():
                    removed_paths.append(f)
        elif d['action'] == 'R':
            # (R)eplaced directories can have added and removed files without
            # them being mentioned in the SVN log => we must detect those files
            # ourselves.
            # (http://svn.python.org/projects/python/branches/py3k, rev 59625)
            if os.path.isdir(p):
                replaced_paths[p] = get_svn_versioned_files(
                    os.path.join(svn_wc, p))
            else:
                # We never know what twisty semantics (R) can have. addremove
                # is safest.
                added_paths.append(p)

    # 2. Update SVN + add/remove/commit hg
    try:
        if changed_paths:
            args = ["up", "--ignore-externals"]
            if get_svn_client_version() >= (1, 5):
                args.extend(['--accept', 'postpone'])
            ui.status('Attempting to update to revision %s...', str(svn_rev))
            once_or_more("SVN update", retry, run_svn,
                         args + ["-r", svn_rev, svn_wc])
            conflicts = []
            for status_entry in get_svn_status('.'):
                if status_entry['status'] == 'conflicted':
                    conflicts.append(status_entry['path'])
            if conflicts:
                ui.status('SVN updated resulted in conflicts!', level=ui.ERROR)
                ui.status('Conflicted files: %s', ','.join(conflicts))
                ui.status('Please report a bug.')
                return None
            for p, old_contents in replaced_paths.items():
                old_contents = set(old_contents)
                new_contents = set(
                    get_svn_versioned_files(os.path.join(svn_wc, p)))
                added_paths.extend(p + '/' + f
                                   for f in new_contents - old_contents)
                removed_paths.extend(p + '/' + f
                                     for f in old_contents - new_contents)
            if added_paths:
                # Use 'addremove' because an added SVN directory may
                # overwrite a previous directory with the same name.
                # XXX what about untracked files in those directories?
                run_hg(["addremove"] + hg_exclude_options, added_paths)
            for old, new in copied_paths:
                try:
                    run_hg(["copy", "-A"], [old, new])
                except ExternalCommandFailed:
                    # Maybe the "old" path is obsolete, i.e. it comes from an
                    # old SVN revision and was later removed.
                    s = run_hg(['st', '-nd'], [old],
                               output_is_locale_encoding=True)
                    if s.strip():
                        # The old path is known by hg, something else happened.
                        raise
                    run_hg(["add"], [new])
            if removed_paths:
                try:
                    for file_path in removed_paths:
                        if os.path.exists(file_path):
                            run_hg(["remove", "-A"], file_path)
                except (ExternalCommandFailed), e:
                    if str(e).find("file is untracked") > 0:
                        ui.status(
                            "Ignoring warnings about untracked files: '%s'" %
                            str(e),
                            level=ui.VERBOSE)
                    else:
                        raise
            hg_commit_from_svn_log_entry(log_entry)
        elif unrelated_paths:
            detect_overwritten_svn_branch(wc_url, svn_rev)
Ejemplo n.º 2
0
def pull_svn_rev(log_entry, current_rev, svn_wc, wc_url, wc_base, retry):
    """
    Pull SVN changes from the given log entry.
    Returns the new SVN revision. If an exception occurs, it will rollback
    to revision 'current_rev'.
    """
    svn_rev = log_entry['revision']

    added_paths = []
    copied_paths = []
    removed_paths = []
    changed_paths = []
    unrelated_paths = []
    replaced_paths = {}

    # 1. Prepare for the `svn up` changes that are pulled in the second step
    #    by analyzing log_entry for the changeset
    for d in log_entry['changed_paths']:
        # e.g. u'/branches/xmpp-subprotocols-2178-2/twisted/words/test/test_jabberxmlstream.py'
        p = d['path']
        if not p.startswith(wc_base + "/"):
            # Ignore changed files that are not part of this subdir
            if p != wc_base:
                unrelated_paths.append(p)
            continue
        action = d['action']
        if action not in 'MARD':
            raise UnsupportedSVNAction("In SVN rev. %d: action '%s' not supported. Please report a bug!"
                % (svn_rev, action))
        # e.g. u'twisted/words/test/test_jabberxmlstream.py'
        p = p[len(wc_base):].strip("/")
        # Record for commit
        changed_paths.append(p)
        # Detect special cases
        old_p = d['copyfrom_path']
        if old_p and old_p.startswith(wc_base + "/"):
            old_p = old_p[len(wc_base):].strip("/")
            # Both paths can be identical if copied from an old rev.
            # We treat like it a normal change.
            if old_p != p:
                # Try to hint hg about file and dir copies
                if not os.path.isdir(old_p):
                    copied_paths.append((old_p, p))
                    if action == 'R':
                        removed_paths.append(old_p)
                else:
                    # Extract actual copied files (hg doesn't track dirs
                    # and will refuse "hg copy -A" with dirs)
                    r = run_hg(["st", "-nc"], [old_p], output_is_locale_encoding=True)
                    for old_f in r.splitlines():
                        f = p + old_f[len(old_p):]
                        copied_paths.append((old_f, f))
                        if action == 'R':
                            removed_paths.append(old_f)
                continue
        if d['action'] == 'A':
            added_paths.append(p)
        elif d['action'] == 'D':
            # Same as above: unfold directories into their affected files
            if not os.path.isdir(p):
                removed_paths.append(p)
            else:
                r = run_hg(["st", "-nc"], [p], output_is_locale_encoding=True)
                for f in r.splitlines():
                    removed_paths.append(f)
        elif d['action'] == 'R':
            # (R)eplaced directories can have added and removed files without
            # them being mentioned in the SVN log => we must detect those files
            # ourselves.
            # (http://svn.python.org/projects/python/branches/py3k, rev 59625)
            if os.path.isdir(p):
                replaced_paths[p] = get_svn_versioned_files(
                    os.path.join(svn_wc, p))
            else:
                # We never know what twisty semantics (R) can have. addremove
                # is safest.
                added_paths.append(p)

    # 2. Update SVN + add/remove/commit hg
    try:
        if changed_paths:
            args = ["up", "--ignore-externals"]
            if get_svn_client_version() >= (1, 5):
                args.extend(['--accept', 'postpone'])
            ui.status('Attempting to update to revision %s...', str(svn_rev))
            once_or_more("SVN update", retry, run_svn, args + ["-r", svn_rev, svn_wc])
            conflicts = []
            for status_entry in get_svn_status('.'):
                if status_entry['status'] == 'conflicted':
                    conflicts.append(status_entry['path'])
            if conflicts:
                ui.status('SVN updated resulted in conflicts!', level=ui.ERROR)
                ui.status('Conflicted files: %s', ','.join(conflicts))
                ui.status('Please report a bug.')
                return None
            for p, old_contents in replaced_paths.items():
                old_contents = set(old_contents)
                new_contents = set(get_svn_versioned_files(
                    os.path.join(svn_wc, p)))
                added_paths.extend(p + '/' + f for f in new_contents - old_contents)
                removed_paths.extend(p + '/' + f for f in old_contents - new_contents)
            if added_paths:
                # Use 'addremove' because an added SVN directory may
                # overwrite a previous directory with the same name.
                # XXX what about untracked files in those directories?
                run_hg(["addremove"] + hg_exclude_options, added_paths)
            for old, new in copied_paths:
                try:
                    run_hg(["copy", "-A"], [old, new])
                except ExternalCommandFailed:
                    # Maybe the "old" path is obsolete, i.e. it comes from an
                    # old SVN revision and was later removed.
                    s = run_hg(['st', '-nd'], [old], output_is_locale_encoding=True)
                    if s.strip():
                        # The old path is known by hg, something else happened.
                        raise
                    run_hg(["add"], [new])
            if removed_paths:
                try: 
                    for file_path in removed_paths:
                       if os.path.exists(file_path):
                           run_hg(["remove", "-A"], file_path)
                except (ExternalCommandFailed), e:
                    if str(e).find("file is untracked") > 0:
                        ui.status("Ignoring warnings about untracked files: '%s'" % str(e), level=ui.VERBOSE)
                    else:
                        raise
            hg_commit_from_svn_log_entry(log_entry)
        elif unrelated_paths:
            detect_overwritten_svn_branch(wc_url, svn_rev)
Ejemplo n.º 3
0
def cleanup_svn_unversioned(files):
    svn_status = get_svn_status(".")
    for entry in svn_status:
        if (entry['type'] == 'unversioned') and entry['path'] in files:
            files.remove(entry['path'])
    return files
Ejemplo n.º 4
0
def main():
    usage = """1. %prog [-r SVN rev] [-p SVN peg rev] <SVN URL> [local checkout]
       2. %prog --local-only <local checkout>"""
    parser = OptionParser(usage)
    parser.add_option("-r", "--svn-rev", type="int", dest="svn_rev",
        help="SVN revision to checkout from")
    parser.add_option("-p", "--svn-peg", type="int", dest="svn_peg",
        help="SVN peg revision to locate checkout URL")
    parser.add_option("--no-hgignore", dest="hgignore", action="store_false",
        default=True, help="Don't autogenerate .hgignore")
    parser.add_option("--local-only", action="store_true", default=False,
        help="Don't use the server, just build from a local checkout")
    parser.add_option("--branch", type="string", dest="svn_branch",
        help="Override branch name (defaults to last path component of <SVN URL>)")
    (options, args) = run_parser(parser, __doc__)
    if not 1 <= len(args) <= 2:
        display_parser_error(parser, "incorrect number of arguments")

    target_svn_url = args.pop(0).rstrip("/")
    local_repo = args and args.pop(0) or None
    if options.svn_peg:
       target_svn_url += "@" + str(options.svn_peg)

    if options.local_only:
        if options.svn_peg:
            display_parser_error(parser,
                    "--local-only and --svn-peg are mutually exclusive")
        if options.svn_rev:
            display_parser_error(parser,
                    "--local-only and --svn-rev are mutually exclusive")
        if local_repo:
            display_parser_error(parser,
                    "--local-only does not accept a target directory")



    # Get SVN info
    svn_info = get_svn_info(target_svn_url, options.svn_rev)
    # e.g. u'svn://svn.twistedmatrix.com/svn/Twisted'
    repos_url = svn_info['repos_url']
    # e.g. u'svn://svn.twistedmatrix.com/svn/Twisted/branches/xmpp-subprotocols-2178-2'
    actual_svn_url = svn_info['url']
    assert actual_svn_url.startswith(repos_url)

    # if we are allowed to go to the server, lets use it
    if options.local_only:
        target_svn_url = os.path.abspath(target_svn_url)
        local_repo = target_svn_url

    # e.g. u'/branches/xmpp-subprotocols-2178-2'
    svn_path = actual_svn_url[len(repos_url):]
    # e.g. 'xmpp-subprotocols-2178-2'
    svn_branch = actual_svn_url.split("/")[-1]

    # if --branch was passed, override the branch name derived above
    if options.svn_branch:
        svn_branch = options.svn_branch

    svn_greatest_rev = svn_info['last_changed_rev']
    if options.svn_peg:
       actual_svn_url += "@" + str(options.svn_peg)

    if not local_repo:
        local_repo = actual_svn_url.split("/")[-1]
    if os.path.exists(local_repo):
        if not os.path.isdir(local_repo):
            raise ValueError("%s is not a directory" % local_repo)
        is_svn_dir = os.path.isdir(os.path.join(local_repo, '.svn'))
        if is_svn_dir and not options.local_only:
            nlocal_repo = os.path.abspath(local_repo)
            print "%s is already a SVN checkout." % nlocal_repo
            sys.exit(1)
        elif not is_svn_dir and options.local_only:
            nlocal_repo = os.path.abspath(local_repo)
            print "%s must be a SVN checkout for --local-only" % nlocal_repo
            sys.exit(1)
        elif options.local_only and is_svn_dir:
            status = get_svn_status(local_repo, quiet=True)
            if status:
                print ("There are uncommitted changes. Either commit them "
                       "or put them aside before running hgimportsvn.")
                sys.exit(1)
    else:
        os.mkdir(local_repo)
    fixup_hgsvn_dir(local_repo)
    os.chdir(local_repo)

    # Get log entry for the SVN revision we will check out
    svn_copyfrom_path = None
    svn_copyfrom_revision = None
    if options.local_only or svn_greatest_rev == 0:
        # fake up a log message for the initial commit
        svn_start_log = {}
        svn_start_log['message'] = 'initial import by hgimportsvn'
        svn_start_log['revision'] = svn_info['last_changed_rev']
        svn_start_log['author'] = svn_info.get('last_changed_author')
        svn_start_log['date'] = svn_info['last_changed_date']
    elif options.svn_rev:
        # If a specific rev was requested, get log entry just before or at rev
        svn_start_log = get_last_svn_log_entry(actual_svn_url, 1, options.svn_rev)
    else:
        # Otherwise, get log entry of branch creation
        svn_start_log = get_first_svn_log_entry(actual_svn_url, 1, svn_greatest_rev)
        for p in svn_start_log['changed_paths']:
            if p['path'] == svn_path:
                svn_copyfrom_path = p['copyfrom_path']
                svn_copyfrom_revision = p['copyfrom_revision']
                break
        if svn_copyfrom_path:
            print "SVN branch was copied from '%s' at rev %s" % (
                svn_copyfrom_path, svn_copyfrom_revision)
        else:
            print "SVN branch isn't a copy"
    # This is the revision we will checkout from
    svn_rev = svn_start_log['revision']

    # Initialize hg repo
    if not os.path.exists(".hg"):
        run_hg(["init"])
    if svn_copyfrom_path:
        # Try to find an hg repo tracking the SVN branch which was copied
        copyfrom_branch = svn_copyfrom_path.split("/")[-1]
        hg_copyfrom = os.path.join("..", copyfrom_branch)
        if (os.path.exists(os.path.join(hg_copyfrom, ".hg")) and
            os.path.exists(os.path.join(hg_copyfrom, svn_private_dir))):
            u = get_svn_info(hg_copyfrom)['url']
            if u != repos_url + svn_copyfrom_path:
                print "SVN URL %s in working copy %s doesn't match, ignoring" % (u, hg_copyfrom)
            else:
                # Find closest hg tag before requested SVN rev
                best_tag = None
                for line in run_hg(["tags", "-R", hg_copyfrom]).splitlines():
                    if not line.startswith("svn."):
                        continue
                    tag = line.split(None, 1)[0]
                    tag_num = int(tag.split(".")[1])
                    if tag_num <= svn_copyfrom_revision and (not best_tag or best_tag < tag_num):
                        best_tag = tag_num
                if not best_tag:
                    print "No hg tag matching rev %s in %s" % (svn_rev, hg_copyfrom)
                else:
                    # Don't use "pull -u" because it fails with hg 0.9.5
                    # ("branch default not found")
                    run_hg(["pull", "-r", "svn.%d" % best_tag, hg_copyfrom])
                    # Not specifying "tip" fails with hg 0.9.5
                    # ("branch default not found")
                    run_hg(["up", "tip"])
    run_hg(["branch", svn_branch])


    # Stay on the same filesystem so as to have fast moves
    if options.local_only:
        checkout_dir = target_svn_url
    else:
        checkout_dir = tempfile.mkdtemp(dir=".")

    try:
        # Get SVN manifest and checkout
        if not options.local_only:
            svn_checkout(target_svn_url, checkout_dir, svn_rev)
        svn_manifest = get_svn_versioned_files(checkout_dir)

        svn_files = set(skip_dirs(svn_manifest, checkout_dir))
        svn_dirs = sorted(set(svn_manifest) - svn_files)
        svn_files = list(svn_files)

        # in the local case the files here already
        if not options.local_only:
            # All directories must exist, including empty ones
            # (both for hg and for moving .svn dirs later)
            for d in svn_dirs:
                if not os.path.isdir(d):
                    if os.path.exists(d):
                        os.remove(d)
                    os.mkdir(d)
            # Replace checked out files
            for f in svn_files:
                if os.path.exists(f):
                    os.remove(f)
                os.rename(os.path.join(checkout_dir, f), f)

        try:
            # Add/remove new/old files
            if svn_files:
                run_hg(["addremove"] + hg_exclude_options, svn_files)
            hg_commit_from_svn_log_entry(svn_start_log)
            #hg_commit_from_svn_log_entry(svn_start_log, svn_files)
        except:
            run_hg(["revert", "--all"])
            raise

        # in the local case the files here already
        if not options.local_only:
            # Move SVN working copy here (don't forget base directory)
            for d in chain([""], svn_dirs):
                os.rename(os.path.join(checkout_dir, d, svn_private_dir), os.path.join(d, svn_private_dir))

    finally:
        # in the local case the checkout_dir is the target, so don't delete it
        if not options.local_only:
            rmtree(checkout_dir)

    if options.hgignore:
        svn_ignores = []
        # we use '.' because that it the location of the hg repo that we are
        # building and at this point we have already copied all of svn to the
        # checkout (for both the local and non-local case)
        for (path, dirs, files) in os.walk('.'):
            if '.svn' in dirs:
                dirs.remove('.svn')

                # the [2:] strips off the initial './'
                local_ignores = [os.path.join(path, s.strip())[2:]
                    for s in run_svn(['propget', 'svn:ignore', path],
                                     mask_atsign=True).splitlines()
                    if bool(s.strip())
                ]
                svn_ignores += local_ignores
            else:
                # if we are not inside an svn world
                # we can just bail
                del dirs[:]


        # Generate .hgignore file to ignore .svn and .hgsvn directories
        f = open(".hgignore", "a")
        try:
            f.write("\n# Automatically generated by `hgimportsvn`\n")
            f.write("syntax:glob\n%s\n" %
                "\n".join([svn_private_dir, hgsvn_private_dir]))
            f.write("\n# These lines are suggested according to the svn:ignore property")
            f.write("\n# Feel free to enable them by uncommenting them")
            f.write("\nsyntax:glob\n")
            f.write("".join("#%s\n" % s for s in svn_ignores))
        finally:
            f.close()

    print "Finished! You can now pull all SVN history with 'hgpullsvn'."
Ejemplo n.º 5
0
def cleanup_svn_unversioned(files):
    svn_status = get_svn_status(".")
    for entry in svn_status:
        if(entry['type'] == 'unversioned') and entry['path'] in files:
            files.remove(entry['path'])
    return files