示例#1
0
def uncrustify_sources(svn_path, svn_co_root, uncrustify_config, uncrustify_path_accept, uncrustify_path_reject):
    ## @brief Uncrustify code before import
    #  @param svn_path Filesystem path to parsed up SVN checkout
    #  @param svn_co_root Base directory of SVN checkout
    #  @param uncrustify_config Config file for uncrustify pass
    #  @param uncrustify_path_accept Paths to force
    #  @param uncrustify_path_reject Paths to exclude

    for root, dirs, files in os.walk(svn_path):
        for name in files:
            filename = os.path.join(root, name)
            svn_filename = filename[len(svn_co_root) + 1:]
            path_veto = False
            for filter in uncrustify_path_reject:
                if re.match(filter, svn_filename):
                    logger.debug("File {0} will not go through uncrustify".format(svn_filename, filter.pattern))
                    path_veto = True
                    break
            for filter in uncrustify_path_accept:
                if re.match(filter, svn_filename):
                    logger.debug("File {0} will be passed to uncrustify".format(svn_filename, filter.pattern))
                    path_veto = False
                    break
            if path_veto:
                continue
            extension = filename.rsplit(".", 1)[1] if "." in filename else ""
            if extension in ("cxx", "cpp", "icc", "cc", "c", "C", "h", "hpp", "hh"):
                logger.debug("Uncrustifying {0}".format(filename))
                cmd = ("uncrustify", "-c", uncrustify_config, "--no-backup", "-l", "CPP", filename)
                # We do not consider uncrustify errors as fatal for the import... this can
                # happen because of a source file issue or picking the wrong language
                try:
                    check_output_with_retry(cmd, retries=0)
                except RuntimeError:
                    logger.warning("Uncrustify failed on {0}".format(filename))
示例#2
0
def init_git(gitrepo):
    ## @brief Initialise git repo, if needed
    #  @param gitrepo Git repository path
    if not os.path.exists(gitrepo):
        os.makedirs(gitrepo)
    os.chdir(gitrepo)
    if os.path.exists(os.path.join(gitrepo, ".git")):
        logger.info("Found existing git repo, {0}".format(gitrepo))
        check_output_with_retry(("git", "reset", "--hard"))
    else:
        logger.info("Initialising git repo: {0}".format(gitrepo))
        check_output_with_retry(("git", "init"))
示例#3
0
def svn_get_path_metadata(svnroot, package, package_path, revision=None):
    # # @brief Get SVN metadata and return as a simple dictionary keyed on date, author and commit revision
    logger.info("Querying SVN metadata for {0}".format(os.path.join(package, package_path)))
    cmd = ["svn", "info", os.path.join(svnroot, package, package_path), "--xml"]
    svn_info = check_output_with_retry(cmd)
    tree = eltree.fromstring(svn_info)
    info = {"date": tree.find(".//date").text.rsplit(".",1)[0], # Strip off sub-second part
            "author": tree.find(".//author").text,
            "revision": tree.find(".//commit").attrib['revision']}

    cmd = ["svn", "log", os.path.join(svnroot, package, package_path), "-r", info["revision"], "--xml"]
    svn_log = check_output_with_retry(cmd)
    tree = eltree.fromstring(svn_log)
    info["msg"] = tree.find(".//msg").text.strip().encode('ascii', 'ignore')
    return info
示例#4
0
def author_info_lookup(author_name):
    try:
        cmd = ["phonebook", "--login", author_name, "--terse", "firstname", "--terse", "surname", "--terse", "email"]
        author_info = check_output_with_retry(cmd, retries=1).strip().split(";")
        return {"name": " ".join(author_info[:2]), "email": author_info[2]}
    except IndexError:
        raise RuntimeError("Had a problem decoding phonebook info for '{0}'".format(author_name))
def prepare_branch_point(branch, parentbranch=None):
    ## @brief Using information about the target branch and any parent
    #  switch to the correct point in history to start/continue
    #  @param branch Target branch name
    #  @param parentbranch If creating a new branch, this is the @c BRANCH:COMMIT_ID of 
    #  where to make the new branch from; syntax @c BRANCH:@FILE and @c BRANCH:@TIMESTAMP
    #  is also supported, where the timestamp will be used to find the branch directly
    #  (and can be taken from @c JSON release data in @c FILE) 
    if not parentbranch or branch_exists(branch):
        logger.info("Switching to branch {0}".format(branch))
        switch_to_branch(branch, orphan=True)
    else:
        parent, commit = parentbranch.split(":")
        check_output_with_retry(("git", "checkout", parent), retries=1) # needed?
        if commit.startswith("@"):
            timestamp = commit[1:]
            # If this maps to a file, try to open it as a release JSON, otherwise treat it as
            # a plain timestamp
            if os.access(timestamp, os.F_OK):
                logger.info("Taking branching timestamp from file {0}".format(timestamp))
                with open(timestamp) as fh:
                    branch_point_release = json.load(fh)
                timestamp = branch_point_release["release"]["timestamp"]
            logger.info("Using timestamp {0} for branch point".format(timestamp))
            commit = check_output_with_retry(["git", "log", "--until", str(timestamp), "-n1", "--pretty=format:%H"],
                                             retries=1).strip()
            logger.info("Mapped timestamp {0} to commit {1}".format(timestamp, commit))
        check_output_with_retry(("git", "checkout", commit), retries=1)
        check_output_with_retry(("git", "checkout", "-b", branch), retries=1)
def branch_builder(gitrepo, branch, tag_files, svn_metadata_cache, author_metadata_cache,
                   parentbranch=None, baserelease=None,
                   skipreleasetag=False, dryrun=False, only_forward=False, commit_date="now"):
    ## @brief Main branch builder function
    #  @param gitrepo The git repository location
    #  @param branch The git branch to work on
    #  @param tag_files The plain tag content files to process
    #  @param svn_metadata_cache The standard metadata cache from SVN
    #  @param author_metadata_cache Cached author data
    #  @param parentbranch If creating a new branch, this is the BRANCH:COMMIT_ID of where to make the new branch from
    #  @param skipreleasetag If @c True then skip creating git tags for each processed release
    #  @param dryrun If @c True, do nothing except print commands that would have been executed
    #  @param only_forward If @c True then never revert a package to a previous version or import a branch tag 
    #  @param commit_date Choice for commit date when building branches
    
    # Prepare - chdir and then make sure we are on the correct branch
    os.chdir(gitrepo)
    prepare_branch_point(branch, parentbranch)            

    # Main loop starts here, with one pass for each tag file we are processing
    for tag_file in tag_files:
        with open(tag_file) as tag_file_fh:
            release_data = json.load(tag_file_fh)

        tag_list = get_current_git_tags(gitrepo)
        current_release_tags = get_current_release_tag_dict(tag_list, branch) # Markers for which packages have been processed
        logger.info("Processing release {0} ({1} current tags)".format(release_data["release"]["name"], len(current_release_tags)))
        release_tag = git_release_tag(release_data["release"], branch)
        if release_tag in tag_list and not skipreleasetag:
            logger.info("Release tag {0} already made - skipping".format(release_tag))
            continue

        if commit_date == "release":
            logger.info("Setting committer date to {0:.0f}".format(release_data["release"]["timestamp"]))
            os.environ["GIT_COMMITTER_DATE"] = "{0:.0f}".format(release_data["release"]["timestamp"])
        
        # Find which packages need updated for this new tag content file
        import_list, packages_considered = find_packages_for_update(release_data, tag_list, branch, 
                                                                    svn_metadata_cache, current_release_tags, only_forward)

        ## Sort the list of tags to be imported by SVN revision number for a
        #  more or less sensible package by package commit history
        sorted_import_revisions = import_list.keys()
        sorted_import_revisions.sort(cmp=lambda x,y: cmp(int(x), int(y)))

        ## Now loop over all the packages we have to import and update them
        pkg_processed = 0
        for revision in sorted_import_revisions:
            for pkg_import in import_list[revision]:
                pkg_processed += 1
                do_package_import(pkg_import, svn_metadata_cache, author_metadata_cache, release_name=release_data["release"]["name"], 
                                  branch=branch, dryrun=dryrun, commit_date=commit_date)
                logger.info("Processed {0}/{1} revisions".format(pkg_processed, len(import_list)))


        ## After all packages are updated, look for packages which present in the last
        #  release, but not this one, so they need to be removed
        new_current_release_tags = get_current_release_tag_dict(tag_list, branch) # Updated package list after upgrade
        packages_to_remove = []
        packages_to_revert = {}
        for package_name, old_package_state in current_release_tags.iteritems():
            if package_name in packages_considered:
                logger.debug("Package {0} was processed for {1}".format(package_name, release_data["release"]["name"]))
                continue
            ## @note We have a package that was not "considered" in the current release.
            #  If we don't have a baserelease then this has been removed, so we zap it.
            #  If there is a baserelease... 
            #   ...and this package is not in it, it was in the cache, then was removed, so zap it.
            #   ...and this package is in it, then compare the versions and "revert" to the base 
            #      release version if they are different.
            if baserelease:
                base_package_version = None
                for package, base_package_data in baserelease["tags"].iteritems():
                    if base_package_data["package_name"] == package_name:
                        base_package_version = base_package_data
                        break
                if base_package_version:
                    if base_package_version["svn_tag"] == old_package_state["svn_tag"]:
                        logger.debug("Package {0} remains at base release version {1}".format(base_package_data["package_name"],
                                                                                              base_package_version["svn_tag"]))
                        packages_considered.append(package_name) # Flag we dealt this package
                    else:
                        logger.info("Package {0} was removed from cache - reverting to base "
                                    "release version {1}".format(base_package_data["package_name"],
                                                     base_package_version["svn_tag"]))
                        package_name = base_package_data["package_name"]
                        svn_meta_tag_key = os.path.join("tags", base_package_version["svn_tag"])
                        svn_revision = svn_metadata_cache[package_name]["svn"][svn_meta_tag_key].keys()[0]
                        git_import_tag = get_flattened_git_tag(package, base_package_version["svn_tag"], svn_revision)
                        packages_to_revert[package_name] = {"package": package,
                                                            "package_name": os.path.basename(package),
                                                            "git_import_tag": get_flattened_git_tag(package, base_package_version["svn_tag"], svn_revision),
                                                            "svn_tag": base_package_version["svn_tag"],
                                                            "svn_revision": svn_revision,
                                                            "branch_import_tag": get_flattened_git_tag(package, base_package_version["svn_tag"], svn_revision, branch),
                                                            "svn_meta_tag_key": svn_meta_tag_key,
                                                            "current_branch_import_tag": current_release_tags[package_name]["git_tag"]}
                else:
                    logger.info("Package {0} was removed from the cache and is not in the base release".format(package_name))
                    packages_to_remove.append(package_name)
            else:
                logger.info("Package {0} has been removed from the release".format(package_name))
                packages_to_remove.append(package_name)

        if baserelease:
            logger.info("{0} packages have been reverted to their base SVN state".format(len(packages_to_revert)))
            for package_name, revert_data in packages_to_revert.iteritems():
                do_package_import(revert_data, svn_metadata_cache, author_metadata_cache, release_name=release_data["release"]["name"], 
                                  branch=branch, dryrun=dryrun, commit_date=commit_date)

        logger.info("{0} packages have been removed from the release".format(len(packages_to_remove)))
        for package in packages_to_remove:
            logger.info("Removing {0} from {1}".format(package, branch))
            if not dryrun:
                package_path = os.path.join(svn_metadata_cache[package]["path"], package)
                recursive_delete(package_path)
            check_output_with_retry(("git", "add", "-A"), dryrun=dryrun)
            cmd = ["git", "commit", "--allow-empty", "-m", "{0} deleted from {1}".format(package_path, branch)]
            check_output_with_retry(cmd, dryrun=dryrun)
            check_output_with_retry(("git", "tag", "-d", current_release_tags[package]["git_tag"]), retries=1, dryrun=dryrun)
            pkg_processed += 1

        ## Now, finally, tag the release as done
        if not skipreleasetag:
            if release_data["release"]["nightly"]:
                check_output_with_retry(("git", "tag", release_tag), retries=1, dryrun=dryrun)
            else:
                check_output_with_retry(("git", "tag", release_tag, "-a",
                                         "-m", "Tagging release {0}".format(release_data["release"]["name"])), 
                                        retries=1, dryrun=dryrun)
            logger.info("Tagged release {0} as {1} "
                        "({2} packages processed)".format(release_data["release"]["name"],
                                                          release_tag, pkg_processed))
        else:
            logger.info("Processed release {0} (no tag; {1} packages processed)".format(release_data["release"]["name"], pkg_processed))
def do_package_import(pkg_import, svn_metadata_cache, author_metadata_cache, release_name="unknown", branch="unknown", 
                      dryrun=False, commit_date="now"):
    ## @brief Import a package's SVN tag onto the current git branch
    #  updating the corresponding git tags
    #  @param pkg_import package import dictionary (see find_packages_for_update for the
    #  structure)
    #  @param svn_metadata_cache The standard metadata cache from SVN
    #  @param author_metadata_cache Cached author data
    #  @param release_name Name of current release being built (used only for generating log messages)
    #  @param branch Current branch name (used only for generating log messages)
    #  @param dryrun Boolean, if @c true then don't actually act
    #  @param commit_date Choices for setting committer date 
    logger.info("Migrating {0} from {1} to {2} for {3}...".format(pkg_import["package"], 
                                                          pkg_import["current_branch_import_tag"], 
                                                          pkg_import["svn_tag"], release_name))
    # Need to wipe out all contents in case files were removed from package
    if not dryrun:
        recursive_delete(pkg_import["package"])
    check_output_with_retry(("git", "checkout", pkg_import["git_import_tag"], pkg_import["package"]), dryrun=dryrun)
    # Splat Changelog file - we do not want these on the production branches
    try:
        os.remove(os.path.join(pkg_import["package"], "ChangeLog"))
    except OSError:
        pass
    # Done - now commit and tag
    if logger.level <= logging.DEBUG:
        cmd = ["git", "status"]
        logger.debug(check_output_with_retry(cmd))
    check_output_with_retry(("git", "add", "-A", pkg_import["package"]), dryrun=dryrun)
    staged = check_output_with_retry(("git", "diff", "--name-only", "--staged"), dryrun=dryrun)
    if len(staged) == 0 and (not dryrun): 
        # Nothing staged, so skip doing any commit, but do make the import tag for this branch
        # so that we don't repeat this step again
        logger.warning("Package {0} - no changes staged for {1}, " 
                       "git tagging and skipping commit".format(pkg_import["package"], release_name))
        check_output_with_retry(("git", "tag", pkg_import["branch_import_tag"]), retries=1, dryrun=dryrun)
        return

    rev_meta = svn_metadata_cache[pkg_import["package_name"]]["svn"][pkg_import["svn_meta_tag_key"]][pkg_import["svn_revision"]]
    msg = rev_meta["msg"]
    if pkg_import["svn_tag"] == "trunk":
        msg += " (trunk r{0})".format(rev_meta["revision"])
    else:
        msg += " ({0})".format(pkg_import["svn_tag"])
    cl_diff = changelog_diff(pkg_import["package"],
                             from_tag="/".join(pkg_import["current_branch_import_tag"].split("/")[1:]) if pkg_import["current_branch_import_tag"] else None,
                             to_tag=pkg_import["git_import_tag"])
    if cl_diff:
        msg += "\n\n" + "\n".join(cl_diff)
    cmd = ["git", "commit", "-m", msg]
    author = author_string(rev_meta["author"], author_metadata_cache)
    cmd.append("--author='{0}'".format(author))
    cmd.append("--date={0}".format(rev_meta["date"]))
    
    if commit_date == "author":
        os.environ["GIT_COMMITTER_DATE"] = rev_meta["date"]
    check_output_with_retry(cmd, retries=1, dryrun=dryrun)
    if commit_date == "author":
        del os.environ["GIT_COMMITTER_DATE"]
    
    check_output_with_retry(("git", "tag", pkg_import["branch_import_tag"]), retries=1, dryrun=dryrun)
    if pkg_import["current_branch_import_tag"]:
        check_output_with_retry(("git", "tag", "-d", pkg_import["current_branch_import_tag"]), retries=1, dryrun=dryrun)
    logger.info("Committed {0} ({1}) onto {2} for {3}".format(pkg_import["package"], 
                                                              pkg_import["svn_tag"], branch, release_name))
示例#8
0
def svn_co_tag_and_commit(svnroot, gitrepo, package, tag, svn_metadata=None, author_metadata_cache=None, branch=None,
                          svn_path_accept=[], svn_path_reject=[], package_veto=[], commit=True, revision=None,
                          license_text=None, license_path_accept=[], license_path_reject=[],
                          uncrustify_config=None, uncrustify_path_accept=[], uncrustify_path_reject=[]):
    ## @brief Make a temporary space, check out from svn, clean-up, copy and then git commit and tag
    #  @param svnroot Base path to SVN repository
    #  @param gitrepo Path to git repository to import to
    #  @param package Path to package root (in git and svn)
    #  @param tag Package tag to import (i.e., path after base package path)
    #  @param svn_metadata SVN metadata cache
    #  @param author_metadata_cache Author name/email cache
    #  @param branch Git branch to switch to before import
    #  @param svn_path_accept Paths to force import to git
    #  @param svn_path_reject Paths to force reject from the import
    #  @param package_veto List of packages to just plain refuse to handle
    #  @param commit Boolean flag to manage commit (can be set to @c False to only checkout and process)
    #  @param license_text List of strings containing the license text to add (if @c False, then no
    #  license file is added)
    #  @param revision Force SVN revision number (useful for svnpull.py, where
    #  no svn metadata is available)
    #  @param license_path_accept Paths to force include in license file addition
    #  @param license_path_reject Paths to exclude from license file addition
    #  @param uncrustify_config Uncrustify configuration file
    #  @param uncrustify_path_accept Paths to force uncrustify to run on
    #  @param uncrustify_path_reject Paths to exclude from uncrustify
    if package in package_veto:
        logger.info("Package {0} is vetoed - skipping import".format(package))
        return

    msg = "Importing SVN path {0}/{1} to {0}".format(package, tag)
    if svn_metadata and tag == "trunk":
        msg += " (r{0})".format(svn_metadata["revision"])
    logger.info(msg)

    if branch:
        logger.info("Switching to branch {0}".format(branch))
        switch_to_branch(args.targetbranch)

    tempdir = tempfile.mkdtemp()
    full_svn_path = os.path.join(tempdir, package)
    cmd = ["svn", "checkout"]
    if revision:
        cmd.extend(["-r", str(revision)])
    elif svn_metadata:
        cmd.extend(["-r", svn_metadata["revision"]])
    cmd.extend([os.path.join(svnroot, package, tag), os.path.join(tempdir, package)])
    check_output_with_retry(cmd, retries=1, wait=3)

    # Clean out directory of things we don't want to import
    svn_cleanup(full_svn_path, svn_co_root=tempdir,
                svn_path_accept=svn_path_accept, svn_path_reject=svn_path_reject)
    
    # If desired, inject a licence into the source code
    if license_text:
        svn_license_injector(full_svn_path, svn_co_root=tempdir, license_text=license_text,
                             license_path_accept=license_path_accept, license_path_reject=license_path_reject)

    # Pass C++ sources through uncrustify
    if uncrustify_config:
        uncrustify_sources(full_svn_path, svn_co_root=tempdir, uncrustify_config=uncrustify_config,
                           uncrustify_path_accept=uncrustify_path_accept, uncrustify_path_reject=uncrustify_path_reject)

    # Copy to git
    full_git_path = os.path.join(gitrepo, package)
    package_root, package_name = os.path.split(full_git_path)
    try:
        if os.path.isdir(full_git_path):
            shutil.rmtree(full_git_path, ignore_errors=True)
        os.makedirs(package_root)
    except OSError:
        pass
    shutil.move(full_svn_path, package_root)

    if commit:
        # get ChangeLog diff
        cl_diff = changelog_diff(package)

        # Commit
        check_output_with_retry(("git", "add", "-A", package))
        if logger.level <= logging.DEBUG:
            logger.debug(check_output_with_retry(("git", "status")))


        cmd = ["git", "commit", "--allow-empty", "-m", "{0} ({1} - r{2})".
               format(svn_metadata['msg'], tag.replace('tags/','',1), svn_metadata['revision'])]
        if svn_metadata:
            cmd.extend(("--author='{0}'".format(author_string(svn_metadata["author"], author_metadata_cache)),
                        "--date={0}".format(svn_metadata["date"])))
        if cl_diff:
            cmd.extend(("-m", "Diff in ChangeLog:\n" + '\n'.join(cl_diff)))
        check_output_with_retry(cmd)
        cmd = ["git", "tag", "-a", get_flattened_git_tag(package, tag, svn_metadata["revision"]), "-m", ""]
        check_output_with_retry(cmd)

    # Clean up
    shutil.rmtree(tempdir)
示例#9
0
def main():
    parser = argparse.ArgumentParser(
        description='SVN to git migrator, ATLAS style')
    parser.add_argument('svnroot',
                        metavar='SVNDIR',
                        help="Location of svn repository root")
    parser.add_argument('gitrepo',
                        metavar='GITDIR',
                        help="Location of git repository")
    parser.add_argument(
        'tagfiles',
        nargs="+",
        metavar='TAGFILE',
        help=
        "List of release tag content files to process - all tags found in these files will "
        "be imported (any already imported tags will be skipped)")
    parser.add_argument(
        '--targetbranch',
        default="package",
        help=
        "Target git branch for import. Default is the special value 'package' in which "
        "each package is imported onto its own branch")
    parser.add_argument(
        '--svnpath',
        metavar='PATH',
        nargs='+',
        default=[],
        help="Restrict actions to this list of paths in the SVN tree (use to "
        "make small scale tests of the import workflow).")
    parser.add_argument(
        '--intermediatetags',
        action="store_true",
        help=
        "Import all tags from oldest release tag found, instead of just release tags"
    )
    parser.add_argument(
        '--processtrunk',
        action="store_true",
        help=
        "Update trunk versions during the import (False by default, the trunk will be skipped)."
    )
    parser.add_argument(
        '--svncachefile',
        metavar='FILE',
        help=
        "File containing cache of SVN information - default '[gitrepo].svn.metadata'"
    )
    parser.add_argument(
        '--authorcachefile',
        metavar='FILE',
        help=
        "File containing cache of author name and email information - default '[gitrepo].author.metadata'"
    )
    parser.add_argument(
        '--importtimingfile',
        metavar="FILE",
        help=
        "File to dump SVN->git import timing information - default '[gitrepo]-timing.json'"
    )
    parser.add_argument(
        '--svnfilterexceptions',
        '--sfe',
        metavar="FILE",
        help=
        "File listing path globs to exempt from SVN import filter (lines with '+PATH') or "
        "to always reject (lines with '-PATH'); default %(default)s. Use NONE to have no exceptions.",
        default=os.path.join(os.path.dirname(os.path.abspath(__file__)),
                             "atlasoffline-exceptions.txt"))
    parser.add_argument(
        '--packageveto',
        metavar="FILE",
        help="File listing packages that will be skipped completely on import.",
        default=os.path.join(os.path.dirname(os.path.abspath(__file__)),
                             "atlaspackage-exceptions.txt"))
    parser.add_argument(
        '--licensefile',
        metavar="FILE",
        help="License file to add to source code files (default "
        "is to add %(default)s license file)",
        default=os.path.join(os.path.dirname(os.path.abspath(__file__)),
                             "cerncopy.txt"))
    parser.add_argument(
        '--licenseexceptions',
        metavar="FILE",
        help="File listing path globs to exempt from or  "
        "always apply license file to (same format as --svnfilterexceptions)",
        default=os.path.join(os.path.dirname(os.path.abspath(__file__)),
                             "atlaslicense-exceptions.txt"))
    parser.add_argument(
        '--uncrustify',
        metavar="FILE",
        help="Uncrustify configuration file to use to process C++ "
        "sources through before git import (by default uncrustify will not be used)"
    )
    parser.add_argument(
        '--uncrustifyexceptions',
        metavar="FILE",
        help="File listing path globs to exempt from or  "
        "always apply uncrustify to (same format as --svnfilterexceptions)",
        default=os.path.join(os.path.dirname(os.path.abspath(__file__)),
                             "atlasuncrustify-exceptions.txt"))
    parser.add_argument('--debug',
                        '--verbose',
                        "-v",
                        action="store_true",
                        help="Switch logging into DEBUG mode")

    # Parse and handle initial arguments
    args = parser.parse_args()
    if args.debug:
        logger.setLevel(logging.DEBUG)

    # Massage default values
    if not args.svncachefile:
        args.svncachefile = os.path.basename(args.gitrepo) + ".svn.metadata"
    if not args.authorcachefile:
        args.authorcachefile = os.path.basename(
            args.gitrepo) + ".author.metadata"
    if not args.importtimingfile:
        args.importtimingfile = os.path.basename(args.gitrepo) + "-timing.json"

    # Set svnroot and git repo, get some starting values
    svnroot = args.svnroot
    gitrepo = os.path.abspath(args.gitrepo)
    start_cwd = os.getcwd()
    start_timestamp_string = time.strftime("%Y%m%dT%H%M.%S")
    logger.debug("Set SVN root to {0} and git repo to {1}".format(
        svnroot, gitrepo))

    # Load exception globs
    svn_path_accept, svn_path_reject = load_exceptions_file(
        args.svnfilterexceptions)

    # Load package vetos
    if args.packageveto:
        package_veto = load_package_veto(args.packageveto)
    else:
        package_veto = []

    # License file loading
    if args.licensefile:
        with open(args.licensefile) as lfh:
            license_text = [line.rstrip() for line in lfh.readlines()]
    else:
        license_text = None
    if args.licenseexceptions:
        license_path_accept, license_path_reject = load_exceptions_file(
            args.licenseexceptions)
    else:
        license_path_accept = license_path_reject = []

    # Uncrustify exceptions file
    if args.uncrustify:
        uncrustify_path_accept, uncrustify_path_reject = load_exceptions_file(
            args.uncrustifyexceptions)
    else:
        uncrustify_path_accept = uncrustify_path_reject = []

    ### Main actions start here
    # Setup the git repository
    init_git(gitrepo)
    # Pull current list of tags here, to fast skip any work already done
    if args.targetbranch != "package":
        switch_to_branch(args.targetbranch, orphan=True)
    current_git_tags = get_current_git_tags(gitrepo)
    os.chdir(start_cwd)

    ## SVN interactions and reloading state
    # Decide which svn packages we will import
    # Note that if we're pulling the packages from a tag diff file, we also get tags
    # at this point, otherwise the tag list is empty.
    svn_packages = get_tags(args.tagfiles, args.svnpath)
    # Add "trunk" packages, if required
    if args.processtrunk:
        for package, tags in svn_packages.iteritems():
            if "trunk" not in tags:
                tags.append("trunk")

    # Initialise SVN and author metadata cache with any stored values
    svn_metadata_cache = initialise_metadata(args.svncachefile)
    author_metadata_cache = initialise_metadata(args.authorcachefile)

    # Prepare package import
    scan_svn_tags_and_get_metadata(svnroot,
                                   svn_packages,
                                   svn_metadata_cache,
                                   author_metadata_cache,
                                   args.intermediatetags,
                                   package_veto=package_veto)

    # Now presistify metadata cache
    backup_metadata(svn_metadata_cache, start_cwd, args.svncachefile,
                    start_timestamp_string)
    backup_metadata(author_metadata_cache, start_cwd, args.authorcachefile,
                    start_timestamp_string)

    # Setup dictionary for keying by SVN revision number
    svn_cache_revision_dict = svn_cache_revision_dict_init(svn_metadata_cache)

    ## git processing actions
    # Process each SVN tag in order
    ordered_revisions = svn_cache_revision_dict.keys()
    ordered_revisions.sort(cmp=lambda x, y: cmp(int(x), int(y)))
    logger.info("Will process {0} SVN revisions in total".format(
        len(ordered_revisions)))
    counter = 0
    processed_tags = 0
    timing = []
    os.chdir(gitrepo)

    for rev in ordered_revisions:
        counter += 1
        start = time.time()
        logger.info("SVN Revsion {0} ({1} of {2})".format(
            rev, counter, len(ordered_revisions)))
        for pkg_tag in svn_cache_revision_dict[rev]:
            if get_flattened_git_tag(pkg_tag["package"], pkg_tag["tag"],
                                     rev) in current_git_tags:
                logger.info("Tag {0} exists already - skipping".format(
                    os.path.join(pkg_tag["package"], pkg_tag["tag"])))
                continue
            if args.targetbranch == "package":
                switch_to_branch(os.path.basename(pkg_tag["package"]),
                                 orphan=True)
            svn_co_tag_and_commit(
                svnroot,
                gitrepo,
                pkg_tag["package"],
                pkg_tag["tag"],
                svn_metadata_cache[os.path.basename(
                    pkg_tag["package"])]["svn"][pkg_tag["tag"]][rev],
                author_metadata_cache,
                svn_path_accept=svn_path_accept,
                svn_path_reject=svn_path_reject,
                package_veto=package_veto,
                license_text=license_text,
                license_path_accept=license_path_accept,
                license_path_reject=license_path_reject,
                uncrustify_config=args.uncrustify,
                uncrustify_path_accept=uncrustify_path_accept,
                uncrustify_path_reject=uncrustify_path_reject)
            processed_tags += 1
        elapsed = time.time() - start
        logger.info(
            "{0} processed in {1}s ({2} packages really processed)".format(
                counter, elapsed, processed_tags))
        timing.append(elapsed)

    # Last task, clean all empty directories (git does not track these, but they are clutter)
    check_output_with_retry(("git", "clean", "-f", "-d"))

    if args.importtimingfile:
        os.chdir(start_cwd)
        with open(args.importtimingfile, "w") as time_file:
            json.dump(timing, time_file)