def uncrustify_sources(svn_path, svn_co_root, uncrustify_config, uncrustify_path_accept, uncrustify_path_reject): ## @brief Uncrustify code before import # @param svn_path Filesystem path to parsed up SVN checkout # @param svn_co_root Base directory of SVN checkout # @param uncrustify_config Config file for uncrustify pass # @param uncrustify_path_accept Paths to force # @param uncrustify_path_reject Paths to exclude for root, dirs, files in os.walk(svn_path): for name in files: filename = os.path.join(root, name) svn_filename = filename[len(svn_co_root) + 1:] path_veto = False for filter in uncrustify_path_reject: if re.match(filter, svn_filename): logger.debug("File {0} will not go through uncrustify".format(svn_filename, filter.pattern)) path_veto = True break for filter in uncrustify_path_accept: if re.match(filter, svn_filename): logger.debug("File {0} will be passed to uncrustify".format(svn_filename, filter.pattern)) path_veto = False break if path_veto: continue extension = filename.rsplit(".", 1)[1] if "." in filename else "" if extension in ("cxx", "cpp", "icc", "cc", "c", "C", "h", "hpp", "hh"): logger.debug("Uncrustifying {0}".format(filename)) cmd = ("uncrustify", "-c", uncrustify_config, "--no-backup", "-l", "CPP", filename) # We do not consider uncrustify errors as fatal for the import... this can # happen because of a source file issue or picking the wrong language try: check_output_with_retry(cmd, retries=0) except RuntimeError: logger.warning("Uncrustify failed on {0}".format(filename))
def init_git(gitrepo): ## @brief Initialise git repo, if needed # @param gitrepo Git repository path if not os.path.exists(gitrepo): os.makedirs(gitrepo) os.chdir(gitrepo) if os.path.exists(os.path.join(gitrepo, ".git")): logger.info("Found existing git repo, {0}".format(gitrepo)) check_output_with_retry(("git", "reset", "--hard")) else: logger.info("Initialising git repo: {0}".format(gitrepo)) check_output_with_retry(("git", "init"))
def svn_get_path_metadata(svnroot, package, package_path, revision=None): # # @brief Get SVN metadata and return as a simple dictionary keyed on date, author and commit revision logger.info("Querying SVN metadata for {0}".format(os.path.join(package, package_path))) cmd = ["svn", "info", os.path.join(svnroot, package, package_path), "--xml"] svn_info = check_output_with_retry(cmd) tree = eltree.fromstring(svn_info) info = {"date": tree.find(".//date").text.rsplit(".",1)[0], # Strip off sub-second part "author": tree.find(".//author").text, "revision": tree.find(".//commit").attrib['revision']} cmd = ["svn", "log", os.path.join(svnroot, package, package_path), "-r", info["revision"], "--xml"] svn_log = check_output_with_retry(cmd) tree = eltree.fromstring(svn_log) info["msg"] = tree.find(".//msg").text.strip().encode('ascii', 'ignore') return info
def author_info_lookup(author_name): try: cmd = ["phonebook", "--login", author_name, "--terse", "firstname", "--terse", "surname", "--terse", "email"] author_info = check_output_with_retry(cmd, retries=1).strip().split(";") return {"name": " ".join(author_info[:2]), "email": author_info[2]} except IndexError: raise RuntimeError("Had a problem decoding phonebook info for '{0}'".format(author_name))
def prepare_branch_point(branch, parentbranch=None): ## @brief Using information about the target branch and any parent # switch to the correct point in history to start/continue # @param branch Target branch name # @param parentbranch If creating a new branch, this is the @c BRANCH:COMMIT_ID of # where to make the new branch from; syntax @c BRANCH:@FILE and @c BRANCH:@TIMESTAMP # is also supported, where the timestamp will be used to find the branch directly # (and can be taken from @c JSON release data in @c FILE) if not parentbranch or branch_exists(branch): logger.info("Switching to branch {0}".format(branch)) switch_to_branch(branch, orphan=True) else: parent, commit = parentbranch.split(":") check_output_with_retry(("git", "checkout", parent), retries=1) # needed? if commit.startswith("@"): timestamp = commit[1:] # If this maps to a file, try to open it as a release JSON, otherwise treat it as # a plain timestamp if os.access(timestamp, os.F_OK): logger.info("Taking branching timestamp from file {0}".format(timestamp)) with open(timestamp) as fh: branch_point_release = json.load(fh) timestamp = branch_point_release["release"]["timestamp"] logger.info("Using timestamp {0} for branch point".format(timestamp)) commit = check_output_with_retry(["git", "log", "--until", str(timestamp), "-n1", "--pretty=format:%H"], retries=1).strip() logger.info("Mapped timestamp {0} to commit {1}".format(timestamp, commit)) check_output_with_retry(("git", "checkout", commit), retries=1) check_output_with_retry(("git", "checkout", "-b", branch), retries=1)
def branch_builder(gitrepo, branch, tag_files, svn_metadata_cache, author_metadata_cache, parentbranch=None, baserelease=None, skipreleasetag=False, dryrun=False, only_forward=False, commit_date="now"): ## @brief Main branch builder function # @param gitrepo The git repository location # @param branch The git branch to work on # @param tag_files The plain tag content files to process # @param svn_metadata_cache The standard metadata cache from SVN # @param author_metadata_cache Cached author data # @param parentbranch If creating a new branch, this is the BRANCH:COMMIT_ID of where to make the new branch from # @param skipreleasetag If @c True then skip creating git tags for each processed release # @param dryrun If @c True, do nothing except print commands that would have been executed # @param only_forward If @c True then never revert a package to a previous version or import a branch tag # @param commit_date Choice for commit date when building branches # Prepare - chdir and then make sure we are on the correct branch os.chdir(gitrepo) prepare_branch_point(branch, parentbranch) # Main loop starts here, with one pass for each tag file we are processing for tag_file in tag_files: with open(tag_file) as tag_file_fh: release_data = json.load(tag_file_fh) tag_list = get_current_git_tags(gitrepo) current_release_tags = get_current_release_tag_dict(tag_list, branch) # Markers for which packages have been processed logger.info("Processing release {0} ({1} current tags)".format(release_data["release"]["name"], len(current_release_tags))) release_tag = git_release_tag(release_data["release"], branch) if release_tag in tag_list and not skipreleasetag: logger.info("Release tag {0} already made - skipping".format(release_tag)) continue if commit_date == "release": logger.info("Setting committer date to {0:.0f}".format(release_data["release"]["timestamp"])) os.environ["GIT_COMMITTER_DATE"] = "{0:.0f}".format(release_data["release"]["timestamp"]) # Find which packages need updated for this new tag content file import_list, packages_considered = find_packages_for_update(release_data, tag_list, branch, svn_metadata_cache, current_release_tags, only_forward) ## Sort the list of tags to be imported by SVN revision number for a # more or less sensible package by package commit history sorted_import_revisions = import_list.keys() sorted_import_revisions.sort(cmp=lambda x,y: cmp(int(x), int(y))) ## Now loop over all the packages we have to import and update them pkg_processed = 0 for revision in sorted_import_revisions: for pkg_import in import_list[revision]: pkg_processed += 1 do_package_import(pkg_import, svn_metadata_cache, author_metadata_cache, release_name=release_data["release"]["name"], branch=branch, dryrun=dryrun, commit_date=commit_date) logger.info("Processed {0}/{1} revisions".format(pkg_processed, len(import_list))) ## After all packages are updated, look for packages which present in the last # release, but not this one, so they need to be removed new_current_release_tags = get_current_release_tag_dict(tag_list, branch) # Updated package list after upgrade packages_to_remove = [] packages_to_revert = {} for package_name, old_package_state in current_release_tags.iteritems(): if package_name in packages_considered: logger.debug("Package {0} was processed for {1}".format(package_name, release_data["release"]["name"])) continue ## @note We have a package that was not "considered" in the current release. # If we don't have a baserelease then this has been removed, so we zap it. # If there is a baserelease... # ...and this package is not in it, it was in the cache, then was removed, so zap it. # ...and this package is in it, then compare the versions and "revert" to the base # release version if they are different. if baserelease: base_package_version = None for package, base_package_data in baserelease["tags"].iteritems(): if base_package_data["package_name"] == package_name: base_package_version = base_package_data break if base_package_version: if base_package_version["svn_tag"] == old_package_state["svn_tag"]: logger.debug("Package {0} remains at base release version {1}".format(base_package_data["package_name"], base_package_version["svn_tag"])) packages_considered.append(package_name) # Flag we dealt this package else: logger.info("Package {0} was removed from cache - reverting to base " "release version {1}".format(base_package_data["package_name"], base_package_version["svn_tag"])) package_name = base_package_data["package_name"] svn_meta_tag_key = os.path.join("tags", base_package_version["svn_tag"]) svn_revision = svn_metadata_cache[package_name]["svn"][svn_meta_tag_key].keys()[0] git_import_tag = get_flattened_git_tag(package, base_package_version["svn_tag"], svn_revision) packages_to_revert[package_name] = {"package": package, "package_name": os.path.basename(package), "git_import_tag": get_flattened_git_tag(package, base_package_version["svn_tag"], svn_revision), "svn_tag": base_package_version["svn_tag"], "svn_revision": svn_revision, "branch_import_tag": get_flattened_git_tag(package, base_package_version["svn_tag"], svn_revision, branch), "svn_meta_tag_key": svn_meta_tag_key, "current_branch_import_tag": current_release_tags[package_name]["git_tag"]} else: logger.info("Package {0} was removed from the cache and is not in the base release".format(package_name)) packages_to_remove.append(package_name) else: logger.info("Package {0} has been removed from the release".format(package_name)) packages_to_remove.append(package_name) if baserelease: logger.info("{0} packages have been reverted to their base SVN state".format(len(packages_to_revert))) for package_name, revert_data in packages_to_revert.iteritems(): do_package_import(revert_data, svn_metadata_cache, author_metadata_cache, release_name=release_data["release"]["name"], branch=branch, dryrun=dryrun, commit_date=commit_date) logger.info("{0} packages have been removed from the release".format(len(packages_to_remove))) for package in packages_to_remove: logger.info("Removing {0} from {1}".format(package, branch)) if not dryrun: package_path = os.path.join(svn_metadata_cache[package]["path"], package) recursive_delete(package_path) check_output_with_retry(("git", "add", "-A"), dryrun=dryrun) cmd = ["git", "commit", "--allow-empty", "-m", "{0} deleted from {1}".format(package_path, branch)] check_output_with_retry(cmd, dryrun=dryrun) check_output_with_retry(("git", "tag", "-d", current_release_tags[package]["git_tag"]), retries=1, dryrun=dryrun) pkg_processed += 1 ## Now, finally, tag the release as done if not skipreleasetag: if release_data["release"]["nightly"]: check_output_with_retry(("git", "tag", release_tag), retries=1, dryrun=dryrun) else: check_output_with_retry(("git", "tag", release_tag, "-a", "-m", "Tagging release {0}".format(release_data["release"]["name"])), retries=1, dryrun=dryrun) logger.info("Tagged release {0} as {1} " "({2} packages processed)".format(release_data["release"]["name"], release_tag, pkg_processed)) else: logger.info("Processed release {0} (no tag; {1} packages processed)".format(release_data["release"]["name"], pkg_processed))
def do_package_import(pkg_import, svn_metadata_cache, author_metadata_cache, release_name="unknown", branch="unknown", dryrun=False, commit_date="now"): ## @brief Import a package's SVN tag onto the current git branch # updating the corresponding git tags # @param pkg_import package import dictionary (see find_packages_for_update for the # structure) # @param svn_metadata_cache The standard metadata cache from SVN # @param author_metadata_cache Cached author data # @param release_name Name of current release being built (used only for generating log messages) # @param branch Current branch name (used only for generating log messages) # @param dryrun Boolean, if @c true then don't actually act # @param commit_date Choices for setting committer date logger.info("Migrating {0} from {1} to {2} for {3}...".format(pkg_import["package"], pkg_import["current_branch_import_tag"], pkg_import["svn_tag"], release_name)) # Need to wipe out all contents in case files were removed from package if not dryrun: recursive_delete(pkg_import["package"]) check_output_with_retry(("git", "checkout", pkg_import["git_import_tag"], pkg_import["package"]), dryrun=dryrun) # Splat Changelog file - we do not want these on the production branches try: os.remove(os.path.join(pkg_import["package"], "ChangeLog")) except OSError: pass # Done - now commit and tag if logger.level <= logging.DEBUG: cmd = ["git", "status"] logger.debug(check_output_with_retry(cmd)) check_output_with_retry(("git", "add", "-A", pkg_import["package"]), dryrun=dryrun) staged = check_output_with_retry(("git", "diff", "--name-only", "--staged"), dryrun=dryrun) if len(staged) == 0 and (not dryrun): # Nothing staged, so skip doing any commit, but do make the import tag for this branch # so that we don't repeat this step again logger.warning("Package {0} - no changes staged for {1}, " "git tagging and skipping commit".format(pkg_import["package"], release_name)) check_output_with_retry(("git", "tag", pkg_import["branch_import_tag"]), retries=1, dryrun=dryrun) return rev_meta = svn_metadata_cache[pkg_import["package_name"]]["svn"][pkg_import["svn_meta_tag_key"]][pkg_import["svn_revision"]] msg = rev_meta["msg"] if pkg_import["svn_tag"] == "trunk": msg += " (trunk r{0})".format(rev_meta["revision"]) else: msg += " ({0})".format(pkg_import["svn_tag"]) cl_diff = changelog_diff(pkg_import["package"], from_tag="/".join(pkg_import["current_branch_import_tag"].split("/")[1:]) if pkg_import["current_branch_import_tag"] else None, to_tag=pkg_import["git_import_tag"]) if cl_diff: msg += "\n\n" + "\n".join(cl_diff) cmd = ["git", "commit", "-m", msg] author = author_string(rev_meta["author"], author_metadata_cache) cmd.append("--author='{0}'".format(author)) cmd.append("--date={0}".format(rev_meta["date"])) if commit_date == "author": os.environ["GIT_COMMITTER_DATE"] = rev_meta["date"] check_output_with_retry(cmd, retries=1, dryrun=dryrun) if commit_date == "author": del os.environ["GIT_COMMITTER_DATE"] check_output_with_retry(("git", "tag", pkg_import["branch_import_tag"]), retries=1, dryrun=dryrun) if pkg_import["current_branch_import_tag"]: check_output_with_retry(("git", "tag", "-d", pkg_import["current_branch_import_tag"]), retries=1, dryrun=dryrun) logger.info("Committed {0} ({1}) onto {2} for {3}".format(pkg_import["package"], pkg_import["svn_tag"], branch, release_name))
def svn_co_tag_and_commit(svnroot, gitrepo, package, tag, svn_metadata=None, author_metadata_cache=None, branch=None, svn_path_accept=[], svn_path_reject=[], package_veto=[], commit=True, revision=None, license_text=None, license_path_accept=[], license_path_reject=[], uncrustify_config=None, uncrustify_path_accept=[], uncrustify_path_reject=[]): ## @brief Make a temporary space, check out from svn, clean-up, copy and then git commit and tag # @param svnroot Base path to SVN repository # @param gitrepo Path to git repository to import to # @param package Path to package root (in git and svn) # @param tag Package tag to import (i.e., path after base package path) # @param svn_metadata SVN metadata cache # @param author_metadata_cache Author name/email cache # @param branch Git branch to switch to before import # @param svn_path_accept Paths to force import to git # @param svn_path_reject Paths to force reject from the import # @param package_veto List of packages to just plain refuse to handle # @param commit Boolean flag to manage commit (can be set to @c False to only checkout and process) # @param license_text List of strings containing the license text to add (if @c False, then no # license file is added) # @param revision Force SVN revision number (useful for svnpull.py, where # no svn metadata is available) # @param license_path_accept Paths to force include in license file addition # @param license_path_reject Paths to exclude from license file addition # @param uncrustify_config Uncrustify configuration file # @param uncrustify_path_accept Paths to force uncrustify to run on # @param uncrustify_path_reject Paths to exclude from uncrustify if package in package_veto: logger.info("Package {0} is vetoed - skipping import".format(package)) return msg = "Importing SVN path {0}/{1} to {0}".format(package, tag) if svn_metadata and tag == "trunk": msg += " (r{0})".format(svn_metadata["revision"]) logger.info(msg) if branch: logger.info("Switching to branch {0}".format(branch)) switch_to_branch(args.targetbranch) tempdir = tempfile.mkdtemp() full_svn_path = os.path.join(tempdir, package) cmd = ["svn", "checkout"] if revision: cmd.extend(["-r", str(revision)]) elif svn_metadata: cmd.extend(["-r", svn_metadata["revision"]]) cmd.extend([os.path.join(svnroot, package, tag), os.path.join(tempdir, package)]) check_output_with_retry(cmd, retries=1, wait=3) # Clean out directory of things we don't want to import svn_cleanup(full_svn_path, svn_co_root=tempdir, svn_path_accept=svn_path_accept, svn_path_reject=svn_path_reject) # If desired, inject a licence into the source code if license_text: svn_license_injector(full_svn_path, svn_co_root=tempdir, license_text=license_text, license_path_accept=license_path_accept, license_path_reject=license_path_reject) # Pass C++ sources through uncrustify if uncrustify_config: uncrustify_sources(full_svn_path, svn_co_root=tempdir, uncrustify_config=uncrustify_config, uncrustify_path_accept=uncrustify_path_accept, uncrustify_path_reject=uncrustify_path_reject) # Copy to git full_git_path = os.path.join(gitrepo, package) package_root, package_name = os.path.split(full_git_path) try: if os.path.isdir(full_git_path): shutil.rmtree(full_git_path, ignore_errors=True) os.makedirs(package_root) except OSError: pass shutil.move(full_svn_path, package_root) if commit: # get ChangeLog diff cl_diff = changelog_diff(package) # Commit check_output_with_retry(("git", "add", "-A", package)) if logger.level <= logging.DEBUG: logger.debug(check_output_with_retry(("git", "status"))) cmd = ["git", "commit", "--allow-empty", "-m", "{0} ({1} - r{2})". format(svn_metadata['msg'], tag.replace('tags/','',1), svn_metadata['revision'])] if svn_metadata: cmd.extend(("--author='{0}'".format(author_string(svn_metadata["author"], author_metadata_cache)), "--date={0}".format(svn_metadata["date"]))) if cl_diff: cmd.extend(("-m", "Diff in ChangeLog:\n" + '\n'.join(cl_diff))) check_output_with_retry(cmd) cmd = ["git", "tag", "-a", get_flattened_git_tag(package, tag, svn_metadata["revision"]), "-m", ""] check_output_with_retry(cmd) # Clean up shutil.rmtree(tempdir)
def main(): parser = argparse.ArgumentParser( description='SVN to git migrator, ATLAS style') parser.add_argument('svnroot', metavar='SVNDIR', help="Location of svn repository root") parser.add_argument('gitrepo', metavar='GITDIR', help="Location of git repository") parser.add_argument( 'tagfiles', nargs="+", metavar='TAGFILE', help= "List of release tag content files to process - all tags found in these files will " "be imported (any already imported tags will be skipped)") parser.add_argument( '--targetbranch', default="package", help= "Target git branch for import. Default is the special value 'package' in which " "each package is imported onto its own branch") parser.add_argument( '--svnpath', metavar='PATH', nargs='+', default=[], help="Restrict actions to this list of paths in the SVN tree (use to " "make small scale tests of the import workflow).") parser.add_argument( '--intermediatetags', action="store_true", help= "Import all tags from oldest release tag found, instead of just release tags" ) parser.add_argument( '--processtrunk', action="store_true", help= "Update trunk versions during the import (False by default, the trunk will be skipped)." ) parser.add_argument( '--svncachefile', metavar='FILE', help= "File containing cache of SVN information - default '[gitrepo].svn.metadata'" ) parser.add_argument( '--authorcachefile', metavar='FILE', help= "File containing cache of author name and email information - default '[gitrepo].author.metadata'" ) parser.add_argument( '--importtimingfile', metavar="FILE", help= "File to dump SVN->git import timing information - default '[gitrepo]-timing.json'" ) parser.add_argument( '--svnfilterexceptions', '--sfe', metavar="FILE", help= "File listing path globs to exempt from SVN import filter (lines with '+PATH') or " "to always reject (lines with '-PATH'); default %(default)s. Use NONE to have no exceptions.", default=os.path.join(os.path.dirname(os.path.abspath(__file__)), "atlasoffline-exceptions.txt")) parser.add_argument( '--packageveto', metavar="FILE", help="File listing packages that will be skipped completely on import.", default=os.path.join(os.path.dirname(os.path.abspath(__file__)), "atlaspackage-exceptions.txt")) parser.add_argument( '--licensefile', metavar="FILE", help="License file to add to source code files (default " "is to add %(default)s license file)", default=os.path.join(os.path.dirname(os.path.abspath(__file__)), "cerncopy.txt")) parser.add_argument( '--licenseexceptions', metavar="FILE", help="File listing path globs to exempt from or " "always apply license file to (same format as --svnfilterexceptions)", default=os.path.join(os.path.dirname(os.path.abspath(__file__)), "atlaslicense-exceptions.txt")) parser.add_argument( '--uncrustify', metavar="FILE", help="Uncrustify configuration file to use to process C++ " "sources through before git import (by default uncrustify will not be used)" ) parser.add_argument( '--uncrustifyexceptions', metavar="FILE", help="File listing path globs to exempt from or " "always apply uncrustify to (same format as --svnfilterexceptions)", default=os.path.join(os.path.dirname(os.path.abspath(__file__)), "atlasuncrustify-exceptions.txt")) parser.add_argument('--debug', '--verbose', "-v", action="store_true", help="Switch logging into DEBUG mode") # Parse and handle initial arguments args = parser.parse_args() if args.debug: logger.setLevel(logging.DEBUG) # Massage default values if not args.svncachefile: args.svncachefile = os.path.basename(args.gitrepo) + ".svn.metadata" if not args.authorcachefile: args.authorcachefile = os.path.basename( args.gitrepo) + ".author.metadata" if not args.importtimingfile: args.importtimingfile = os.path.basename(args.gitrepo) + "-timing.json" # Set svnroot and git repo, get some starting values svnroot = args.svnroot gitrepo = os.path.abspath(args.gitrepo) start_cwd = os.getcwd() start_timestamp_string = time.strftime("%Y%m%dT%H%M.%S") logger.debug("Set SVN root to {0} and git repo to {1}".format( svnroot, gitrepo)) # Load exception globs svn_path_accept, svn_path_reject = load_exceptions_file( args.svnfilterexceptions) # Load package vetos if args.packageveto: package_veto = load_package_veto(args.packageveto) else: package_veto = [] # License file loading if args.licensefile: with open(args.licensefile) as lfh: license_text = [line.rstrip() for line in lfh.readlines()] else: license_text = None if args.licenseexceptions: license_path_accept, license_path_reject = load_exceptions_file( args.licenseexceptions) else: license_path_accept = license_path_reject = [] # Uncrustify exceptions file if args.uncrustify: uncrustify_path_accept, uncrustify_path_reject = load_exceptions_file( args.uncrustifyexceptions) else: uncrustify_path_accept = uncrustify_path_reject = [] ### Main actions start here # Setup the git repository init_git(gitrepo) # Pull current list of tags here, to fast skip any work already done if args.targetbranch != "package": switch_to_branch(args.targetbranch, orphan=True) current_git_tags = get_current_git_tags(gitrepo) os.chdir(start_cwd) ## SVN interactions and reloading state # Decide which svn packages we will import # Note that if we're pulling the packages from a tag diff file, we also get tags # at this point, otherwise the tag list is empty. svn_packages = get_tags(args.tagfiles, args.svnpath) # Add "trunk" packages, if required if args.processtrunk: for package, tags in svn_packages.iteritems(): if "trunk" not in tags: tags.append("trunk") # Initialise SVN and author metadata cache with any stored values svn_metadata_cache = initialise_metadata(args.svncachefile) author_metadata_cache = initialise_metadata(args.authorcachefile) # Prepare package import scan_svn_tags_and_get_metadata(svnroot, svn_packages, svn_metadata_cache, author_metadata_cache, args.intermediatetags, package_veto=package_veto) # Now presistify metadata cache backup_metadata(svn_metadata_cache, start_cwd, args.svncachefile, start_timestamp_string) backup_metadata(author_metadata_cache, start_cwd, args.authorcachefile, start_timestamp_string) # Setup dictionary for keying by SVN revision number svn_cache_revision_dict = svn_cache_revision_dict_init(svn_metadata_cache) ## git processing actions # Process each SVN tag in order ordered_revisions = svn_cache_revision_dict.keys() ordered_revisions.sort(cmp=lambda x, y: cmp(int(x), int(y))) logger.info("Will process {0} SVN revisions in total".format( len(ordered_revisions))) counter = 0 processed_tags = 0 timing = [] os.chdir(gitrepo) for rev in ordered_revisions: counter += 1 start = time.time() logger.info("SVN Revsion {0} ({1} of {2})".format( rev, counter, len(ordered_revisions))) for pkg_tag in svn_cache_revision_dict[rev]: if get_flattened_git_tag(pkg_tag["package"], pkg_tag["tag"], rev) in current_git_tags: logger.info("Tag {0} exists already - skipping".format( os.path.join(pkg_tag["package"], pkg_tag["tag"]))) continue if args.targetbranch == "package": switch_to_branch(os.path.basename(pkg_tag["package"]), orphan=True) svn_co_tag_and_commit( svnroot, gitrepo, pkg_tag["package"], pkg_tag["tag"], svn_metadata_cache[os.path.basename( pkg_tag["package"])]["svn"][pkg_tag["tag"]][rev], author_metadata_cache, svn_path_accept=svn_path_accept, svn_path_reject=svn_path_reject, package_veto=package_veto, license_text=license_text, license_path_accept=license_path_accept, license_path_reject=license_path_reject, uncrustify_config=args.uncrustify, uncrustify_path_accept=uncrustify_path_accept, uncrustify_path_reject=uncrustify_path_reject) processed_tags += 1 elapsed = time.time() - start logger.info( "{0} processed in {1}s ({2} packages really processed)".format( counter, elapsed, processed_tags)) timing.append(elapsed) # Last task, clean all empty directories (git does not track these, but they are clutter) check_output_with_retry(("git", "clean", "-f", "-d")) if args.importtimingfile: os.chdir(start_cwd) with open(args.importtimingfile, "w") as time_file: json.dump(timing, time_file)