def uncrustify_sources(svn_path, svn_co_root, uncrustify_config, uncrustify_path_accept, uncrustify_path_reject):
## @brief Uncrustify code before import
# @param svn_path Filesystem path to parsed up SVN checkout
# @param svn_co_root Base directory of SVN checkout
# @param uncrustify_config Config file for uncrustify pass
# @param uncrustify_path_accept Paths to force
# @param uncrustify_path_reject Paths to exclude
for root, dirs, files in os.walk(svn_path):
for name in files:
filename = os.path.join(root, name)
svn_filename = filename[len(svn_co_root) + 1:]
path_veto = False
for filter in uncrustify_path_reject:
if re.match(filter, svn_filename):
logger.debug("File {0} will not go through uncrustify".format(svn_filename, filter.pattern))
path_veto = True
break
for filter in uncrustify_path_accept:
if re.match(filter, svn_filename):
logger.debug("File {0} will be passed to uncrustify".format(svn_filename, filter.pattern))
path_veto = False
break
if path_veto:
continue
extension = filename.rsplit(".", 1)[1] if "." in filename else ""
if extension in ("cxx", "cpp", "icc", "cc", "c", "C", "h", "hpp", "hh"):
logger.debug("Uncrustifying {0}".format(filename))
cmd = ("uncrustify", "-c", uncrustify_config, "--no-backup", "-l", "CPP", filename)
# We do not consider uncrustify errors as fatal for the import... this can
# happen because of a source file issue or picking the wrong language
try:
check_output_with_retry(cmd, retries=0)
except RuntimeError:
logger.warning("Uncrustify failed on {0}".format(filename))
def init_git(gitrepo):
## @brief Initialise git repo, if needed
# @param gitrepo Git repository path
if not os.path.exists(gitrepo):
os.makedirs(gitrepo)
os.chdir(gitrepo)
if os.path.exists(os.path.join(gitrepo, ".git")):
logger.info("Found existing git repo, {0}".format(gitrepo))
check_output_with_retry(("git", "reset", "--hard"))
else:
logger.info("Initialising git repo: {0}".format(gitrepo))
check_output_with_retry(("git", "init"))
def svn_get_path_metadata(svnroot, package, package_path, revision=None):
# # @brief Get SVN metadata and return as a simple dictionary keyed on date, author and commit revision
logger.info("Querying SVN metadata for {0}".format(os.path.join(package, package_path)))
cmd = ["svn", "info", os.path.join(svnroot, package, package_path), "--xml"]
svn_info = check_output_with_retry(cmd)
tree = eltree.fromstring(svn_info)
info = {"date": tree.find(".//date").text.rsplit(".",1)[0], # Strip off sub-second part
"author": tree.find(".//author").text,
"revision": tree.find(".//commit").attrib['revision']}
cmd = ["svn", "log", os.path.join(svnroot, package, package_path), "-r", info["revision"], "--xml"]
svn_log = check_output_with_retry(cmd)
tree = eltree.fromstring(svn_log)
info["msg"] = tree.find(".//msg").text.strip().encode('ascii', 'ignore')
return info
def author_info_lookup(author_name):
try:
cmd = ["phonebook", "--login", author_name, "--terse", "firstname", "--terse", "surname", "--terse", "email"]
author_info = check_output_with_retry(cmd, retries=1).strip().split(";")
return {"name": " ".join(author_info[:2]), "email": author_info[2]}
except IndexError:
raise RuntimeError("Had a problem decoding phonebook info for '{0}'".format(author_name))
def prepare_branch_point(branch, parentbranch=None):
## @brief Using information about the target branch and any parent
# switch to the correct point in history to start/continue
# @param branch Target branch name
# @param parentbranch If creating a new branch, this is the @c BRANCH:COMMIT_ID of
# where to make the new branch from; syntax @c BRANCH:@FILE and @c BRANCH:@TIMESTAMP
# is also supported, where the timestamp will be used to find the branch directly
# (and can be taken from @c JSON release data in @c FILE)
if not parentbranch or branch_exists(branch):
logger.info("Switching to branch {0}".format(branch))
switch_to_branch(branch, orphan=True)
else:
parent, commit = parentbranch.split(":")
check_output_with_retry(("git", "checkout", parent), retries=1) # needed?
if commit.startswith("@"):
timestamp = commit[1:]
# If this maps to a file, try to open it as a release JSON, otherwise treat it as
# a plain timestamp
if os.access(timestamp, os.F_OK):
logger.info("Taking branching timestamp from file {0}".format(timestamp))
with open(timestamp) as fh:
branch_point_release = json.load(fh)
timestamp = branch_point_release["release"]["timestamp"]
logger.info("Using timestamp {0} for branch point".format(timestamp))
commit = check_output_with_retry(["git", "log", "--until", str(timestamp), "-n1", "--pretty=format:%H"],
retries=1).strip()
logger.info("Mapped timestamp {0} to commit {1}".format(timestamp, commit))
check_output_with_retry(("git", "checkout", commit), retries=1)
check_output_with_retry(("git", "checkout", "-b", branch), retries=1)
def branch_builder(gitrepo, branch, tag_files, svn_metadata_cache, author_metadata_cache,
parentbranch=None, baserelease=None,
skipreleasetag=False, dryrun=False, only_forward=False, commit_date="now"):
## @brief Main branch builder function
# @param gitrepo The git repository location
# @param branch The git branch to work on
# @param tag_files The plain tag content files to process
# @param svn_metadata_cache The standard metadata cache from SVN
# @param author_metadata_cache Cached author data
# @param parentbranch If creating a new branch, this is the BRANCH:COMMIT_ID of where to make the new branch from
# @param skipreleasetag If @c True then skip creating git tags for each processed release
# @param dryrun If @c True, do nothing except print commands that would have been executed
# @param only_forward If @c True then never revert a package to a previous version or import a branch tag
# @param commit_date Choice for commit date when building branches
# Prepare - chdir and then make sure we are on the correct branch
os.chdir(gitrepo)
prepare_branch_point(branch, parentbranch)
# Main loop starts here, with one pass for each tag file we are processing
for tag_file in tag_files:
with open(tag_file) as tag_file_fh:
release_data = json.load(tag_file_fh)
tag_list = get_current_git_tags(gitrepo)
current_release_tags = get_current_release_tag_dict(tag_list, branch) # Markers for which packages have been processed
logger.info("Processing release {0} ({1} current tags)".format(release_data["release"]["name"], len(current_release_tags)))
release_tag = git_release_tag(release_data["release"], branch)
if release_tag in tag_list and not skipreleasetag:
logger.info("Release tag {0} already made - skipping".format(release_tag))
continue
if commit_date == "release":
logger.info("Setting committer date to {0:.0f}".format(release_data["release"]["timestamp"]))
os.environ["GIT_COMMITTER_DATE"] = "{0:.0f}".format(release_data["release"]["timestamp"])
# Find which packages need updated for this new tag content file
import_list, packages_considered = find_packages_for_update(release_data, tag_list, branch,
svn_metadata_cache, current_release_tags, only_forward)
## Sort the list of tags to be imported by SVN revision number for a
# more or less sensible package by package commit history
sorted_import_revisions = import_list.keys()
sorted_import_revisions.sort(cmp=lambda x,y: cmp(int(x), int(y)))
## Now loop over all the packages we have to import and update them
pkg_processed = 0
for revision in sorted_import_revisions:
for pkg_import in import_list[revision]:
pkg_processed += 1
do_package_import(pkg_import, svn_metadata_cache, author_metadata_cache, release_name=release_data["release"]["name"],
branch=branch, dryrun=dryrun, commit_date=commit_date)
logger.info("Processed {0}/{1} revisions".format(pkg_processed, len(import_list)))
## After all packages are updated, look for packages which present in the last
# release, but not this one, so they need to be removed
new_current_release_tags = get_current_release_tag_dict(tag_list, branch) # Updated package list after upgrade
packages_to_remove = []
packages_to_revert = {}
for package_name, old_package_state in current_release_tags.iteritems():
if package_name in packages_considered:
logger.debug("Package {0} was processed for {1}".format(package_name, release_data["release"]["name"]))
continue
## @note We have a package that was not "considered" in the current release.
# If we don't have a baserelease then this has been removed, so we zap it.
# If there is a baserelease...
# ...and this package is not in it, it was in the cache, then was removed, so zap it.
# ...and this package is in it, then compare the versions and "revert" to the base
# release version if they are different.
if baserelease:
base_package_version = None
for package, base_package_data in baserelease["tags"].iteritems():
if base_package_data["package_name"] == package_name:
base_package_version = base_package_data
break
if base_package_version:
if base_package_version["svn_tag"] == old_package_state["svn_tag"]:
logger.debug("Package {0} remains at base release version {1}".format(base_package_data["package_name"],
base_package_version["svn_tag"]))
packages_considered.append(package_name) # Flag we dealt this package
else:
logger.info("Package {0} was removed from cache - reverting to base "
"release version {1}".format(base_package_data["package_name"],
base_package_version["svn_tag"]))
package_name = base_package_data["package_name"]
svn_meta_tag_key = os.path.join("tags", base_package_version["svn_tag"])
svn_revision = svn_metadata_cache[package_name]["svn"][svn_meta_tag_key].keys()[0]
git_import_tag = get_flattened_git_tag(package, base_package_version["svn_tag"], svn_revision)
packages_to_revert[package_name] = {"package": package,
"package_name": os.path.basename(package),
"git_import_tag": get_flattened_git_tag(package, base_package_version["svn_tag"], svn_revision),
"svn_tag": base_package_version["svn_tag"],
"svn_revision": svn_revision,
"branch_import_tag": get_flattened_git_tag(package, base_package_version["svn_tag"], svn_revision, branch),
"svn_meta_tag_key": svn_meta_tag_key,
"current_branch_import_tag": current_release_tags[package_name]["git_tag"]}
else:
logger.info("Package {0} was removed from the cache and is not in the base release".format(package_name))
packages_to_remove.append(package_name)
else:
logger.info("Package {0} has been removed from the release".format(package_name))
packages_to_remove.append(package_name)
if baserelease:
logger.info("{0} packages have been reverted to their base SVN state".format(len(packages_to_revert)))
for package_name, revert_data in packages_to_revert.iteritems():
do_package_import(revert_data, svn_metadata_cache, author_metadata_cache, release_name=release_data["release"]["name"],
branch=branch, dryrun=dryrun, commit_date=commit_date)
logger.info("{0} packages have been removed from the release".format(len(packages_to_remove)))
for package in packages_to_remove:
logger.info("Removing {0} from {1}".format(package, branch))
if not dryrun:
package_path = os.path.join(svn_metadata_cache[package]["path"], package)
recursive_delete(package_path)
check_output_with_retry(("git", "add", "-A"), dryrun=dryrun)
cmd = ["git", "commit", "--allow-empty", "-m", "{0} deleted from {1}".format(package_path, branch)]
check_output_with_retry(cmd, dryrun=dryrun)
check_output_with_retry(("git", "tag", "-d", current_release_tags[package]["git_tag"]), retries=1, dryrun=dryrun)
pkg_processed += 1
## Now, finally, tag the release as done
if not skipreleasetag:
if release_data["release"]["nightly"]:
check_output_with_retry(("git", "tag", release_tag), retries=1, dryrun=dryrun)
else:
check_output_with_retry(("git", "tag", release_tag, "-a",
"-m", "Tagging release {0}".format(release_data["release"]["name"])),
retries=1, dryrun=dryrun)
logger.info("Tagged release {0} as {1} "
"({2} packages processed)".format(release_data["release"]["name"],
release_tag, pkg_processed))
else:
logger.info("Processed release {0} (no tag; {1} packages processed)".format(release_data["release"]["name"], pkg_processed))
def do_package_import(pkg_import, svn_metadata_cache, author_metadata_cache, release_name="unknown", branch="unknown",
dryrun=False, commit_date="now"):
## @brief Import a package's SVN tag onto the current git branch
# updating the corresponding git tags
# @param pkg_import package import dictionary (see find_packages_for_update for the
# structure)
# @param svn_metadata_cache The standard metadata cache from SVN
# @param author_metadata_cache Cached author data
# @param release_name Name of current release being built (used only for generating log messages)
# @param branch Current branch name (used only for generating log messages)
# @param dryrun Boolean, if @c true then don't actually act
# @param commit_date Choices for setting committer date
logger.info("Migrating {0} from {1} to {2} for {3}...".format(pkg_import["package"],
pkg_import["current_branch_import_tag"],
pkg_import["svn_tag"], release_name))
# Need to wipe out all contents in case files were removed from package
if not dryrun:
recursive_delete(pkg_import["package"])
check_output_with_retry(("git", "checkout", pkg_import["git_import_tag"], pkg_import["package"]), dryrun=dryrun)
# Splat Changelog file - we do not want these on the production branches
try:
os.remove(os.path.join(pkg_import["package"], "ChangeLog"))
except OSError:
pass
# Done - now commit and tag
if logger.level <= logging.DEBUG:
cmd = ["git", "status"]
logger.debug(check_output_with_retry(cmd))
check_output_with_retry(("git", "add", "-A", pkg_import["package"]), dryrun=dryrun)
staged = check_output_with_retry(("git", "diff", "--name-only", "--staged"), dryrun=dryrun)
if len(staged) == 0 and (not dryrun):
# Nothing staged, so skip doing any commit, but do make the import tag for this branch
# so that we don't repeat this step again
logger.warning("Package {0} - no changes staged for {1}, "
"git tagging and skipping commit".format(pkg_import["package"], release_name))
check_output_with_retry(("git", "tag", pkg_import["branch_import_tag"]), retries=1, dryrun=dryrun)
return
rev_meta = svn_metadata_cache[pkg_import["package_name"]]["svn"][pkg_import["svn_meta_tag_key"]][pkg_import["svn_revision"]]
msg = rev_meta["msg"]
if pkg_import["svn_tag"] == "trunk":
msg += " (trunk r{0})".format(rev_meta["revision"])
else:
msg += " ({0})".format(pkg_import["svn_tag"])
cl_diff = changelog_diff(pkg_import["package"],
from_tag="/".join(pkg_import["current_branch_import_tag"].split("/")[1:]) if pkg_import["current_branch_import_tag"] else None,
to_tag=pkg_import["git_import_tag"])
if cl_diff:
msg += "\n\n" + "\n".join(cl_diff)
cmd = ["git", "commit", "-m", msg]
author = author_string(rev_meta["author"], author_metadata_cache)
cmd.append("--author='{0}'".format(author))
cmd.append("--date={0}".format(rev_meta["date"]))
if commit_date == "author":
os.environ["GIT_COMMITTER_DATE"] = rev_meta["date"]
check_output_with_retry(cmd, retries=1, dryrun=dryrun)
if commit_date == "author":
del os.environ["GIT_COMMITTER_DATE"]
check_output_with_retry(("git", "tag", pkg_import["branch_import_tag"]), retries=1, dryrun=dryrun)
if pkg_import["current_branch_import_tag"]:
check_output_with_retry(("git", "tag", "-d", pkg_import["current_branch_import_tag"]), retries=1, dryrun=dryrun)
logger.info("Committed {0} ({1}) onto {2} for {3}".format(pkg_import["package"],
pkg_import["svn_tag"], branch, release_name))
def svn_co_tag_and_commit(svnroot, gitrepo, package, tag, svn_metadata=None, author_metadata_cache=None, branch=None,
svn_path_accept=[], svn_path_reject=[], package_veto=[], commit=True, revision=None,
license_text=None, license_path_accept=[], license_path_reject=[],
uncrustify_config=None, uncrustify_path_accept=[], uncrustify_path_reject=[]):
## @brief Make a temporary space, check out from svn, clean-up, copy and then git commit and tag
# @param svnroot Base path to SVN repository
# @param gitrepo Path to git repository to import to
# @param package Path to package root (in git and svn)
# @param tag Package tag to import (i.e., path after base package path)
# @param svn_metadata SVN metadata cache
# @param author_metadata_cache Author name/email cache
# @param branch Git branch to switch to before import
# @param svn_path_accept Paths to force import to git
# @param svn_path_reject Paths to force reject from the import
# @param package_veto List of packages to just plain refuse to handle
# @param commit Boolean flag to manage commit (can be set to @c False to only checkout and process)
# @param license_text List of strings containing the license text to add (if @c False, then no
# license file is added)
# @param revision Force SVN revision number (useful for svnpull.py, where
# no svn metadata is available)
# @param license_path_accept Paths to force include in license file addition
# @param license_path_reject Paths to exclude from license file addition
# @param uncrustify_config Uncrustify configuration file
# @param uncrustify_path_accept Paths to force uncrustify to run on
# @param uncrustify_path_reject Paths to exclude from uncrustify
if package in package_veto:
logger.info("Package {0} is vetoed - skipping import".format(package))
return
msg = "Importing SVN path {0}/{1} to {0}".format(package, tag)
if svn_metadata and tag == "trunk":
msg += " (r{0})".format(svn_metadata["revision"])
logger.info(msg)
if branch:
logger.info("Switching to branch {0}".format(branch))
switch_to_branch(args.targetbranch)
tempdir = tempfile.mkdtemp()
full_svn_path = os.path.join(tempdir, package)
cmd = ["svn", "checkout"]
if revision:
cmd.extend(["-r", str(revision)])
elif svn_metadata:
cmd.extend(["-r", svn_metadata["revision"]])
cmd.extend([os.path.join(svnroot, package, tag), os.path.join(tempdir, package)])
check_output_with_retry(cmd, retries=1, wait=3)
# Clean out directory of things we don't want to import
svn_cleanup(full_svn_path, svn_co_root=tempdir,
svn_path_accept=svn_path_accept, svn_path_reject=svn_path_reject)
# If desired, inject a licence into the source code
if license_text:
svn_license_injector(full_svn_path, svn_co_root=tempdir, license_text=license_text,
license_path_accept=license_path_accept, license_path_reject=license_path_reject)
# Pass C++ sources through uncrustify
if uncrustify_config:
uncrustify_sources(full_svn_path, svn_co_root=tempdir, uncrustify_config=uncrustify_config,
uncrustify_path_accept=uncrustify_path_accept, uncrustify_path_reject=uncrustify_path_reject)
# Copy to git
full_git_path = os.path.join(gitrepo, package)
package_root, package_name = os.path.split(full_git_path)
try:
if os.path.isdir(full_git_path):
shutil.rmtree(full_git_path, ignore_errors=True)
os.makedirs(package_root)
except OSError:
pass
shutil.move(full_svn_path, package_root)
if commit:
# get ChangeLog diff
cl_diff = changelog_diff(package)
# Commit
check_output_with_retry(("git", "add", "-A", package))
if logger.level <= logging.DEBUG:
logger.debug(check_output_with_retry(("git", "status")))
cmd = ["git", "commit", "--allow-empty", "-m", "{0} ({1} - r{2})".
format(svn_metadata['msg'], tag.replace('tags/','',1), svn_metadata['revision'])]
if svn_metadata:
cmd.extend(("--author='{0}'".format(author_string(svn_metadata["author"], author_metadata_cache)),
"--date={0}".format(svn_metadata["date"])))
if cl_diff:
cmd.extend(("-m", "Diff in ChangeLog:\n" + '\n'.join(cl_diff)))
check_output_with_retry(cmd)
cmd = ["git", "tag", "-a", get_flattened_git_tag(package, tag, svn_metadata["revision"]), "-m", ""]
check_output_with_retry(cmd)
# Clean up
shutil.rmtree(tempdir)
def main():
parser = argparse.ArgumentParser(
description='SVN to git migrator, ATLAS style')
parser.add_argument('svnroot',
metavar='SVNDIR',
help="Location of svn repository root")
parser.add_argument('gitrepo',
metavar='GITDIR',
help="Location of git repository")
parser.add_argument(
'tagfiles',
nargs="+",
metavar='TAGFILE',
help=
"List of release tag content files to process - all tags found in these files will "
"be imported (any already imported tags will be skipped)")
parser.add_argument(
'--targetbranch',
default="package",
help=
"Target git branch for import. Default is the special value 'package' in which "
"each package is imported onto its own branch")
parser.add_argument(
'--svnpath',
metavar='PATH',
nargs='+',
default=[],
help="Restrict actions to this list of paths in the SVN tree (use to "
"make small scale tests of the import workflow).")
parser.add_argument(
'--intermediatetags',
action="store_true",
help=
"Import all tags from oldest release tag found, instead of just release tags"
)
parser.add_argument(
'--processtrunk',
action="store_true",
help=
"Update trunk versions during the import (False by default, the trunk will be skipped)."
)
parser.add_argument(
'--svncachefile',
metavar='FILE',
help=
"File containing cache of SVN information - default '[gitrepo].svn.metadata'"
)
parser.add_argument(
'--authorcachefile',
metavar='FILE',
help=
"File containing cache of author name and email information - default '[gitrepo].author.metadata'"
)
parser.add_argument(
'--importtimingfile',
metavar="FILE",
help=
"File to dump SVN->git import timing information - default '[gitrepo]-timing.json'"
)
parser.add_argument(
'--svnfilterexceptions',
'--sfe',
metavar="FILE",
help=
"File listing path globs to exempt from SVN import filter (lines with '+PATH') or "
"to always reject (lines with '-PATH'); default %(default)s. Use NONE to have no exceptions.",
default=os.path.join(os.path.dirname(os.path.abspath(__file__)),
"atlasoffline-exceptions.txt"))
parser.add_argument(
'--packageveto',
metavar="FILE",
help="File listing packages that will be skipped completely on import.",
default=os.path.join(os.path.dirname(os.path.abspath(__file__)),
"atlaspackage-exceptions.txt"))
parser.add_argument(
'--licensefile',
metavar="FILE",
help="License file to add to source code files (default "
"is to add %(default)s license file)",
default=os.path.join(os.path.dirname(os.path.abspath(__file__)),
"cerncopy.txt"))
parser.add_argument(
'--licenseexceptions',
metavar="FILE",
help="File listing path globs to exempt from or "
"always apply license file to (same format as --svnfilterexceptions)",
default=os.path.join(os.path.dirname(os.path.abspath(__file__)),
"atlaslicense-exceptions.txt"))
parser.add_argument(
'--uncrustify',
metavar="FILE",
help="Uncrustify configuration file to use to process C++ "
"sources through before git import (by default uncrustify will not be used)"
)
parser.add_argument(
'--uncrustifyexceptions',
metavar="FILE",
help="File listing path globs to exempt from or "
"always apply uncrustify to (same format as --svnfilterexceptions)",
default=os.path.join(os.path.dirname(os.path.abspath(__file__)),
"atlasuncrustify-exceptions.txt"))
parser.add_argument('--debug',
'--verbose',
"-v",
action="store_true",
help="Switch logging into DEBUG mode")
# Parse and handle initial arguments
args = parser.parse_args()
if args.debug:
logger.setLevel(logging.DEBUG)
# Massage default values
if not args.svncachefile:
args.svncachefile = os.path.basename(args.gitrepo) + ".svn.metadata"
if not args.authorcachefile:
args.authorcachefile = os.path.basename(
args.gitrepo) + ".author.metadata"
if not args.importtimingfile:
args.importtimingfile = os.path.basename(args.gitrepo) + "-timing.json"
# Set svnroot and git repo, get some starting values
svnroot = args.svnroot
gitrepo = os.path.abspath(args.gitrepo)
start_cwd = os.getcwd()
start_timestamp_string = time.strftime("%Y%m%dT%H%M.%S")
logger.debug("Set SVN root to {0} and git repo to {1}".format(
svnroot, gitrepo))
# Load exception globs
svn_path_accept, svn_path_reject = load_exceptions_file(
args.svnfilterexceptions)
# Load package vetos
if args.packageveto:
package_veto = load_package_veto(args.packageveto)
else:
package_veto = []
# License file loading
if args.licensefile:
with open(args.licensefile) as lfh:
license_text = [line.rstrip() for line in lfh.readlines()]
else:
license_text = None
if args.licenseexceptions:
license_path_accept, license_path_reject = load_exceptions_file(
args.licenseexceptions)
else:
license_path_accept = license_path_reject = []
# Uncrustify exceptions file
if args.uncrustify:
uncrustify_path_accept, uncrustify_path_reject = load_exceptions_file(
args.uncrustifyexceptions)
else:
uncrustify_path_accept = uncrustify_path_reject = []
### Main actions start here
# Setup the git repository
init_git(gitrepo)
# Pull current list of tags here, to fast skip any work already done
if args.targetbranch != "package":
switch_to_branch(args.targetbranch, orphan=True)
current_git_tags = get_current_git_tags(gitrepo)
os.chdir(start_cwd)
## SVN interactions and reloading state
# Decide which svn packages we will import
# Note that if we're pulling the packages from a tag diff file, we also get tags
# at this point, otherwise the tag list is empty.
svn_packages = get_tags(args.tagfiles, args.svnpath)
# Add "trunk" packages, if required
if args.processtrunk:
for package, tags in svn_packages.iteritems():
if "trunk" not in tags:
tags.append("trunk")
# Initialise SVN and author metadata cache with any stored values
svn_metadata_cache = initialise_metadata(args.svncachefile)
author_metadata_cache = initialise_metadata(args.authorcachefile)
# Prepare package import
scan_svn_tags_and_get_metadata(svnroot,
svn_packages,
svn_metadata_cache,
author_metadata_cache,
args.intermediatetags,
package_veto=package_veto)
# Now presistify metadata cache
backup_metadata(svn_metadata_cache, start_cwd, args.svncachefile,
start_timestamp_string)
backup_metadata(author_metadata_cache, start_cwd, args.authorcachefile,
start_timestamp_string)
# Setup dictionary for keying by SVN revision number
svn_cache_revision_dict = svn_cache_revision_dict_init(svn_metadata_cache)
## git processing actions
# Process each SVN tag in order
ordered_revisions = svn_cache_revision_dict.keys()
ordered_revisions.sort(cmp=lambda x, y: cmp(int(x), int(y)))
logger.info("Will process {0} SVN revisions in total".format(
len(ordered_revisions)))
counter = 0
processed_tags = 0
timing = []
os.chdir(gitrepo)
for rev in ordered_revisions:
counter += 1
start = time.time()
logger.info("SVN Revsion {0} ({1} of {2})".format(
rev, counter, len(ordered_revisions)))
for pkg_tag in svn_cache_revision_dict[rev]:
if get_flattened_git_tag(pkg_tag["package"], pkg_tag["tag"],
rev) in current_git_tags:
logger.info("Tag {0} exists already - skipping".format(
os.path.join(pkg_tag["package"], pkg_tag["tag"])))
continue
if args.targetbranch == "package":
switch_to_branch(os.path.basename(pkg_tag["package"]),
orphan=True)
svn_co_tag_and_commit(
svnroot,
gitrepo,
pkg_tag["package"],
pkg_tag["tag"],
svn_metadata_cache[os.path.basename(
pkg_tag["package"])]["svn"][pkg_tag["tag"]][rev],
author_metadata_cache,
svn_path_accept=svn_path_accept,
svn_path_reject=svn_path_reject,
package_veto=package_veto,
license_text=license_text,
license_path_accept=license_path_accept,
license_path_reject=license_path_reject,
uncrustify_config=args.uncrustify,
uncrustify_path_accept=uncrustify_path_accept,
uncrustify_path_reject=uncrustify_path_reject)
processed_tags += 1
elapsed = time.time() - start
logger.info(
"{0} processed in {1}s ({2} packages really processed)".format(
counter, elapsed, processed_tags))
timing.append(elapsed)
# Last task, clean all empty directories (git does not track these, but they are clutter)
check_output_with_retry(("git", "clean", "-f", "-d"))
if args.importtimingfile:
os.chdir(start_cwd)
with open(args.importtimingfile, "w") as time_file:
json.dump(timing, time_file)