def getNodeType(svnExternal):
cmd = ['svn', 'info', '--xml']
if svnExternal.operativeRev:
cmd += ['-r', str(svnExternal.operativeRev)]
if svnExternal.pegRev is not None:
cmd += [svnExternal.QualifiedUrl + '@' + str(svnExternal.pegRev)]
else:
cmd += [svnExternal.QualifiedUrl]
try:
DebugLog.print(str(cmd))
xmlStr = subprocess.check_output(cmd).decode()
xmlRootNode = ET.fromstring(xmlStr)
assert xmlRootNode.tag == 'info'
typeStr = xmlRootNode.find('./entry').attrib['kind']
if typeStr == 'file':
return SvnNodeType.FILE
elif typeStr == 'dir':
return SvnNodeType.DIR
else:
raise Exception('svn info returned unknown type: ' + typeStr)
except subprocess.CalledProcessError:
return None
def sparse_dir_checkout(dir):
if len(dir) == 0:
return
assert dir[-1] != '/'
assert dir[-1] != os.sep
(head, tail) = os.path.split(dir)
if svn.IsSvnWc(os.path.join(args.checkoutPath, dir)):
# if dir is already versioned, then it got checked out earlier!
# hence it must now be skipped, to prevent setting the depth back to empty!
return
# recursively checkout the parent directories first
if len(head) > 0:
sparse_dir_checkout(head)
# checkout dir itself.
cmd = [
'svn', 'update', '--force', '--set-depth', 'empty',
os.path.join(args.checkoutPath, dir)
]
if args.rev is not None:
cmd += ['-r', args.rev]
DebugLog.print(str(cmd))
if not args.dry_run:
subprocess.check_output(cmd)
def wrap(*args, **kw):
with DebugLog.scopedPush("Enter func: " + f.__name__ + " args:[" +
str(args) + ", " + str(kw) + "]"):
ts = time()
result = f(*args, **kw)
te = time()
DebugLog.print(
'Exit func:{} args:[{}, {}] took: {:2.4f} sec'.format(
f.__name__, args, kw, te - ts))
return result
def parse_cli_args():
global args
"""parse the script input arguments"""
parser = argparse.ArgumentParser(
description="Svn Sparse checkout given a '.svnSparseCheckout'")
parser.add_argument("-v",
"--verbose",
help="increase output verbosity",
action="store_true")
parser.add_argument("-d",
"--debug",
help="enable debug output",
action="store_true")
parser.add_argument("--username", help="svn username")
parser.add_argument("--password", help="svn credentials")
parser.add_argument("-N",
"--dry-run",
help="Do not perform any actions, only simulate them.",
action="store_true")
parser.add_argument("-r", "--rev", help="svn revision to checkout")
parser.add_argument(
"configFilePath",
nargs="?",
default=".svnSparseCheckout.yml",
help=".svnSparseCheckout file decscribing the sparse checkout")
parser.add_argument("checkoutPath",
nargs="?",
default=".",
help="path to perform the sparse checkout")
args = parser.parse_args()
# register custom exception handler
h = ExceptionHandle(args.debug)
sys.excepthook = h.exception_handler
DebugLog.enabled = args.debug
with DebugLogScopedPush("cli arguments:"):
DebugLog.print(str(args))
return args
def parse_cli_args():
"""parse the script input arguments"""
parser = argparse.ArgumentParser(
description="create a git svn bridge repo in an svn working copy")
parser.add_argument("-v",
"--verbose",
help="increase output verbosity",
action="store_true")
parser.add_argument("-d",
"--debug",
help="enable debug output",
action="store_true")
parser.add_argument("-N",
"--dry-run",
help="Do not perform any actions, only simulate them.",
action="store_true")
parser.add_argument("--ignore-dir",
help="exclude an svn folder from git.",
default=[],
nargs="*")
parser.add_argument(
"-r",
"--revision",
help=
"specify the revision form where to start git svn fetch (defaults to svn BASE revision)."
)
parser.add_argument("-f",
"--force",
help="force git svn init even if svn wc is dirty",
action="store_true")
args = parser.parse_args()
# register custom exception handler
h = ExceptionHandle(args.debug)
sys.excepthook = h.exception_handler
DebugLog.enabled = args.debug
with DebugLogScopedPush("cli arguments:"):
DebugLog.print(str(args))
return args
def find_svn_branch_point_for_current_gitbranch():
""" find the svn branch point for the current git branch
return a (qualified_url:str, svn_rev:int) tuple
"""
# find the git commit where HEAD branched of from the SVN branch
# i.e. find the most recent contained commit with a log entry as follows
# git-svn-id: http://vsrv-bele-svn1/svn/Software/Main/NMAPI/NMAPI_Main@72264 cfd94225-6148-4c34-bb2a-21ea3148c527
cmd = ['git', 'log', '--grep=^git-svn-id:', '--date-order', '-1']
DebugLog.print(str(cmd))
output = subprocess.check_output(cmd).decode()
m = re.search(r"git-svn-id: ([^@]*)@([0-9]*)", output)
url = m.group(1)
svn_rev = int(m.group(2))
return (url, svn_rev)
def sparse_file_checkout(file):
(head, tail) = os.path.split(file)
# first checout the intermediate directories
sparse_dir_checkout(head)
# now checkout the file itself
cmd = [
'svn', 'update', '--force', '--set-depth', 'immediates',
'--accept=working',
os.path.join(args.checkoutPath, file)
]
if args.rev is not None:
cmd += ['-r', args.rev]
DebugLog.print(str(cmd))
if not args.dry_run:
subprocess.check_output(cmd)
def main(arguments=None):
# parse cli arguments if no arguments are given
global args
if arguments is None:
parse_cli_args()
else:
args = arguments
if svn.IsSvnWc(args.checkoutPath):
raise Exception(
"checkoutPath arg is already an svn working copy, can not check out another in the same place: "
+ args.checkoutPath)
if not os.path.isfile(args.configFilePath):
raise Exception("the provided config file does not exists: " +
args.configFilePath)
with open(args.configFilePath, 'rt') as f:
sparseCheckout = yaml.load(f)
DebugLog.print(str(sparseCheckout))
cmd = ['svn', 'checkout', '--force', '--depth', 'empty']
if args.rev is not None:
cmd += ['-r', args.rev]
if args.username is not None:
cmd += ['--username', args.username]
if args.password is not None:
cmd += ['--password', args.password]
cmd += [sparseCheckout['svnRepoUrl'], args.checkoutPath]
DebugLog.print(str(cmd))
if not args.dry_run:
subprocess.check_output(cmd)
for f in sparseCheckout['files']:
sparse_file_checkout(f)
def parse(hostRepoUrl, svnWCFolderPath, definition):
"""Create a SvnExternal instance given a single svn:externals definition
[-r <operativeRev>] <url>[@<pegRev>] <path>
"""
DebugLog.print("parsing: " + str(definition))
remainder = definition
# parse operative Revision
terms = remainder.split(' ')
if terms[0] == '-r':
operativeRev = terms[1]
remainder = " ".join(terms[2:])
del terms
else:
operativeRev = None
del terms
# parse url & peg revision
terms = remainder.split(' ')
PegSeparatorIdx = terms[0].find('@')
if PegSeparatorIdx == -1:
url = terms[0]
pegRev = None
remainder = " ".join(terms[1:])
else:
url = terms[0][:PegSeparatorIdx]
pegRev = terms[0][(PegSeparatorIdx + 1):]
remainder = " ".join(terms[1:])
del terms
# parse path
assert (len(remainder) > 0)
path = remainder
if path[0] == "'" and path[-1] == "'":
path = path[1:-1]
return SvnExternal(hostRepoUrl, svnWCFolderPath, operativeRev, url,
pegRev, path)
def GetSvnExternalsFromGitSvnBridge():
hostRepoUrl = GetGitSvnUrl()
# get all the svn:externals properties recursively
cmd = ['git', 'svn', 'show-externals']
out = subprocess.check_output(cmd).decode()
# parse the output line by line fail in case or problems
currentPathDef = ""
externalDefinitions = []
for line in out.splitlines():
if len(line) == 0:
continue
DebugLog.print("Processing: " + line)
if line.startswith("# /"):
DebugLog.print(" -> currentPathDef: " + currentPathDef)
# key is a new pathDef
currentPathDef = line[2:]
continue
# current line must be a externalDef
externaldefString = line[len(currentPathDef):]
DebugLog.print(" -> externaldefString: " + externaldefString)
# derive windows path from currentPathDef relative to repo root
svnWCFolderPath = currentPathDef
svnWCFolderPath = '.' + svnWCFolderPath
svnWCFolderPath.replace("/", os.sep)
externaldef = svn.SvnExternal.parse(hostRepoUrl, svnWCFolderPath,
externaldefString)
externalDefinitions.append(externaldef)
return externalDefinitions
def GetGitSvnBranchPointRev():
# find the git commit where HEAD branched of from the SVN branch
cmd = [
'git', 'log', '--grep=^git-svn-id:', '--date-order', '-1',
"--format=%H"
]
DebugLog.print(str(cmd))
output = subprocess.check_output(cmd).decode()
DebugLog.print(output)
output = output.splitlines()
assert len(output) == 1
gitSvnBranchPoint_gitSHA = output[0]
# determine the Svn commit revision associated with the branch point
cmd = ['git', 'svn', 'find-rev', '-B', str(gitSvnBranchPoint_gitSHA)]
DebugLog.print(str(cmd))
output = subprocess.check_output(cmd).decode()
DebugLog.print(output)
output = output.splitlines()
assert len(output) == 1
gitSvnBranchPoint_SvnRev = output[0]
DebugLog.print("git-svn branchpoint (git-Sha - svn-Rev): " +
gitSvnBranchPoint_gitSHA + " - " + gitSvnBranchPoint_SvnRev)
return (gitSvnBranchPoint_gitSHA, gitSvnBranchPoint_SvnRev)
def SvnCountCommits(startRev, endRev):
cmd = ["svn", "log", "--xml", "-r", str(startRev) + ":" + str(endRev)]
DebugLog.print(str(cmd))
xmlStr = subprocess.check_output(cmd).decode()
xmlNode = ET.fromstring(xmlStr)
return len(xmlNode.findall('logentry'))
def checkoutSvnExternal(svnExternal, discard_local_changes=False):
"""checkout or update an svn external
"""
WCExternalPath = os.path.join(svnExternal.svnWCFolderPath,
svnExternal.path.replace('/', os.sep))
DebugLog.print("check external at : " + WCExternalPath)
svnWc_is_dirty = False
# check for existing svn external pointing to wrong url
# in which case the external needs to be deleted and a clean checkout is needed
# instead of only updating the existing svnExternal to the proper revision
if os.path.isdir(WCExternalPath):
# an svn working copy is expected!
if not IsSvnWc(WCExternalPath):
raise Exception(
"Terminating: svn external expected, but no svn WC is found:" +
WCExternalPath)
# svn wc may not be dirty, since this action would result in lost data!
svnWc_is_dirty = IsSvnWcDirty(WCExternalPath)
if svnWc_is_dirty and discard_local_changes is False:
raise Exception(
"Terminating: dirty svn external is not allowed (risk of losing changes!): "
+ WCExternalPath)
# if the working copy is a checkout of the wrong svn url then delete it.
# e.g. the external has updated and new checkout is needed
existingExternalQualifiedUrl = GetQualifiedUrlForFolder(WCExternalPath)
forceCleanCheckout = (svnExternal.QualifiedUrl !=
existingExternalQualifiedUrl)
# if the pegRev and operatative revision are set but not equal, then lets be conservative and do a clean checkout.
forceCleanCheckout |= (
(svnExternal.operativeRev is not None)
and (svnExternal.pegRev != svnExternal.operativeRev))
if forceCleanCheckout and svnWc_is_dirty and discard_local_changes is False:
raise Exception(
"Terminating: conflicting requirements: local changes are not discarded, yet a clean checkout is required: "
+ WCExternalPath)
if forceCleanCheckout:
DebugLog.print("removing : " + WCExternalPath)
DebugLog.print("existing external points to")
DebugLog.print(existingExternalQualifiedUrl)
DebugLog.print("but new external points to")
DebugLog.print(svnExternal.QualifiedUrl)
DebugLog.print("So a new checkout is needed.")
def onerror(func, path, exc_info):
"""
Error handler for ``shutil.rmtree``.
If the error is due to an access error (read only file)
it attempts to add write permission and then retries.
If the error is for another reason it re-raises the error.
Usage : ``shutil.rmtree(path, onerror=onerror)``
"""
import stat
if not os.access(path, os.W_OK):
# Is the error an access error ?
os.chmod(path, stat.S_IWUSR)
func(path)
else:
raise
shutil.rmtree(WCExternalPath, onerror=onerror)
if os.path.isdir(WCExternalPath):
DebugLog.print("udpate external dir at: " + WCExternalPath)
if svnWc_is_dirty is True:
assert discard_local_changes is True # bug: should have failed earlier!
# revert local changes
cmd = ['svn', 'revert', '-R', WCExternalPath]
DebugLog.print(str(cmd))
svnOutput = subprocess.check_output(cmd).decode()
DebugLog.print(svnOutput)
# build svn cli arguments
cmd = ['svn', 'up', '-q']
if svnExternal.pegRev:
assert (svnExternal.operativeRev is
None) or (svnExternal.operativeRev == svnExternal.pegRev)
cmd += ['-r', str(svnExternal.pegRev)]
cmd += ['.']
# checkout already exists, just update it
pwd = os.getcwd()
os.chdir(WCExternalPath)
try:
DebugLog.print(str(cmd))
svnOutput = subprocess.check_output(cmd).decode()
DebugLog.print(svnOutput)
finally:
os.chdir(pwd)
elif os.path.isfile(WCExternalPath):
DebugLog.print("udpate external file at: " + WCExternalPath)
if svnWc_is_dirty is True:
assert discard_local_changes is True # bug: should have failed earlier!
# revert local changes
cmd = ['svn', 'revert', WCExternalPath]
DebugLog.print(str(cmd))
svnOutput = subprocess.check_output(cmd).decode()
DebugLog.print(svnOutput)
# build svn cli arguments
cmd = ['svn', 'up', '-q']
if svnExternal.pegRev:
assert (svnExternal.operativeRev is
None) or (svnExternal.operativeRev == svnExternal.pegRev)
cmd += ['-r', str(svnExternal.pegRev)]
cmd += [os.path.basename(WCExternalPath)]
# checkout already exists, just update it
pwd = os.getcwd()
os.chdir(os.path.dirname(WCExternalPath))
try:
DebugLog.print(str(cmd))
svnOutput = subprocess.check_output(cmd).decode()
DebugLog.print(svnOutput)
finally:
os.chdir(pwd)
else:
DebugLog.print("new checkout at: " + WCExternalPath)
assert not os.path.exists(WCExternalPath)
type = getNodeType(svnExternal)
if type == SvnNodeType.DIR:
DebugLog.print("new checkout of dir at: " + WCExternalPath)
# build svn cli arguments
cmd = ['svn', 'checkout', '-q']
if svnExternal.operativeRev:
cmd += ['-r', str(svnExternal.operativeRev)]
if svnExternal.pegRev:
cmd += [
svnExternal.QualifiedUrl + '@' + str(svnExternal.pegRev)
]
else:
cmd += [svnExternal.QualifiedUrl]
cmd += [svnExternal.path.replace('/', os.sep)]
# external doesn't yet exists, check it out from the svn repo
pwd = os.getcwd()
os.makedirs(svnExternal.svnWCFolderPath, exist_ok=True)
os.chdir(svnExternal.svnWCFolderPath)
try:
DebugLog.print(str(cmd))
svnOutput = subprocess.check_output(cmd).decode()
DebugLog.print(svnOutput)
finally:
os.chdir(pwd)
elif type == SvnNodeType.FILE:
DebugLog.print("new checkout of file at: " + WCExternalPath)
# external doesn't yet exists, check it out from the svn repo
pwd = os.getcwd()
dirpath = os.path.dirname(
os.path.join(svnExternal.svnWCFolderPath, svnExternal.path))
os.makedirs(dirpath, exist_ok=True)
os.chdir(dirpath)
DebugLog.print("cwd: " + os.getcwd())
try:
urlparts = urllib.parse.urlparse(svnExternal.QualifiedUrl)
parentDirUrl = urllib.parse.urlunparse(
(urlparts.scheme, urlparts.netloc,
os.path.dirname(urlparts.path), "", "", ""))
cmd = [
'svn', 'checkout', '--force', '--depth', 'empty',
parentDirUrl, "."
]
DebugLog.print(str(cmd))
subprocess.check_call(cmd)
cmd = [
'svn', 'update', '--force', '--set-depth', 'immediates',
'--accept=working',
os.path.basename(urlparts.path)
]
DebugLog.print(str(cmd))
subprocess.check_call(cmd)
finally:
os.chdir(pwd)
elif type is None:
raise Exception(
"svn External does not exist and thus can't be checked out: " +
str(svnEsvnExternal))
else:
# this type can only be any of the 3 values
# this the SvnNodetype enum expand?
assert false