def getNodeType(svnExternal): cmd = ['svn', 'info', '--xml'] if svnExternal.operativeRev: cmd += ['-r', str(svnExternal.operativeRev)] if svnExternal.pegRev is not None: cmd += [svnExternal.QualifiedUrl + '@' + str(svnExternal.pegRev)] else: cmd += [svnExternal.QualifiedUrl] try: DebugLog.print(str(cmd)) xmlStr = subprocess.check_output(cmd).decode() xmlRootNode = ET.fromstring(xmlStr) assert xmlRootNode.tag == 'info' typeStr = xmlRootNode.find('./entry').attrib['kind'] if typeStr == 'file': return SvnNodeType.FILE elif typeStr == 'dir': return SvnNodeType.DIR else: raise Exception('svn info returned unknown type: ' + typeStr) except subprocess.CalledProcessError: return None
def sparse_dir_checkout(dir): if len(dir) == 0: return assert dir[-1] != '/' assert dir[-1] != os.sep (head, tail) = os.path.split(dir) if svn.IsSvnWc(os.path.join(args.checkoutPath, dir)): # if dir is already versioned, then it got checked out earlier! # hence it must now be skipped, to prevent setting the depth back to empty! return # recursively checkout the parent directories first if len(head) > 0: sparse_dir_checkout(head) # checkout dir itself. cmd = [ 'svn', 'update', '--force', '--set-depth', 'empty', os.path.join(args.checkoutPath, dir) ] if args.rev is not None: cmd += ['-r', args.rev] DebugLog.print(str(cmd)) if not args.dry_run: subprocess.check_output(cmd)
def wrap(*args, **kw): with DebugLog.scopedPush("Enter func: " + f.__name__ + " args:[" + str(args) + ", " + str(kw) + "]"): ts = time() result = f(*args, **kw) te = time() DebugLog.print( 'Exit func:{} args:[{}, {}] took: {:2.4f} sec'.format( f.__name__, args, kw, te - ts)) return result
def parse_cli_args(): global args """parse the script input arguments""" parser = argparse.ArgumentParser( description="Svn Sparse checkout given a '.svnSparseCheckout'") parser.add_argument("-v", "--verbose", help="increase output verbosity", action="store_true") parser.add_argument("-d", "--debug", help="enable debug output", action="store_true") parser.add_argument("--username", help="svn username") parser.add_argument("--password", help="svn credentials") parser.add_argument("-N", "--dry-run", help="Do not perform any actions, only simulate them.", action="store_true") parser.add_argument("-r", "--rev", help="svn revision to checkout") parser.add_argument( "configFilePath", nargs="?", default=".svnSparseCheckout.yml", help=".svnSparseCheckout file decscribing the sparse checkout") parser.add_argument("checkoutPath", nargs="?", default=".", help="path to perform the sparse checkout") args = parser.parse_args() # register custom exception handler h = ExceptionHandle(args.debug) sys.excepthook = h.exception_handler DebugLog.enabled = args.debug with DebugLogScopedPush("cli arguments:"): DebugLog.print(str(args)) return args
def parse_cli_args(): """parse the script input arguments""" parser = argparse.ArgumentParser( description="create a git svn bridge repo in an svn working copy") parser.add_argument("-v", "--verbose", help="increase output verbosity", action="store_true") parser.add_argument("-d", "--debug", help="enable debug output", action="store_true") parser.add_argument("-N", "--dry-run", help="Do not perform any actions, only simulate them.", action="store_true") parser.add_argument("--ignore-dir", help="exclude an svn folder from git.", default=[], nargs="*") parser.add_argument( "-r", "--revision", help= "specify the revision form where to start git svn fetch (defaults to svn BASE revision)." ) parser.add_argument("-f", "--force", help="force git svn init even if svn wc is dirty", action="store_true") args = parser.parse_args() # register custom exception handler h = ExceptionHandle(args.debug) sys.excepthook = h.exception_handler DebugLog.enabled = args.debug with DebugLogScopedPush("cli arguments:"): DebugLog.print(str(args)) return args
def find_svn_branch_point_for_current_gitbranch(): """ find the svn branch point for the current git branch return a (qualified_url:str, svn_rev:int) tuple """ # find the git commit where HEAD branched of from the SVN branch # i.e. find the most recent contained commit with a log entry as follows # git-svn-id: http://vsrv-bele-svn1/svn/Software/Main/NMAPI/NMAPI_Main@72264 cfd94225-6148-4c34-bb2a-21ea3148c527 cmd = ['git', 'log', '--grep=^git-svn-id:', '--date-order', '-1'] DebugLog.print(str(cmd)) output = subprocess.check_output(cmd).decode() m = re.search(r"git-svn-id: ([^@]*)@([0-9]*)", output) url = m.group(1) svn_rev = int(m.group(2)) return (url, svn_rev)
def sparse_file_checkout(file): (head, tail) = os.path.split(file) # first checout the intermediate directories sparse_dir_checkout(head) # now checkout the file itself cmd = [ 'svn', 'update', '--force', '--set-depth', 'immediates', '--accept=working', os.path.join(args.checkoutPath, file) ] if args.rev is not None: cmd += ['-r', args.rev] DebugLog.print(str(cmd)) if not args.dry_run: subprocess.check_output(cmd)
def main(arguments=None): # parse cli arguments if no arguments are given global args if arguments is None: parse_cli_args() else: args = arguments if svn.IsSvnWc(args.checkoutPath): raise Exception( "checkoutPath arg is already an svn working copy, can not check out another in the same place: " + args.checkoutPath) if not os.path.isfile(args.configFilePath): raise Exception("the provided config file does not exists: " + args.configFilePath) with open(args.configFilePath, 'rt') as f: sparseCheckout = yaml.load(f) DebugLog.print(str(sparseCheckout)) cmd = ['svn', 'checkout', '--force', '--depth', 'empty'] if args.rev is not None: cmd += ['-r', args.rev] if args.username is not None: cmd += ['--username', args.username] if args.password is not None: cmd += ['--password', args.password] cmd += [sparseCheckout['svnRepoUrl'], args.checkoutPath] DebugLog.print(str(cmd)) if not args.dry_run: subprocess.check_output(cmd) for f in sparseCheckout['files']: sparse_file_checkout(f)
def parse(hostRepoUrl, svnWCFolderPath, definition): """Create a SvnExternal instance given a single svn:externals definition [-r <operativeRev>] <url>[@<pegRev>] <path> """ DebugLog.print("parsing: " + str(definition)) remainder = definition # parse operative Revision terms = remainder.split(' ') if terms[0] == '-r': operativeRev = terms[1] remainder = " ".join(terms[2:]) del terms else: operativeRev = None del terms # parse url & peg revision terms = remainder.split(' ') PegSeparatorIdx = terms[0].find('@') if PegSeparatorIdx == -1: url = terms[0] pegRev = None remainder = " ".join(terms[1:]) else: url = terms[0][:PegSeparatorIdx] pegRev = terms[0][(PegSeparatorIdx + 1):] remainder = " ".join(terms[1:]) del terms # parse path assert (len(remainder) > 0) path = remainder if path[0] == "'" and path[-1] == "'": path = path[1:-1] return SvnExternal(hostRepoUrl, svnWCFolderPath, operativeRev, url, pegRev, path)
def GetSvnExternalsFromGitSvnBridge(): hostRepoUrl = GetGitSvnUrl() # get all the svn:externals properties recursively cmd = ['git', 'svn', 'show-externals'] out = subprocess.check_output(cmd).decode() # parse the output line by line fail in case or problems currentPathDef = "" externalDefinitions = [] for line in out.splitlines(): if len(line) == 0: continue DebugLog.print("Processing: " + line) if line.startswith("# /"): DebugLog.print(" -> currentPathDef: " + currentPathDef) # key is a new pathDef currentPathDef = line[2:] continue # current line must be a externalDef externaldefString = line[len(currentPathDef):] DebugLog.print(" -> externaldefString: " + externaldefString) # derive windows path from currentPathDef relative to repo root svnWCFolderPath = currentPathDef svnWCFolderPath = '.' + svnWCFolderPath svnWCFolderPath.replace("/", os.sep) externaldef = svn.SvnExternal.parse(hostRepoUrl, svnWCFolderPath, externaldefString) externalDefinitions.append(externaldef) return externalDefinitions
def GetGitSvnBranchPointRev(): # find the git commit where HEAD branched of from the SVN branch cmd = [ 'git', 'log', '--grep=^git-svn-id:', '--date-order', '-1', "--format=%H" ] DebugLog.print(str(cmd)) output = subprocess.check_output(cmd).decode() DebugLog.print(output) output = output.splitlines() assert len(output) == 1 gitSvnBranchPoint_gitSHA = output[0] # determine the Svn commit revision associated with the branch point cmd = ['git', 'svn', 'find-rev', '-B', str(gitSvnBranchPoint_gitSHA)] DebugLog.print(str(cmd)) output = subprocess.check_output(cmd).decode() DebugLog.print(output) output = output.splitlines() assert len(output) == 1 gitSvnBranchPoint_SvnRev = output[0] DebugLog.print("git-svn branchpoint (git-Sha - svn-Rev): " + gitSvnBranchPoint_gitSHA + " - " + gitSvnBranchPoint_SvnRev) return (gitSvnBranchPoint_gitSHA, gitSvnBranchPoint_SvnRev)
def SvnCountCommits(startRev, endRev): cmd = ["svn", "log", "--xml", "-r", str(startRev) + ":" + str(endRev)] DebugLog.print(str(cmd)) xmlStr = subprocess.check_output(cmd).decode() xmlNode = ET.fromstring(xmlStr) return len(xmlNode.findall('logentry'))
def checkoutSvnExternal(svnExternal, discard_local_changes=False): """checkout or update an svn external """ WCExternalPath = os.path.join(svnExternal.svnWCFolderPath, svnExternal.path.replace('/', os.sep)) DebugLog.print("check external at : " + WCExternalPath) svnWc_is_dirty = False # check for existing svn external pointing to wrong url # in which case the external needs to be deleted and a clean checkout is needed # instead of only updating the existing svnExternal to the proper revision if os.path.isdir(WCExternalPath): # an svn working copy is expected! if not IsSvnWc(WCExternalPath): raise Exception( "Terminating: svn external expected, but no svn WC is found:" + WCExternalPath) # svn wc may not be dirty, since this action would result in lost data! svnWc_is_dirty = IsSvnWcDirty(WCExternalPath) if svnWc_is_dirty and discard_local_changes is False: raise Exception( "Terminating: dirty svn external is not allowed (risk of losing changes!): " + WCExternalPath) # if the working copy is a checkout of the wrong svn url then delete it. # e.g. the external has updated and new checkout is needed existingExternalQualifiedUrl = GetQualifiedUrlForFolder(WCExternalPath) forceCleanCheckout = (svnExternal.QualifiedUrl != existingExternalQualifiedUrl) # if the pegRev and operatative revision are set but not equal, then lets be conservative and do a clean checkout. forceCleanCheckout |= ( (svnExternal.operativeRev is not None) and (svnExternal.pegRev != svnExternal.operativeRev)) if forceCleanCheckout and svnWc_is_dirty and discard_local_changes is False: raise Exception( "Terminating: conflicting requirements: local changes are not discarded, yet a clean checkout is required: " + WCExternalPath) if forceCleanCheckout: DebugLog.print("removing : " + WCExternalPath) DebugLog.print("existing external points to") DebugLog.print(existingExternalQualifiedUrl) DebugLog.print("but new external points to") DebugLog.print(svnExternal.QualifiedUrl) DebugLog.print("So a new checkout is needed.") def onerror(func, path, exc_info): """ Error handler for ``shutil.rmtree``. If the error is due to an access error (read only file) it attempts to add write permission and then retries. If the error is for another reason it re-raises the error. Usage : ``shutil.rmtree(path, onerror=onerror)`` """ import stat if not os.access(path, os.W_OK): # Is the error an access error ? os.chmod(path, stat.S_IWUSR) func(path) else: raise shutil.rmtree(WCExternalPath, onerror=onerror) if os.path.isdir(WCExternalPath): DebugLog.print("udpate external dir at: " + WCExternalPath) if svnWc_is_dirty is True: assert discard_local_changes is True # bug: should have failed earlier! # revert local changes cmd = ['svn', 'revert', '-R', WCExternalPath] DebugLog.print(str(cmd)) svnOutput = subprocess.check_output(cmd).decode() DebugLog.print(svnOutput) # build svn cli arguments cmd = ['svn', 'up', '-q'] if svnExternal.pegRev: assert (svnExternal.operativeRev is None) or (svnExternal.operativeRev == svnExternal.pegRev) cmd += ['-r', str(svnExternal.pegRev)] cmd += ['.'] # checkout already exists, just update it pwd = os.getcwd() os.chdir(WCExternalPath) try: DebugLog.print(str(cmd)) svnOutput = subprocess.check_output(cmd).decode() DebugLog.print(svnOutput) finally: os.chdir(pwd) elif os.path.isfile(WCExternalPath): DebugLog.print("udpate external file at: " + WCExternalPath) if svnWc_is_dirty is True: assert discard_local_changes is True # bug: should have failed earlier! # revert local changes cmd = ['svn', 'revert', WCExternalPath] DebugLog.print(str(cmd)) svnOutput = subprocess.check_output(cmd).decode() DebugLog.print(svnOutput) # build svn cli arguments cmd = ['svn', 'up', '-q'] if svnExternal.pegRev: assert (svnExternal.operativeRev is None) or (svnExternal.operativeRev == svnExternal.pegRev) cmd += ['-r', str(svnExternal.pegRev)] cmd += [os.path.basename(WCExternalPath)] # checkout already exists, just update it pwd = os.getcwd() os.chdir(os.path.dirname(WCExternalPath)) try: DebugLog.print(str(cmd)) svnOutput = subprocess.check_output(cmd).decode() DebugLog.print(svnOutput) finally: os.chdir(pwd) else: DebugLog.print("new checkout at: " + WCExternalPath) assert not os.path.exists(WCExternalPath) type = getNodeType(svnExternal) if type == SvnNodeType.DIR: DebugLog.print("new checkout of dir at: " + WCExternalPath) # build svn cli arguments cmd = ['svn', 'checkout', '-q'] if svnExternal.operativeRev: cmd += ['-r', str(svnExternal.operativeRev)] if svnExternal.pegRev: cmd += [ svnExternal.QualifiedUrl + '@' + str(svnExternal.pegRev) ] else: cmd += [svnExternal.QualifiedUrl] cmd += [svnExternal.path.replace('/', os.sep)] # external doesn't yet exists, check it out from the svn repo pwd = os.getcwd() os.makedirs(svnExternal.svnWCFolderPath, exist_ok=True) os.chdir(svnExternal.svnWCFolderPath) try: DebugLog.print(str(cmd)) svnOutput = subprocess.check_output(cmd).decode() DebugLog.print(svnOutput) finally: os.chdir(pwd) elif type == SvnNodeType.FILE: DebugLog.print("new checkout of file at: " + WCExternalPath) # external doesn't yet exists, check it out from the svn repo pwd = os.getcwd() dirpath = os.path.dirname( os.path.join(svnExternal.svnWCFolderPath, svnExternal.path)) os.makedirs(dirpath, exist_ok=True) os.chdir(dirpath) DebugLog.print("cwd: " + os.getcwd()) try: urlparts = urllib.parse.urlparse(svnExternal.QualifiedUrl) parentDirUrl = urllib.parse.urlunparse( (urlparts.scheme, urlparts.netloc, os.path.dirname(urlparts.path), "", "", "")) cmd = [ 'svn', 'checkout', '--force', '--depth', 'empty', parentDirUrl, "." ] DebugLog.print(str(cmd)) subprocess.check_call(cmd) cmd = [ 'svn', 'update', '--force', '--set-depth', 'immediates', '--accept=working', os.path.basename(urlparts.path) ] DebugLog.print(str(cmd)) subprocess.check_call(cmd) finally: os.chdir(pwd) elif type is None: raise Exception( "svn External does not exist and thus can't be checked out: " + str(svnEsvnExternal)) else: # this type can only be any of the 3 values # this the SvnNodetype enum expand? assert false