def migrate(): config = configuration.read() rtc = ImportHandler(config) rtcworkspace = WorkspaceHandler(config) git = Commiter initialize(config) streamuuid = config.streamuuid streamname = config.streamname branchname = streamname + "_branchpoint" componentbaselineentries = rtc.getcomponentbaselineentriesfromstream(streamuuid) rtcworkspace.setnewflowtargets(streamuuid) git.branch(branchname) history = rtc.readhistory(componentbaselineentries, streamname) changeentries = rtc.getchangeentriesofstreamcomponents(componentbaselineentries) rtc.acceptchangesintoworkspace(rtc.getchangeentriestoaccept(changeentries, history)) shouter.shout("All changes until creation of stream '%s' accepted" % streamname) git.pushbranch(branchname) git.branch(streamname) rtcworkspace.setcomponentstobaseline(componentbaselineentries, streamuuid) rtcworkspace.load() changeentries = rtc.getchangeentriesofstream(streamuuid) rtc.acceptchangesintoworkspace(rtc.getchangeentriestoaccept(changeentries, history)) git.pushbranch(streamname) shouter.shout("All changes of stream '%s' accepted - Migration of stream completed" % streamname)
def collectstreamuuid(self): shouter.shout("Get UUID of configured stream") showuuidcommand = "%s --show-alias n --show-uuid y show attributes -r %s -w %s" % ( self.scmcommand, self.repo, self.streamname) output = shell.getoutput(showuuidcommand) splittedfirstline = output[0].split(" ") self.streamuuid = splittedfirstline[0].strip()[1:-1]
def pushbranch(branchname, force=False): if branchname: shouter.shout("Push of branch " + branchname) if force: return shell.execute("git push -f origin " + branchname) else: return shell.execute("git push origin " + branchname)
def shout_command_to_log(command, outputfile=None): if logcommands: logmessage = "Executed Command: " + quote(command) if outputfile: shouter.shout(logmessage + " --> " + outputfile) else: shouter.shout(logmessage)
def acceptchangesintoworkspace(self, changeentries): amountofchanges = len(changeentries) if amountofchanges == 0: shouter.shout("Found no changes to accept") else: shouter.shoutwithdate("Start accepting %s changesets" % amountofchanges) amountofacceptedchanges = 0 for changeEntry in changeentries: amountofacceptedchanges += 1 if not changeEntry.isAccepted( ): # change could already be accepted from a retry if not Changes.accept(self.acceptlogpath, changeEntry): shouter.shout( "Change wasnt succesfully accepted into workspace, please load your workspace in eclipse and check whats wrong" ) self.is_user_aborting(changeEntry) # self.retryacceptincludingnextchangesets(changeEntry, changeentries) if not Differ.has_diff(): # no differences found - force reload of the workspace shouter.shout( "No changes for commiting in git detected, going to reload the workspace" ) WorkspaceHandler().load() if not Differ.has_diff(): shouter.shout( "Still no changes... Please load your workspace in eclipse and check whats wrong" ) # still no differences, something wrong self.is_user_aborting(changeEntry) shouter.shout("Accepted change %d/%d into working directory" % (amountofacceptedchanges, amountofchanges)) Commiter.addandcommit(changeEntry) return amountofacceptedchanges
def accept(config, logpath, *changeentries): for changeEntry in changeentries: shouter.shout("Accepting: " + changeEntry.tostring()) revisions = Changes._collectids(changeentries) Changes.latest_accept_command = config.scmcommand + " accept -v -o -r " + config.repo + " -t " + \ config.workspace + " --changes" + revisions return shell.execute(Changes.latest_accept_command, logpath, "a")
def migrate(): config = configuration.read() rtc = ImportHandler(config) rtcworkspace = WorkspaceHandler(config) git = Commiter initialize(config) streamuuids = config.streamuuids for streamuuid in streamuuids: componentbaselineentries = rtc.getcomponentbaselineentriesfromstream(streamuuid) streamname = config.streamnames[streamuuids.index(streamuuid)] rtcworkspace.setnewflowtargets(streamuuid) git.branch(streamname) history = rtc.readhistory(componentbaselineentries, streamname) changeentries = rtc.getchangeentriesofstreamcomponents(componentbaselineentries) rtc.acceptchangesintoworkspace(rtc.getchangeentriestoaccept(changeentries, history)) shouter.shout("All changes of components of stream '%s' accepted" % streamname) git.pushbranch(streamname) rtcworkspace.setcomponentstobaseline(componentbaselineentries, streamuuid) rtcworkspace.load() changeentries = rtc.getchangeentriesofstream(streamuuid) rtc.acceptchangesintoworkspace(rtc.getchangeentriestoaccept(changeentries, history)) git.pushbranch(streamname) shouter.shout("All changes of stream '%s' accepted - Migration of stream completed" % streamname) morestreamstomigrate = streamuuids.index(streamuuid) + 1 is not len(streamuuids) if morestreamstomigrate: git.checkout("master") rtcworkspace.recreateoldestworkspace()
def setnewflowtargets(self, streamuuid): shouter.shout("Set new Flowtargets") if not self.hasflowtarget(streamuuid): shell.execute("%s add flowtarget -r %s %s %s" % (self.scmcommand, self.repo, self.workspace, streamuuid)) command = "%s set flowtarget -r %s %s --default --current %s" % (self.scmcommand, self.repo, self.workspace, streamuuid) shell.execute(command)
def getchangeentriesofstream(self, streamtocompare): shouter.shout("Start collecting changes since baseline creation") missingchangeentries = {} changeentries = self.getchangeentriesbytypeandvalue("stream", streamtocompare) for changeentry in changeentries: missingchangeentries[changeentry.revision] = changeentry return missingchangeentries
def initalize(self): shell.execute("git init --bare " + self.repoName) shouter.shout("Repository was created in " + os.getcwd()) shell.execute("git clone " + self.repoName) os.chdir(self.clonedRepoName) shell.execute("git config push.default current") self.createignore()
def determineinitialbaseline(self, stream): regex = "\(_[\w-]+\)" pattern = re.compile(regex) config = self.config componentbaselinesentries = self.getcomponentbaselineentriesfromstream( stream) logincredentials = "" if not config.stored: logincredentials = loginCredentialsCommand % (config.user, config.password) for entry in componentbaselinesentries: shouter.shout("Determine initial baseline of " + entry.componentname) # use always scm, lscm fails when specifying maximum over 10k command = "scm --show-alias n --show-uuid y list baselines --components %s -r %s %s -m 20000" % \ (entry.component, config.repo, logincredentials) baselineslines = shell.getoutput(command) baselineslines.reverse( ) # reverse to have earliest baseline on top for baselineline in baselineslines: matcher = pattern.search(baselineline) if matcher: matchedstring = matcher.group() uuid = matchedstring[1:-1] entry.baseline = uuid entry.baselinename = "Automatically detected initial baseline" shouter.shout("Initial baseline is: %s" % baselineline) break return componentbaselinesentries
def acceptchangesintoworkspace(self, changeentries): git = Commiter amountofchanges = len(changeentries) shouter.shoutwithdate("Start accepting %s changesets" % amountofchanges) amountofacceptedchanges = 0 skipnextchangeset = False reloaded = False for changeEntry in changeentries: amountofacceptedchanges += 1 if skipnextchangeset: skipnextchangeset = False continue acceptedsuccesfully = Changes.accept( changeEntry, logpath=self.acceptlogpath) is 0 if not acceptedsuccesfully: shouter.shout( "Change wasnt succesfully accepted into workspace") skipnextchangeset = self.retryacceptincludingnextchangeset( changeEntry, changeentries) elif not reloaded: if self.is_reloading_necessary(): WorkspaceHandler(self.config).load() reloaded = True shouter.shout("Accepted change %s/%s into working directory" % (amountofacceptedchanges, amountofchanges)) git.addandcommit(changeEntry)
def setcomponentstobaseline(self, componentbaselineentries, streamuuid): for entry in componentbaselineentries: shouter.shout("Set component '%s'(%s) to baseline '%s' (%s)" % (entry.componentname, entry.component, entry.baselinename, entry.baseline)) replacecommand = "%s set component -r %s -b %s %s stream %s %s --overwrite-uncommitted" % \ (self.scmcommand, self.repo, entry.baseline, self.workspace, streamuuid, entry.component) shell.execute(replacecommand)
def load(self): command = "%s load -r %s %s --force" % (self.scmcommand, self.repo, self.workspace) if self.config.includecomponentroots: command += " --include-root" shouter.shout("Start (re)loading current workspace: " + command) shell.execute(command) shouter.shout("Load of workspace finished") Commiter.restore_shed_gitignore(Commiter.get_untracked_statuszlines())
def setcomponentstobaseline(self, componentbaselineentries, streamuuid): for entry in componentbaselineentries: shouter.shout("Set component '%s' to baseline '%s'" % (entry.componentname, entry.baselinename)) replacecommand = "lscm set component -r %s -b %s %s stream %s %s --overwrite-uncommitted" % \ (self.repo, entry.baseline, self.workspace, streamuuid, entry.component) shell.execute(replacecommand)
def is_user_aborting(change): shouter.shout("Last executed command: \n" + Changes.latest_accept_command) shouter.shout("Apropriate git commit command \n" + Commiter.getcommitcommand(change)) reallycontinue = "Do you want to continue? Y for continue, any key for abort" if input(reallycontinue).lower() == "y": return True else: sys.exit("Please check the output/log and rerun program with resume")
def initialize(config): RTCInitializer.loginandcollectstreamuuid(config) workspace = WorkspaceHandler(config) if config.useexistingworkspace: shouter.shout("Use existing workspace to start migration") workspace.load() else: workspace.createandload(config.streamuuid, config.initialcomponentbaselines)
def is_user_agreeing_to_accept_next_change(change): messagetoask = "Press Y for accepting following changes, press N to skip" while True: answer = input(messagetoask).lower() if answer == "y": return True elif answer == "n": return not ImportHandler.is_user_aborting(change) else: shouter.shout("Please answer with Y/N, input was " + answer)
def getchangeentriesofstreamcomponents(self, componentbaselineentries): missingchangeentries = {} shouter.shout("Start collecting changeentries") for componentBaseLineEntry in componentbaselineentries: shouter.shout("Collect changes until baseline %s of component %s" % (componentBaseLineEntry.baselinename, componentBaseLineEntry.componentname)) changeentries = self.getchangeentriesofbaseline(componentBaseLineEntry.baseline) for changeentry in changeentries: missingchangeentries[changeentry.revision] = changeentry return missingchangeentries
def collectstreamuuids(self): shouter.shout("Get UUID's of configured streamnames") for streamname in self.streamnames: streamname = streamname.strip() showuuidcommand = "lscm --show-alias n --show-uuid y show attributes -r %s -w %s" % ( self.repo, streamname) output = shell.getoutput(showuuidcommand) splittedfirstline = output[0].split(" ") streamuuid = splittedfirstline[0].strip()[1:-1] self.streamuuids.append(streamuuid)
def addandcommit(changeentry): Commiter.replaceauthor(changeentry.author, changeentry.email) shell.execute("git add -A") shell.execute(Commiter.getcommitcommand(changeentry)) Commiter.commitcounter += 1 if Commiter.commitcounter is 30: shouter.shout("30 Commits happend, push current branch to avoid out of memory") Commiter.pushbranch("") Commiter.commitcounter = 0 shouter.shout("Commited change in local git repository")
def promotebranchtomaster(branchname): master = "master" masterrename = Commiter.renamebranch(master, "masterRenamedAt_" + datetime.now().strftime('%Y%m%d_%H%M%S')) copybranch = Commiter.copybranch(branchname, master) if masterrename is 0 and copybranch is 0: return Commiter.pushbranch(master, True) else: shouter.shout("Branch %s couldnt get renamed to master, please do that on your own" % branchname) return 1 # branch couldnt get renamed
def getchangeentriesofstreamcomponents(self, componentbaselineentries): missingchangeentries = {} shouter.shout("Start collecting changeentries") changeentriesbycomponentbaselineentry = {} for componentBaseLineEntry in componentbaselineentries: changeentries = self.getchangeentriesofbaseline( componentBaseLineEntry.baseline) for changeentry in changeentries: missingchangeentries[changeentry.revision] = changeentry return missingchangeentries
def readhistory(self, componentbaselineentries, streamname): if not self.config.useprovidedhistory: warning = "Warning - UseProvidedHistory is set to false, merge-conflicts are more likely to happen. \n " \ "For more information see https://github.com/WtfJoke/rtc2git/wiki/Getting-your-History-Files" shouter.shout(warning) return None historyuuids = {} shouter.shout("Start reading history files") for componentBaseLineEntry in componentbaselineentries: history = self.gethistory(componentBaseLineEntry.componentname, streamname) historyuuids[componentBaseLineEntry.component] = history return historyuuids
def getsimplehistoryfromfile(outputfilename): revisions = [] if not os.path.isfile(outputfilename): shouter.shout("History file not found: " + outputfilename) shouter.shout("Skipping this part of history") return revisions with open(outputfilename, 'r') as file: for line in file: revisions.append(line.strip()) revisions.reverse() # to begin by the oldest return revisions
def getsimplehistoryfromfile(outputfilename): revisions = [] if not os.path.isfile(outputfilename): shouter.shout("History file not found: " + outputfilename) shouter.shout("Skipping this part of history") return revisions with open(outputfilename, 'r', encoding=shell.encoding) as file: for line in file: revisions.append(line.strip()) revisions.reverse() # to begin by the oldest return revisions
def resume(): shouter.shout("Found existing git repo in work directory, resuming migration...") config = configuration.get() os.chdir(config.workDirectory) os.chdir(config.clonedGitRepoName) if Differ.has_diff(): sys.exit("Your git repo has some uncommited changes, please add/remove them manually") RTCLogin.loginandcollectstreamuuid() Initializer.preparerepo() if config.previousstreamname: prepare() else: WorkspaceHandler().load()
def accept(logpath, *changeentries): for changeEntry in changeentries: shouter.shout("Accepting: " + changeEntry.tostring()) revisions = Changes._collectids(changeentries) config = configuration.get() Changes.latest_accept_command = config.scmcommand + " accept --verbose --overwrite-uncommitted --accept-missing-changesets --no-merge --repository-uri " + config.repo + " --target " + \ config.workspace + " --changes" + revisions exitcode = shell.execute(Changes.latest_accept_command, logpath, "a") if exitcode is 0: for changeEntry in changeentries: changeEntry.setAccepted() return True else: return False
def promotebranchtomaster(branchname): master = "master" masterrename = Commiter.renamebranch( master, "masterRenamedAt_" + datetime.now().strftime('%Y%m%d_%H%M%S')) copybranch = Commiter.copybranch(branchname, master) if masterrename is 0 and copybranch is 0: return Commiter.pushbranch(master, True) else: shouter.shout( "Branch %s couldnt get renamed to master, please do that on your own" % branchname) return 1 # branch couldnt get renamed
def initialize(): config = configuration.get() directory = config.workDirectory if os.path.exists(directory): sys.exit("Configured directory already exists, please make sure to use a non-existing directory") shouter.shout("Migration will take place in " + directory) os.makedirs(directory) os.chdir(directory) config.deletelogfolder() git = Initializer() git.initalize() RTCInitializer.initialize() if Differ.has_diff(): git.initialcommit() Commiter.pushmaster()
def initialcommitandpush(): shouter.shout("Initial git add") shell.execute("git add -A", os.devnull) shouter.shout("Finished initial git add, starting commit") shell.execute("git commit -m %s -q" % shell.quote("Initial Commit")) shouter.shout("Finished commit") shell.execute("git push origin master") shouter.shout("Finished push")
def acceptchangesintoworkspace(self, changeentries): amountofchanges = len(changeentries) shouter.shoutwithdate("Start accepting %s changesets" % amountofchanges) amountofacceptedchanges = 0 for changeEntry in changeentries: amountofacceptedchanges += 1 if not changeEntry.isAccepted(): # change could already be accepted from a retry if not Changes.accept(self.acceptlogpath, changeEntry): shouter.shout("Change wasnt succesfully accepted into workspace") # self.retryacceptincludingnextchangesets(changeEntry, changeentries) if not Differ.has_diff(): # no differences found - force reload of the workspace WorkspaceHandler().load() shouter.shout("Accepted change %d/%d into working directory" % (amountofacceptedchanges, amountofchanges)) Commiter.addandcommit(changeEntry) return amountofacceptedchanges
def getoutput(command, stripped=True): shout_command_to_log(command) try: outputasbytestring = check_output(command, shell=True) output = outputasbytestring.decode(sys.stdout.encoding).splitlines() except CalledProcessError as e: shouter.shout(e) output = "" if not stripped: return output else: lines = [] for line in output: strippedline = line.strip() if strippedline: lines.append(strippedline) return lines
def retryacceptincludingnextchangesets(self, change, changes): issuccessful = False changestoaccept = ImportHandler.collect_changes_to_accept_to_avoid_conflicts(change, changes, self.config.maxchangesetstoaccepttogether) amountofchangestoaccept = len(changestoaccept) if amountofchangestoaccept > 1: Changes.tostring(*changestoaccept) if self.config.useautomaticconflictresolution or self.is_user_agreeing_to_accept_next_change(change): shouter.shout("Trying to resolve conflict by accepting multiple changes") for index in range(1, amountofchangestoaccept): toaccept = changestoaccept[0:index + 1] # accept least possible amount of changes if Changes.accept(self.acceptlogpath, *toaccept): issuccessful = True break # ++++ check ++++ #else: # Changes.discard(*toaccept) # revert initial state if not issuccessful: self.is_user_aborting(change)
def migrate(): config = configuration.read() rtc = ImportHandler(config) rtcworkspace = WorkspaceHandler(config) git = Commiter initialize(config) streamuuids = config.streamuuids for streamuuid in streamuuids: componentbaselineentries = rtc.getcomponentbaselineentriesfromstream( streamuuid) streamname = config.streamnames[streamuuids.index(streamuuid)] rtcworkspace.setnewflowtargets(streamuuid) git.branch(streamname) history = rtc.readhistory(componentbaselineentries, streamname) changeentries = rtc.getchangeentriesofstreamcomponents( componentbaselineentries) rtc.acceptchangesintoworkspace( rtc.getchangeentriestoaccept(changeentries, history)) shouter.shout("All changes of components of stream '%s' accepted" % streamname) git.pushbranch(streamname) rtcworkspace.setcomponentstobaseline(componentbaselineentries, streamuuid) rtcworkspace.load() changeentries = rtc.getchangeentriesofstream(streamuuid) rtc.acceptchangesintoworkspace( rtc.getchangeentriestoaccept(changeentries, history)) git.pushbranch(streamname) shouter.shout( "All changes of stream '%s' accepted - Migration of stream completed" % streamname) morestreamstomigrate = streamuuids.index(streamuuid) + 1 is not len( streamuuids) if morestreamstomigrate: git.checkout("master") rtcworkspace.recreateoldestworkspace()
def summary(streamname): config = configuration.get() shouter.shout("\nAll changes accepted - Migration of stream '%s' is completed." "\nYou can distribute the git-repo '%s'." % (streamname, config.gitRepoName)) if len(config.ignorefileextensions) > 0: # determine and log the ignored but still present files os.chdir(config.workDirectory) os.chdir(config.clonedGitRepoName) pathtoclonedgitrepo = config.workDirectory + os.sep + config.clonedGitRepoName if pathtoclonedgitrepo[-1:] != os.sep: pathtoclonedgitrepo += os.sep ignoredbutexist = [] with open('.gitignore', 'r') as gitignore: for line in gitignore.readlines(): line = line.strip() if line != ".jazz5" and line != ".metadata" and line != ".jazzShed": pathtoignored = pathtoclonedgitrepo + line if os.path.exists(pathtoignored): ignoredbutexist.append(line) if len(ignoredbutexist) > 0: shouter.shout("\nThe following files have been ignored in the new git repository, " + "but still exist in the actual RTC workspace:") ignoredbutexist.sort() for ignored in ignoredbutexist: shouter.shout("\t" + ignored)
def determineinitialbaseline(self, stream): regex = "\(_\w+\)" pattern = re.compile(regex) config = self.config componentbaselinesentries = self.getcomponentbaselineentriesfromstream(stream) for entry in componentbaselinesentries: shouter.shout("Determine initial baseline of " + entry.componentname) # use always scm, lscm fails when specifying maximum over 10k command = "scm --show-alias n --show-uuid y list baselines --components %s -r %s -u %s -p %s -m 20000" % \ (entry.component, config.user, config.password, config.repo) baselineslines = shell.getoutput(command) baselineslines.reverse() # reverse to have earliest baseline on top for baselineline in baselineslines: matcher = pattern.search(baselineline) if matcher: matchedstring = matcher.group() uuid = matchedstring[1:-1] entry.baseline = uuid entry.baselinename = "Automatically detected initial baseline" break return componentbaselinesentries
def migrate(): rtc = ImportHandler() rtcworkspace = WorkspaceHandler() git = Commiter if existsrepo(): resume() else: initialize() config = configuration.get() streamuuid = config.streamuuid streamname = config.streamname branchname = streamname + "_branchpoint" componentbaselineentries = rtc.getcomponentbaselineentriesfromstream(streamuuid) rtcworkspace.setnewflowtargets(streamuuid) history = rtc.readhistory(componentbaselineentries, streamname) changeentries = rtc.getchangeentriesofstreamcomponents(componentbaselineentries) if len(changeentries) > 0: git.branch(branchname) rtc.acceptchangesintoworkspace(rtc.getchangeentriestoaccept(changeentries, history)) shouter.shout("All changes until creation of stream '%s' accepted" % streamname) git.pushbranch(branchname) rtcworkspace.setcomponentstobaseline(componentbaselineentries, streamuuid) rtcworkspace.load() git.branch(streamname) changeentries = rtc.getchangeentriesofstream(streamuuid) amountofacceptedchanges = rtc.acceptchangesintoworkspace(rtc.getchangeentriestoaccept(changeentries, history)) if amountofacceptedchanges > 0: git.pushbranch(streamname) git.promotebranchtomaster(streamname) RTCLogin.logout() summary(streamname)
def acceptchangesintoworkspace(self, changeentries): amountofchanges = len(changeentries) shouter.shoutwithdate("Start accepting %s changesets" % amountofchanges) amountofacceptedchanges = 0 changestoskip = 0 reloaded = False for changeEntry in changeentries: amountofacceptedchanges += 1 if changestoskip > 0: shouter.shout("Skipping " + changeEntry.tostring()) changestoskip -= 1 continue acceptedsuccesfully = Changes.accept(self.config, self.acceptlogpath, changeEntry) is 0 if not acceptedsuccesfully: shouter.shout("Change wasnt succesfully accepted into workspace") changestoskip = self.retryacceptincludingnextchangesets(changeEntry, changeentries) elif not reloaded: if self.is_reloading_necessary(): WorkspaceHandler(self.config).load() reloaded = True shouter.shout("Accepted change %s/%s into working directory" % (amountofacceptedchanges, amountofchanges)) Commiter.addandcommit(changeEntry)
def initialcommit(): shouter.shout("Initial git add") shell.execute("git add -A", os.devnull) shouter.shout("Finished initial git add, starting commit") shell.execute("git commit -m %s -q" % shell.quote("Initial Commit")) shouter.shout("Finished initial commit")