def cleanup(xtra=""): #print "CLEANUP: %s" % tmpdir(xtra) try: shell.rm(tmpdir(xtra)) except: for l in os.listdir(tmpdir(xtra)): shell.rm(os.path.join(tmpdir(xtra),l))
def process_module(self, imodule): if imodule.type in ["cvs", "distribution"]: tmpdir="branch_tmp_"+str(distributions.my_get_thread_ident()) shell.rm(tmpdir) try: if imodule.type == "cvs": imodule.checkout( date = self.source_date, tag = self.source_tag, as = tmpdir) else: cvs.Checkout(self.source_tag or imodule.cvs_tag, "distribution", imodule.cvs_root, tmpdir, self.source_date or imodule.cvs_date) cmd='cvs tag %s %s "%s"' %( self.cvs_flags,self.dash_b, self.tag ) print "Running: %s (in %s + %s)" % (cmd, os.getcwd(), tmpdir) status, output = shell.run(cmd, dir = tmpdir) print output if status: print "Tagging module %s failed\n" % imodule.id except cvs.cvs_error: print "Tagging module %s failed\n" % imodule.id shell.rm(tmpdir)
def Checkout(tag, module_list, repository_name="", az=None, timestamp=None, nonrecursive=0, zap=None): log.trace('entry', [module_list, repository_name, az, timestamp, nonrecursive, zap]) module_list = listify(module_list) if cvs_checkout_hook: module_list = cvs_checkout_hook(tag, module_list, repository_name, az, timestamp, nonrecursive) if not module_list: return if zap: for mod in module_list: if az: mod = az if os.path.exists(mod): import thread print "DELETING: %s [%d] START" % (mod, thread.get_ident()) shell.rm(mod) print "DELETING: %s [%d] END" % (mod, thread.get_ident()) log.trace('exit') return Get(repository_name).Checkout(tag, module_list, az, timestamp, nonrecursive)
def cleanup(xtra=""): #print "CLEANUP: %s" % tmpdir(xtra) try: shell.rm(tmpdir(xtra)) except: for l in os.listdir(tmpdir(xtra)): shell.rm(os.path.join(tmpdir(xtra), l))
def Checkout(tag, module_list, repository_name = "", az = None, timestamp = None, nonrecursive = 0, zap = None): log.trace( 'entry' , [ module_list , repository_name , az , timestamp , nonrecursive , zap ] ) module_list = listify(module_list) if cvs_checkout_hook: module_list = cvs_checkout_hook(tag, module_list, repository_name, az, timestamp, nonrecursive) if not module_list: return if zap: for mod in module_list: if az: mod = az if os.path.exists(mod): import thread print "DELETING: %s [%d] START" % (mod, thread.get_ident()) shell.rm(mod) print "DELETING: %s [%d] END" % (mod, thread.get_ident()) log.trace( 'exit' ) return Get(repository_name).Checkout(tag, module_list, az, timestamp, nonrecursive)
def cvs_dir_check(self, loc): log.trace( 'entry' , [ loc ] ) global cvs_directory_cache parts=string.split(loc, "/") log.debug( 'Parts = %s' % parts ) x=len(parts)-2 while x > 0: part = string.join(parts[:x],"/") log.debug("CHECKING %s" % part ) # cvs_dir_cache holds the result of prior dir_check results. # Thus, a key with a value of 0 means prior failure, which forces # the early exit rather than a re-check. if not self.cvs_dir_cache.get(part,1): log.debug( 'Prior check reported failure on %s' % part ) log.trace( 'exit' , [ 0 ] ) return 0 # Here, prior success of the cache check means prior success, so # we abort the loop and start the check. if self.cvs_dir_cache.get(part,0): log.debug('Prior check reported success on %s' % part ) break x = x - 1 y=(len(parts) + x - 1)/2 if y <= x: log.debug( 'y <= x - early exit' ) log.trace( 'exit' , [1] ) return 1 part = string.join(parts[:y],"/") log.info( "TESTING %s" % part ) ## Nonrecursive checkout p=os.path.join(tmpdir(), "dir_cache_test") try: cvs.Checkout("", part, self.cvs_root,p, None, 1) except cvs.cvs_error: pass # FIFME: This relies on the cvs client implementation. On Linux, # checking out a dir with no files still puts a local dir and a CVS # subdir. On Windows, it doesn't touch the local filesystem, so # this returns True on Linux, and False on Windows. ret=os.path.exists(p) if ret: shell.rm(p) # This is where we store the results of the check into a cache to # cut down on CVS server load. self.cvs_dir_cache[part]=ret log.info( "RESULT => %s" % repr(ret) ) log.trace( 'exit' , [ ret ] ) return ret
def cvs_dir_check(self, loc): log.trace('entry', [loc]) global cvs_directory_cache parts = string.split(loc, "/") log.debug('Parts = %s' % parts) x = len(parts) - 2 while x > 0: part = string.join(parts[:x], "/") log.debug("CHECKING %s" % part) # cvs_dir_cache holds the result of prior dir_check results. # Thus, a key with a value of 0 means prior failure, which forces # the early exit rather than a re-check. if not self.cvs_dir_cache.get(part, 1): log.debug('Prior check reported failure on %s' % part) log.trace('exit', [0]) return 0 # Here, prior success of the cache check means prior success, so # we abort the loop and start the check. if self.cvs_dir_cache.get(part, 0): log.debug('Prior check reported success on %s' % part) break x = x - 1 y = (len(parts) + x - 1) / 2 if y <= x: log.debug('y <= x - early exit') log.trace('exit', [1]) return 1 part = string.join(parts[:y], "/") log.info("TESTING %s" % part) ## Nonrecursive checkout p = os.path.join(tmpdir(), "dir_cache_test") try: cvs.Checkout("", part, self.cvs_root, p, None, 1) except cvs.cvs_error: pass # FIFME: This relies on the cvs client implementation. On Linux, # checking out a dir with no files still puts a local dir and a CVS # subdir. On Windows, it doesn't touch the local filesystem, so # this returns True on Linux, and False on Windows. ret = os.path.exists(p) if ret: shell.rm(p) # This is where we store the results of the check into a cache to # cut down on CVS server load. self.cvs_dir_cache[part] = ret log.info("RESULT => %s" % repr(ret)) log.trace('exit', [ret]) return ret
def rm(self, path): shell.rm(self.native_path(path))
assertEqual(expected, list(zip(titles, values))) # plagiarism jplagLog = shell.pjoin(tmp, "jplag-log.txt") shell.cp('Bar Foo_1234_assignsubmission_file_', 'Spam Egg_5678_assignsubmission_file') shell.run( f'{checkAssignments} jplag --mode separate --printDiff --minScore 100 | tee {jplagLog}' ) assertFileContains( jplagLog, "File assignment_01.py is identical in Bar Foo_1234_assignsubmission_file_ and Spam Egg_5678_assignsubmission_file" ) assertDirExists('jplag-results/python') shell.rmdir('jplag-results', True) shell.rm(jplagLog) shell.run( f'{checkAssignments} jplag --mode merged --printDiff --minScore 100 | tee {jplagLog}' ) assertDirExists('jplag-results/python') assertDirExists('jplag-results/java') assertFileContains( jplagLog, "File assignment_01.py is identical in Bar Foo_1234_assignsubmission_file_ and Spam Egg_5678_assignsubmission_file" ) assertFileContains( jplagLog, 'File src/assignment_02/MyClass.java is identical in Bar Foo_1234_assignsubmission_file_ and Spam Egg_5678_assignsubmission_file' )
def update(self, tag, module_list, az=None, timestamp=None, nonrecursive=0, dir=None): ### FIXME: ### test this! if dir: odir = os.getcwd() try: os.chdir(dir) ret = self.checkout(tag, module_list, az, timestamp, nonrecursive) finally: os.chdir(odir) return ret if az: odir = os.getcwd() shell.mkdir("cvs_temp") os.chdir("cvs_temp") self.checkout(tag, module_list, None, timestamp, nonrecursive) os.chdir(odir) for m in module_list: shell.cp(os.path.join(odir, "cvs_temp", m), os.path.join(odir, az)) shell.rm("cvs_temp") return if nonrecursive: e = err.Error() e.Set("Nonrecursive not supported by MacCVS") raise err.error, e session_name = self.cvs_session_path[1:-1] session_name = os.path.basename(session_name) script = ascript.CreateAppleScript( 'tell application %s' % (self.cvs_path), ' activate', ' open alias %s' % (self.cvs_session_path), ' set thesession to session named "%s"' % (session_name), ' set local root of thesession to alias "%s"' % (os.getcwd()), ' with timeout of 99999 seconds') for module in module_list: cmd = 'check out thesession module "%s"' % (module) if tag: cmd = cmd + ' revision "%s"' % (tag) if timestamp: cmd = cmd + ' date "%s"' % (timestamp) script.Append(cmd) script.Append(' end timeout', ' quit', 'end tell') if self.script_save_path == '': result = script.CompileAndExecute() else: script.CompileAndSave(self.script_save_path) launch_script = ascript.CreateAppleScript( 'set scriptobj to (load script file "%s")' % (self.script_save_path), 'tell scriptobj', ' run', 'end tell') result = launch_script.CompileAndExecute() if result and result != '': outmsg.error('cvs checkout error %s' % (result))
def update(self, tag, module_list, az=None, timestamp=None, nonrecursive=0, checkout_dir=None): # print "TAG = %s" % tag # print "TIMESTAMP = %s" % timestamp if checkout_dir == None: checkout_dir = os.curdir cmd = "cvs" if self.root: if self.shadow: cmd = "%s -d %s" % (cmd, self.shadow) else: cmd = "%s -d %s" % (cmd, self.root) cmd = "%s checkout" % cmd if nonrecursive: cmd = cmd + " -l" if len(tag): if tag == "HEAD": cmd = "%s -A" % cmd else: cmd = '%s -r "%s"' % (cmd, tag) if timestamp: cmd = '%s -D "%s"' % (cmd, timestamp) out_dirs = [] for x in module_list: #print "%s" % repr( [checkout_dir] + string.split(x,"/")) out_dirs.append( apply(os.path.join, [checkout_dir] + string.split(x, "/"))) as_arg = "" if az: dir, base = os.path.split(az) if dir: utils.mkdirTree(dir) checkout_dir = dir cmd = "%s -d %s" % (cmd, base) out_dirs = [az] cmd = "%s %s" % (cmd, string.join(module_list)) outmsg.verbose( "running %s in %s + %s (as = %s)" % (repr(cmd), repr(os.getcwd()), repr(checkout_dir), repr(az))) def line_cb(line): outmsg.verbose(string.strip(line)) # Tenacious cvs checkout loop, tries <num_cvs_retries> times before # failing. for N in range(0, num_cvs_retries): retcode, output = shell.run(cmd, line_cb, dir=checkout_dir) if retcode: # If we have a fatal error as determined by error output # matching any of the following strings, then we will abort out # of the retry loop. if string.find(output, "cannot find module") != -1: break if string.find(output, "cannot expand modules") != -1: break if string.find(output, "cannot open CVS/Entries for reading") != -1: break if string.find(output, "Cannot access ") != -1: break if string.find(output, "Can't parse date/time") != -1: break if string.find( output, "is modified but no longer in the repository") != -1: break if string.find(output, "\nC ") != -1: break # We didn't abort, so we'll assume non-fatal error, log the # error, and try again. log.info( "CVS command failed with error code %d, trying again in a few seconds." % retcode) time.sleep(1 + N * 5) else: break dirs = out_dirs[:] for dir in dirs: if os.path.isdir(dir): ## Create a timestamp file timestamp = os.path.join(dir, "CVS", "timestamp") shell.rm(timestamp) try: open(timestamp, "w").write(str(int(time.time()))) except IOError: continue if self.shadow: for subdir in os.listdir(dir): if string.lower(subdir) != "cvs": subdir = os.path.join(dir, subdir) if os.path.isdir(subdir): dirs.append(subdir) rootfile = os.path.join(dir, "CVS", "Root") open(rootfile, "w").write("%s\n" % self.root) #print "ODIRS: %s" % repr(out_dirs) return out_dirs
def process_module(self, imodule): if imodule.type in ["cvs", "distribution"]: revisions=0 tmpdir="diff_tmp_"+str(distributions.my_get_thread_ident()) shell.rm(tmpdir) try: if imodule.type == "cvs": imodule.checkout( date = self.source_date, tag = self.source_tag, as = tmpdir) else: cvs.Checkout(self.source_tag or imodule.cvs_tag, "distribution", imodule.cvs_root, tmpdir, self.source_date or imodule.cvs_date) tag = "-b" try: tag=string.strip(open(os.path.join(tmpdir,"CVS","Tag"),"r").read()) ## Fixme: what if the sticky tag is a date?? if tag and tag[0] == "T": tag = "-r"+tag[1:] else: tag = "-b" except IOError: pass cmd='cvs log%s -N %s >cvs-log-temp-file' %( self.date_range, tag ) print "Running: %s (in %s + %s)" % (cmd, os.getcwd(), tmpdir) status, output = shell.run(cmd, dir = tmpdir) print output output=open(os.path.join(tmpdir,"cvs-log-temp-file"),"r").read() output=string.replace(output,"\r\n","\n") root="UNKNOWN" try: root=open(os.path.join(tmpdir,"CVS","Root"),"r").read() root=string.strip(root) except IOError: pass base_repository="UNKNOWN" try: base_repository=open(os.path.join(tmpdir,"CVS","Repository"),"r").read() base_repository=string.strip(base_repository) except IOError: pass viewcvs_base = cvs.GetViewCVSUrl(imodule.cvs_root, imodule.cvs_path or imodule.name) blocks=string.split(output, "=============================================================================") for file in blocks[:-2]: data=string.split(file,"----------------------------\nrevision ") wf=string.split(data[0],"Working file: ",1)[1] wf=string.split(wf,"\n",1)[0] local_name=os.path.join(imodule.name, wf) print " - %s, %d revisions" % (wf, len(data)-1) viewcvs_url=None if viewcvs_base: viewcvs_url=posixpath.join(viewcvs_base,wf) cvs_file=os.path.join(base_repository, wf) for revision in data[1:]: revdata=string.split(revision,";",3) author=string.split(revdata[1])[1] ## Ignore build farm checkins if author in [ "mserver", "drmbuild", "buildq", "codecbuild" ]: continue tmp=string.split(revdata[0]) rev=tmp[0] t="%s %s" % (tmp[2],tmp[3]) t=time.strptime(t,"%Y/%m/%d %H:%M:%S") t=time.mktime(t)-time.timezone author=string.split(revdata[1])[1] comment=string.split(revdata[3],"\n",1)[1] if string.split(comment,":",1)[0] == "branches": comment=string.split(comment,"\n",1)[1] ci=Checkin(author, t, comment) ci.add(File(local_name, rev, root, cvs_file, viewcvs_url)) self.checkins.append(ci) revisions = revisions + 1 if status: print "Diff module %s failed\n" % imodule.id except cvs.cvs_error: print "Diff module %s failed\n" % imodule.id shell.rm(tmpdir) print "%d revisions in %s" % (revisions, imodule.id)
def update(self, tag, module_list, az = None, timestamp = None, nonrecursive = 0, dir = None): ### FIXME: ### test this! if dir: odir=os.getcwd() try: os.chdir(dir) ret=self.checkout(tag, module_list, az, timestamp, nonrecursive) finally: os.chdir(odir) return ret if az: odir=os.getcwd() shell.mkdir("cvs_temp") os.chdir("cvs_temp") self.checkout(tag,module_list, None, timestamp, nonrecursive) os.chdir(odir) for m in module_list: shell.cp(os.path.join(odir, "cvs_temp", m), os.path.join(odir, az)) shell.rm("cvs_temp") return if nonrecursive: e = err.Error() e.Set("Nonrecursive not supported by MacCVS") raise err.error, e session_name = self.cvs_session_path[1:-1] session_name = os.path.basename(session_name) script = ascript.CreateAppleScript( 'tell application %s' % (self.cvs_path), ' activate', ' open alias %s' % (self.cvs_session_path), ' set thesession to session named "%s"' % (session_name), ' set local root of thesession to alias "%s"' % (os.getcwd()), ' with timeout of 99999 seconds') for module in module_list: cmd = 'check out thesession module "%s"' % (module) if tag: cmd = cmd +' revision "%s"' % (tag) if timestamp: cmd = cmd +' date "%s"' % (timestamp) script.Append(cmd) script.Append( ' end timeout', ' quit', 'end tell') if self.script_save_path == '': result = script.CompileAndExecute() else: script.CompileAndSave(self.script_save_path) launch_script = ascript.CreateAppleScript( 'set scriptobj to (load script file "%s")' % ( self.script_save_path), 'tell scriptobj', ' run', 'end tell') result = launch_script.CompileAndExecute() if result and result != '': outmsg.error('cvs checkout error %s' % (result))
def update(self, tag, module_list, az = None, timestamp = None, nonrecursive = 0, checkout_dir = None): # print "TAG = %s" % tag # print "TIMESTAMP = %s" % timestamp if checkout_dir == None: checkout_dir = os.curdir cmd = "cvs" if self.root: if self.shadow: cmd = "%s -d %s" % (cmd, self.shadow) else: cmd = "%s -d %s" % (cmd, self.root) cmd = "%s checkout" % cmd if nonrecursive: cmd = cmd + " -l" if len(tag): if tag == "HEAD": cmd = "%s -A" % cmd else: cmd = '%s -r "%s"' % (cmd, tag) if timestamp: cmd = '%s -D "%s"' % (cmd, timestamp) out_dirs = [] for x in module_list: #print "%s" % repr( [checkout_dir] + string.split(x,"/")) out_dirs.append(apply(os.path.join, [checkout_dir] + string.split(x,"/"))) as_arg = "" if az: dir, base = os.path.split(az) if dir: utils.mkdirTree(dir) checkout_dir = dir cmd = "%s -d %s" % (cmd , base) out_dirs = [az] cmd = "%s %s" % (cmd, string.join(module_list)) outmsg.verbose("running %s in %s + %s (as = %s)" % (repr(cmd), repr(os.getcwd()), repr(checkout_dir), repr(az))) def line_cb(line): outmsg.verbose(string.strip(line)) # Tenacious cvs checkout loop, tries <num_cvs_retries> times before # failing. for N in range(0, num_cvs_retries): retcode, output = shell.run(cmd, line_cb, dir = checkout_dir) if retcode: # If we have a fatal error as determined by error output # matching any of the following strings, then we will abort out # of the retry loop. if string.find(output,"cannot find module") != -1: break if string.find(output,"cannot expand modules")!=-1: break if string.find(output,"cannot open CVS/Entries for reading")!=-1: break if string.find(output,"Cannot access ")!=-1: break if string.find(output,"Can't parse date/time")!=-1: break if string.find(output,"is modified but no longer in the repository")!=-1: break if string.find(output,"\nC ")!=-1: break # We didn't abort, so we'll assume non-fatal error, log the # error, and try again. log.info( "CVS command failed with error code %d, trying again in a few seconds." % retcode ) time.sleep(1 + N*5) else: break dirs = out_dirs[:] for dir in dirs: if os.path.isdir(dir): ## Create a timestamp file timestamp=os.path.join(dir, "CVS", "timestamp") shell.rm(timestamp) try: open(timestamp,"w").write(str(int(time.time()))) except IOError: continue if self.shadow: for subdir in os.listdir(dir): if string.lower(subdir) != "cvs": subdir=os.path.join(dir, subdir) if os.path.isdir(subdir): dirs.append(subdir) rootfile = os.path.join(dir, "CVS", "Root") open(rootfile,"w").write("%s\n" % self.root) #print "ODIRS: %s" % repr(out_dirs) return out_dirs