def cleanup(xtra=""):
#print "CLEANUP: %s" % tmpdir(xtra)
try:
shell.rm(tmpdir(xtra))
except:
for l in os.listdir(tmpdir(xtra)):
shell.rm(os.path.join(tmpdir(xtra),l))
def process_module(self, imodule):
if imodule.type in ["cvs", "distribution"]:
tmpdir="branch_tmp_"+str(distributions.my_get_thread_ident())
shell.rm(tmpdir)
try:
if imodule.type == "cvs":
imodule.checkout(
date = self.source_date,
tag = self.source_tag,
as = tmpdir)
else:
cvs.Checkout(self.source_tag or imodule.cvs_tag,
"distribution",
imodule.cvs_root,
tmpdir,
self.source_date or imodule.cvs_date)
cmd='cvs tag %s %s "%s"' %( self.cvs_flags,self.dash_b, self.tag )
print "Running: %s (in %s + %s)" % (cmd, os.getcwd(), tmpdir)
status, output = shell.run(cmd, dir = tmpdir)
print output
if status:
print "Tagging module %s failed\n" % imodule.id
except cvs.cvs_error:
print "Tagging module %s failed\n" % imodule.id
shell.rm(tmpdir)
def Checkout(tag,
module_list,
repository_name="",
az=None,
timestamp=None,
nonrecursive=0,
zap=None):
log.trace('entry',
[module_list, repository_name, az, timestamp, nonrecursive, zap])
module_list = listify(module_list)
if cvs_checkout_hook:
module_list = cvs_checkout_hook(tag, module_list, repository_name, az,
timestamp, nonrecursive)
if not module_list:
return
if zap:
for mod in module_list:
if az:
mod = az
if os.path.exists(mod):
import thread
print "DELETING: %s [%d] START" % (mod, thread.get_ident())
shell.rm(mod)
print "DELETING: %s [%d] END" % (mod, thread.get_ident())
log.trace('exit')
return Get(repository_name).Checkout(tag, module_list, az, timestamp,
nonrecursive)
def cleanup(xtra=""):
#print "CLEANUP: %s" % tmpdir(xtra)
try:
shell.rm(tmpdir(xtra))
except:
for l in os.listdir(tmpdir(xtra)):
shell.rm(os.path.join(tmpdir(xtra), l))
def Checkout(tag,
module_list,
repository_name = "",
az = None,
timestamp = None,
nonrecursive = 0,
zap = None):
log.trace( 'entry' , [ module_list , repository_name , az , timestamp , nonrecursive , zap ] )
module_list = listify(module_list)
if cvs_checkout_hook:
module_list = cvs_checkout_hook(tag, module_list, repository_name, az, timestamp, nonrecursive)
if not module_list:
return
if zap:
for mod in module_list:
if az:
mod = az
if os.path.exists(mod):
import thread
print "DELETING: %s [%d] START" % (mod, thread.get_ident())
shell.rm(mod)
print "DELETING: %s [%d] END" % (mod, thread.get_ident())
log.trace( 'exit' )
return Get(repository_name).Checkout(tag, module_list, az, timestamp, nonrecursive)
def cvs_dir_check(self, loc):
log.trace( 'entry' , [ loc ] )
global cvs_directory_cache
parts=string.split(loc, "/")
log.debug( 'Parts = %s' % parts )
x=len(parts)-2
while x > 0:
part = string.join(parts[:x],"/")
log.debug("CHECKING %s" % part )
# cvs_dir_cache holds the result of prior dir_check results.
# Thus, a key with a value of 0 means prior failure, which forces
# the early exit rather than a re-check.
if not self.cvs_dir_cache.get(part,1):
log.debug( 'Prior check reported failure on %s' % part )
log.trace( 'exit' , [ 0 ] )
return 0
# Here, prior success of the cache check means prior success, so
# we abort the loop and start the check.
if self.cvs_dir_cache.get(part,0):
log.debug('Prior check reported success on %s' % part )
break
x = x - 1
y=(len(parts) + x - 1)/2
if y <= x:
log.debug( 'y <= x - early exit' )
log.trace( 'exit' , [1] )
return 1
part = string.join(parts[:y],"/")
log.info( "TESTING %s" % part )
## Nonrecursive checkout
p=os.path.join(tmpdir(), "dir_cache_test")
try:
cvs.Checkout("", part, self.cvs_root,p, None, 1)
except cvs.cvs_error:
pass
# FIFME: This relies on the cvs client implementation. On Linux,
# checking out a dir with no files still puts a local dir and a CVS
# subdir. On Windows, it doesn't touch the local filesystem, so
# this returns True on Linux, and False on Windows.
ret=os.path.exists(p)
if ret:
shell.rm(p)
# This is where we store the results of the check into a cache to
# cut down on CVS server load.
self.cvs_dir_cache[part]=ret
log.info( "RESULT => %s" % repr(ret) )
log.trace( 'exit' , [ ret ] )
return ret
def cvs_dir_check(self, loc):
log.trace('entry', [loc])
global cvs_directory_cache
parts = string.split(loc, "/")
log.debug('Parts = %s' % parts)
x = len(parts) - 2
while x > 0:
part = string.join(parts[:x], "/")
log.debug("CHECKING %s" % part)
# cvs_dir_cache holds the result of prior dir_check results.
# Thus, a key with a value of 0 means prior failure, which forces
# the early exit rather than a re-check.
if not self.cvs_dir_cache.get(part, 1):
log.debug('Prior check reported failure on %s' % part)
log.trace('exit', [0])
return 0
# Here, prior success of the cache check means prior success, so
# we abort the loop and start the check.
if self.cvs_dir_cache.get(part, 0):
log.debug('Prior check reported success on %s' % part)
break
x = x - 1
y = (len(parts) + x - 1) / 2
if y <= x:
log.debug('y <= x - early exit')
log.trace('exit', [1])
return 1
part = string.join(parts[:y], "/")
log.info("TESTING %s" % part)
## Nonrecursive checkout
p = os.path.join(tmpdir(), "dir_cache_test")
try:
cvs.Checkout("", part, self.cvs_root, p, None, 1)
except cvs.cvs_error:
pass
# FIFME: This relies on the cvs client implementation. On Linux,
# checking out a dir with no files still puts a local dir and a CVS
# subdir. On Windows, it doesn't touch the local filesystem, so
# this returns True on Linux, and False on Windows.
ret = os.path.exists(p)
if ret:
shell.rm(p)
# This is where we store the results of the check into a cache to
# cut down on CVS server load.
self.cvs_dir_cache[part] = ret
log.info("RESULT => %s" % repr(ret))
log.trace('exit', [ret])
return ret
def rm(self, path):
shell.rm(self.native_path(path))
assertEqual(expected, list(zip(titles, values)))
# plagiarism
jplagLog = shell.pjoin(tmp, "jplag-log.txt")
shell.cp('Bar Foo_1234_assignsubmission_file_',
'Spam Egg_5678_assignsubmission_file')
shell.run(
f'{checkAssignments} jplag --mode separate --printDiff --minScore 100 | tee {jplagLog}'
)
assertFileContains(
jplagLog,
"File assignment_01.py is identical in Bar Foo_1234_assignsubmission_file_ and Spam Egg_5678_assignsubmission_file"
)
assertDirExists('jplag-results/python')
shell.rmdir('jplag-results', True)
shell.rm(jplagLog)
shell.run(
f'{checkAssignments} jplag --mode merged --printDiff --minScore 100 | tee {jplagLog}'
)
assertDirExists('jplag-results/python')
assertDirExists('jplag-results/java')
assertFileContains(
jplagLog,
"File assignment_01.py is identical in Bar Foo_1234_assignsubmission_file_ and Spam Egg_5678_assignsubmission_file"
)
assertFileContains(
jplagLog,
'File src/assignment_02/MyClass.java is identical in Bar Foo_1234_assignsubmission_file_ and Spam Egg_5678_assignsubmission_file'
)
def update(self,
tag,
module_list,
az=None,
timestamp=None,
nonrecursive=0,
dir=None):
### FIXME:
### test this!
if dir:
odir = os.getcwd()
try:
os.chdir(dir)
ret = self.checkout(tag, module_list, az, timestamp,
nonrecursive)
finally:
os.chdir(odir)
return ret
if az:
odir = os.getcwd()
shell.mkdir("cvs_temp")
os.chdir("cvs_temp")
self.checkout(tag, module_list, None, timestamp, nonrecursive)
os.chdir(odir)
for m in module_list:
shell.cp(os.path.join(odir, "cvs_temp", m),
os.path.join(odir, az))
shell.rm("cvs_temp")
return
if nonrecursive:
e = err.Error()
e.Set("Nonrecursive not supported by MacCVS")
raise err.error, e
session_name = self.cvs_session_path[1:-1]
session_name = os.path.basename(session_name)
script = ascript.CreateAppleScript(
'tell application %s' % (self.cvs_path), ' activate',
' open alias %s' % (self.cvs_session_path),
' set thesession to session named "%s"' % (session_name),
' set local root of thesession to alias "%s"' % (os.getcwd()),
' with timeout of 99999 seconds')
for module in module_list:
cmd = 'check out thesession module "%s"' % (module)
if tag:
cmd = cmd + ' revision "%s"' % (tag)
if timestamp:
cmd = cmd + ' date "%s"' % (timestamp)
script.Append(cmd)
script.Append(' end timeout', ' quit', 'end tell')
if self.script_save_path == '':
result = script.CompileAndExecute()
else:
script.CompileAndSave(self.script_save_path)
launch_script = ascript.CreateAppleScript(
'set scriptobj to (load script file "%s")' %
(self.script_save_path), 'tell scriptobj', ' run', 'end tell')
result = launch_script.CompileAndExecute()
if result and result != '':
outmsg.error('cvs checkout error %s' % (result))
def update(self,
tag,
module_list,
az=None,
timestamp=None,
nonrecursive=0,
checkout_dir=None):
# print "TAG = %s" % tag
# print "TIMESTAMP = %s" % timestamp
if checkout_dir == None:
checkout_dir = os.curdir
cmd = "cvs"
if self.root:
if self.shadow:
cmd = "%s -d %s" % (cmd, self.shadow)
else:
cmd = "%s -d %s" % (cmd, self.root)
cmd = "%s checkout" % cmd
if nonrecursive:
cmd = cmd + " -l"
if len(tag):
if tag == "HEAD":
cmd = "%s -A" % cmd
else:
cmd = '%s -r "%s"' % (cmd, tag)
if timestamp:
cmd = '%s -D "%s"' % (cmd, timestamp)
out_dirs = []
for x in module_list:
#print "%s" % repr( [checkout_dir] + string.split(x,"/"))
out_dirs.append(
apply(os.path.join, [checkout_dir] + string.split(x, "/")))
as_arg = ""
if az:
dir, base = os.path.split(az)
if dir:
utils.mkdirTree(dir)
checkout_dir = dir
cmd = "%s -d %s" % (cmd, base)
out_dirs = [az]
cmd = "%s %s" % (cmd, string.join(module_list))
outmsg.verbose(
"running %s in %s + %s (as = %s)" %
(repr(cmd), repr(os.getcwd()), repr(checkout_dir), repr(az)))
def line_cb(line):
outmsg.verbose(string.strip(line))
# Tenacious cvs checkout loop, tries <num_cvs_retries> times before
# failing.
for N in range(0, num_cvs_retries):
retcode, output = shell.run(cmd, line_cb, dir=checkout_dir)
if retcode:
# If we have a fatal error as determined by error output
# matching any of the following strings, then we will abort out
# of the retry loop.
if string.find(output, "cannot find module") != -1:
break
if string.find(output, "cannot expand modules") != -1:
break
if string.find(output,
"cannot open CVS/Entries for reading") != -1:
break
if string.find(output, "Cannot access ") != -1:
break
if string.find(output, "Can't parse date/time") != -1:
break
if string.find(
output,
"is modified but no longer in the repository") != -1:
break
if string.find(output, "\nC ") != -1:
break
# We didn't abort, so we'll assume non-fatal error, log the
# error, and try again.
log.info(
"CVS command failed with error code %d, trying again in a few seconds."
% retcode)
time.sleep(1 + N * 5)
else:
break
dirs = out_dirs[:]
for dir in dirs:
if os.path.isdir(dir):
## Create a timestamp file
timestamp = os.path.join(dir, "CVS", "timestamp")
shell.rm(timestamp)
try:
open(timestamp, "w").write(str(int(time.time())))
except IOError:
continue
if self.shadow:
for subdir in os.listdir(dir):
if string.lower(subdir) != "cvs":
subdir = os.path.join(dir, subdir)
if os.path.isdir(subdir):
dirs.append(subdir)
rootfile = os.path.join(dir, "CVS", "Root")
open(rootfile, "w").write("%s\n" % self.root)
#print "ODIRS: %s" % repr(out_dirs)
return out_dirs
def process_module(self, imodule):
if imodule.type in ["cvs", "distribution"]:
revisions=0
tmpdir="diff_tmp_"+str(distributions.my_get_thread_ident())
shell.rm(tmpdir)
try:
if imodule.type == "cvs":
imodule.checkout(
date = self.source_date,
tag = self.source_tag,
as = tmpdir)
else:
cvs.Checkout(self.source_tag or imodule.cvs_tag,
"distribution",
imodule.cvs_root,
tmpdir,
self.source_date or imodule.cvs_date)
tag = "-b"
try:
tag=string.strip(open(os.path.join(tmpdir,"CVS","Tag"),"r").read())
## Fixme: what if the sticky tag is a date??
if tag and tag[0] == "T":
tag = "-r"+tag[1:]
else:
tag = "-b"
except IOError:
pass
cmd='cvs log%s -N %s >cvs-log-temp-file' %( self.date_range,
tag )
print "Running: %s (in %s + %s)" % (cmd, os.getcwd(), tmpdir)
status, output = shell.run(cmd, dir = tmpdir)
print output
output=open(os.path.join(tmpdir,"cvs-log-temp-file"),"r").read()
output=string.replace(output,"\r\n","\n")
root="UNKNOWN"
try:
root=open(os.path.join(tmpdir,"CVS","Root"),"r").read()
root=string.strip(root)
except IOError:
pass
base_repository="UNKNOWN"
try:
base_repository=open(os.path.join(tmpdir,"CVS","Repository"),"r").read()
base_repository=string.strip(base_repository)
except IOError:
pass
viewcvs_base = cvs.GetViewCVSUrl(imodule.cvs_root,
imodule.cvs_path or
imodule.name)
blocks=string.split(output,
"=============================================================================")
for file in blocks[:-2]:
data=string.split(file,"----------------------------\nrevision ")
wf=string.split(data[0],"Working file: ",1)[1]
wf=string.split(wf,"\n",1)[0]
local_name=os.path.join(imodule.name, wf)
print " - %s, %d revisions" % (wf, len(data)-1)
viewcvs_url=None
if viewcvs_base:
viewcvs_url=posixpath.join(viewcvs_base,wf)
cvs_file=os.path.join(base_repository, wf)
for revision in data[1:]:
revdata=string.split(revision,";",3)
author=string.split(revdata[1])[1]
## Ignore build farm checkins
if author in [ "mserver", "drmbuild", "buildq", "codecbuild" ]:
continue
tmp=string.split(revdata[0])
rev=tmp[0]
t="%s %s" % (tmp[2],tmp[3])
t=time.strptime(t,"%Y/%m/%d %H:%M:%S")
t=time.mktime(t)-time.timezone
author=string.split(revdata[1])[1]
comment=string.split(revdata[3],"\n",1)[1]
if string.split(comment,":",1)[0] == "branches":
comment=string.split(comment,"\n",1)[1]
ci=Checkin(author, t, comment)
ci.add(File(local_name, rev,
root, cvs_file,
viewcvs_url))
self.checkins.append(ci)
revisions = revisions + 1
if status:
print "Diff module %s failed\n" % imodule.id
except cvs.cvs_error:
print "Diff module %s failed\n" % imodule.id
shell.rm(tmpdir)
print "%d revisions in %s" % (revisions, imodule.id)
def rm(self, path):
shell.rm(self.native_path(path))
def update(self, tag, module_list, az = None, timestamp = None, nonrecursive = 0, dir = None):
### FIXME:
### test this!
if dir:
odir=os.getcwd()
try:
os.chdir(dir)
ret=self.checkout(tag, module_list, az, timestamp, nonrecursive)
finally:
os.chdir(odir)
return ret
if az:
odir=os.getcwd()
shell.mkdir("cvs_temp")
os.chdir("cvs_temp")
self.checkout(tag,module_list, None, timestamp, nonrecursive)
os.chdir(odir)
for m in module_list:
shell.cp(os.path.join(odir, "cvs_temp", m),
os.path.join(odir, az))
shell.rm("cvs_temp")
return
if nonrecursive:
e = err.Error()
e.Set("Nonrecursive not supported by MacCVS")
raise err.error, e
session_name = self.cvs_session_path[1:-1]
session_name = os.path.basename(session_name)
script = ascript.CreateAppleScript(
'tell application %s' % (self.cvs_path),
' activate',
' open alias %s' % (self.cvs_session_path),
' set thesession to session named "%s"' % (session_name),
' set local root of thesession to alias "%s"' % (os.getcwd()),
' with timeout of 99999 seconds')
for module in module_list:
cmd = 'check out thesession module "%s"' % (module)
if tag:
cmd = cmd +' revision "%s"' % (tag)
if timestamp:
cmd = cmd +' date "%s"' % (timestamp)
script.Append(cmd)
script.Append(
' end timeout',
' quit',
'end tell')
if self.script_save_path == '':
result = script.CompileAndExecute()
else:
script.CompileAndSave(self.script_save_path)
launch_script = ascript.CreateAppleScript(
'set scriptobj to (load script file "%s")' % (
self.script_save_path),
'tell scriptobj',
' run',
'end tell')
result = launch_script.CompileAndExecute()
if result and result != '':
outmsg.error('cvs checkout error %s' % (result))
def update(self, tag, module_list, az = None, timestamp = None, nonrecursive = 0, checkout_dir = None):
# print "TAG = %s" % tag
# print "TIMESTAMP = %s" % timestamp
if checkout_dir == None:
checkout_dir = os.curdir
cmd = "cvs"
if self.root:
if self.shadow:
cmd = "%s -d %s" % (cmd, self.shadow)
else:
cmd = "%s -d %s" % (cmd, self.root)
cmd = "%s checkout" % cmd
if nonrecursive:
cmd = cmd + " -l"
if len(tag):
if tag == "HEAD":
cmd = "%s -A" % cmd
else:
cmd = '%s -r "%s"' % (cmd, tag)
if timestamp:
cmd = '%s -D "%s"' % (cmd, timestamp)
out_dirs = []
for x in module_list:
#print "%s" % repr( [checkout_dir] + string.split(x,"/"))
out_dirs.append(apply(os.path.join, [checkout_dir] +
string.split(x,"/")))
as_arg = ""
if az:
dir, base = os.path.split(az)
if dir:
utils.mkdirTree(dir)
checkout_dir = dir
cmd = "%s -d %s" % (cmd , base)
out_dirs = [az]
cmd = "%s %s" % (cmd, string.join(module_list))
outmsg.verbose("running %s in %s + %s (as = %s)" % (repr(cmd), repr(os.getcwd()), repr(checkout_dir), repr(az)))
def line_cb(line):
outmsg.verbose(string.strip(line))
# Tenacious cvs checkout loop, tries <num_cvs_retries> times before
# failing.
for N in range(0, num_cvs_retries):
retcode, output = shell.run(cmd, line_cb, dir = checkout_dir)
if retcode:
# If we have a fatal error as determined by error output
# matching any of the following strings, then we will abort out
# of the retry loop.
if string.find(output,"cannot find module") != -1:
break
if string.find(output,"cannot expand modules")!=-1:
break
if string.find(output,"cannot open CVS/Entries for reading")!=-1:
break
if string.find(output,"Cannot access ")!=-1:
break
if string.find(output,"Can't parse date/time")!=-1:
break
if string.find(output,"is modified but no longer in the repository")!=-1:
break
if string.find(output,"\nC ")!=-1:
break
# We didn't abort, so we'll assume non-fatal error, log the
# error, and try again.
log.info( "CVS command failed with error code %d, trying again in a few seconds." % retcode )
time.sleep(1 + N*5)
else:
break
dirs = out_dirs[:]
for dir in dirs:
if os.path.isdir(dir):
## Create a timestamp file
timestamp=os.path.join(dir, "CVS", "timestamp")
shell.rm(timestamp)
try:
open(timestamp,"w").write(str(int(time.time())))
except IOError:
continue
if self.shadow:
for subdir in os.listdir(dir):
if string.lower(subdir) != "cvs":
subdir=os.path.join(dir, subdir)
if os.path.isdir(subdir):
dirs.append(subdir)
rootfile = os.path.join(dir, "CVS", "Root")
open(rootfile,"w").write("%s\n" % self.root)
#print "ODIRS: %s" % repr(out_dirs)
return out_dirs
