def lcb(self, text):
l = len(text)
if l and text[l-1] == '\n':
l = l - 1
if l and text[l-1] == '\r':
l = l - 1
outmsg.verbose(text[:l])
def find_distribution_rna(self, locations):
for dir in locations:
if not self.cvs_dir_check(dir):
continue
dir = dir +".rna"
outmsg.verbose("Looking for %s in %s (don't worry if this fails)" % (self.cvs_path, dir))
try:
cvs.Checkout(self.cvs_tag,
dir,
self.cvs_root,
os.path.dirname(os.path.join(tmpdir(),dir)),
self.cvs_timestamp)
except cvs.cvs_error: ## Ignore cvs errors, report them as missing distributions instead
return
dir = os.path.join(os.curdir, tmpdir(), dir)
try:
print "DOES %s exist????" % dir
if os.path.isfile(dir):
return dir
except OSError:
pass
return None
def find_distribution_cvs(self, locations):
log.trace( 'entry' , [ locations ] )
for dir in locations:
log.debug( 'Trying to find %s' % dir )
if not self.cvs_dir_check(dir):
log.debug( 'Failed cvs_dir_check on "%s" - skipping.' % dir )
continue
# Once we have a dir, let's do something with it.
message = "Looking for %s in %s (don't worry if this fails)" % (self.cvs_path, dir)
log.debug( message )
outmsg.verbose( message )
try:
cvs.Checkout(self.cvs_tag,
dir,
self.cvs_root,
os.path.join(tmpdir(),dir),
self.cvs_timestamp)
except cvs.cvs_error: ## Ignore cvs errors, report them as missing distributions instead
log.warn( 'Failed to check out %s - skipping' % dir )
return
dir = os.path.join(os.curdir, tmpdir(), dir)
try:
for f in os.listdir(dir):
if string.lower(f) not in ["cvs"]:
return dir
except OSError:
pass
log.trace( 'exit' )
return None
def find_distribution_rna(self, locations):
for dir in locations:
if not self.cvs_dir_check(dir):
continue
dir = dir + ".rna"
outmsg.verbose("Looking for %s in %s (don't worry if this fails)" %
(self.cvs_path, dir))
try:
cvs.Checkout(self.cvs_tag, dir, self.cvs_root,
os.path.dirname(os.path.join(tmpdir(), dir)),
self.cvs_timestamp)
except cvs.cvs_error: ## Ignore cvs errors, report them as missing distributions instead
return
dir = os.path.join(os.curdir, tmpdir(), dir)
try:
print "DOES %s exist????" % dir
if os.path.isfile(dir):
return dir
except OSError:
pass
return None
def lcb(self, text):
l = len(text)
if l and text[l - 1] == '\n':
l = l - 1
if l and text[l - 1] == '\r':
l = l - 1
outmsg.verbose(text[:l])
def find_distribution_cvs(self, locations):
log.trace('entry', [locations])
for dir in locations:
log.debug('Trying to find %s' % dir)
if not self.cvs_dir_check(dir):
log.debug('Failed cvs_dir_check on "%s" - skipping.' % dir)
continue
# Once we have a dir, let's do something with it.
message = "Looking for %s in %s (don't worry if this fails)" % (
self.cvs_path, dir)
log.debug(message)
outmsg.verbose(message)
try:
cvs.Checkout(self.cvs_tag, dir, self.cvs_root,
os.path.join(tmpdir(), dir), self.cvs_timestamp)
except cvs.cvs_error: ## Ignore cvs errors, report them as missing distributions instead
log.warn('Failed to check out %s - skipping' % dir)
return
dir = os.path.join(os.curdir, tmpdir(), dir)
try:
for f in os.listdir(dir):
if string.lower(f) not in ["cvs"]:
return dir
except OSError:
pass
log.trace('exit')
return None
def run_cmd(self, cmd):
outmsg.verbose(cmd)
t = time.time()
try:
ret = shell.run(cmd, self.lcb)
except shell.error, se:
e = err.Error()
e.Set(str(se))
raise err.error, e
def run_cmd(self, cmd):
outmsg.verbose(cmd)
t = time.time()
try:
ret=shell.run(cmd, self.lcb)
except shell.error, se:
e = err.Error()
e.Set(str(se))
raise err.error, e
def make(self, arg = ''):
old_dir = os.getcwd()
status = 0
result = ''
path = macfs.FSSpec('Makefile').as_pathname()
## ALL
if arg == '':
script = ascript.CreateAppleScript(
'set errtext to ""',
'set scriptobj to (load script file "%s")' % (path),
'tell scriptobj',
' with timeout of 99999 seconds',
' run',
' set errtext to the result',
' end timeout',
'end tell',
'return errtext')
result = script.CompileAndExecute()
else:
self.error('unsupported MAKE argument')
## so that it doesn't equal None
if result == None:
result = ''
## process output from AppleScript
result = string.translate(result, string.maketrans('\r', '\n'))
result = string.strip(result)
result = string.replace(result,"\n\n","\n")
## strip off the stupid quotes
if len(result) >= 1:
if result[0] == '"' and result[-1] == '"':
result = string.strip(result[1:-1])
outmsg.verbose(result)
## Scan the output for error messages
for line in string.split(result,"\n"):
if not line: ## empty line
continue
words=string.split(line)
if words[1] == "Warning": ## Warning
continue
## Anything else is an error
status = 1
self.error('failed make')
os.chdir(old_dir)
return status, result
def make(self, arg=''):
old_dir = os.getcwd()
status = 0
result = ''
path = macfs.FSSpec('Makefile').as_pathname()
## ALL
if arg == '':
script = ascript.CreateAppleScript(
'set errtext to ""',
'set scriptobj to (load script file "%s")' % (path),
'tell scriptobj', ' with timeout of 99999 seconds', ' run',
' set errtext to the result', ' end timeout', 'end tell',
'return errtext')
result = script.CompileAndExecute()
else:
self.error('unsupported MAKE argument')
## so that it doesn't equal None
if result == None:
result = ''
## process output from AppleScript
result = string.translate(result, string.maketrans('\r', '\n'))
result = string.strip(result)
result = string.replace(result, "\n\n", "\n")
## strip off the stupid quotes
if len(result) >= 1:
if result[0] == '"' and result[-1] == '"':
result = string.strip(result[1:-1])
outmsg.verbose(result)
## Scan the output for error messages
for line in string.split(result, "\n"):
if not line: ## empty line
continue
words = string.split(line)
if words[1] == "Warning": ## Warning
continue
## Anything else is an error
status = 1
self.error('failed make')
os.chdir(old_dir)
return status, result
def find_distribution_filesystem(self, locations):
if not os.path.exists(self.natify_path(self.cvs_base)):
return None
for dir in locations:
dir = self.natify_path(dir)
outmsg.verbose("Looking for %s in %s" % (self.cvs_path, dir))
if os.path.isdir(dir) and os.listdir(dir):
return dir
if os.path.isfile(dir + ".rna"):
return dir + ".rna"
return None
def find_distribution_filesystem(self, locations):
if not os.path.exists(self.natify_path(self.cvs_base)):
return None
for dir in locations:
dir=self.natify_path(dir)
outmsg.verbose("Looking for %s in %s" % (self.cvs_path, dir))
if os.path.isdir(dir) and os.listdir(dir):
return dir
if os.path.isfile(dir+".rna"):
return dir+".rna"
return None
def parse_makefile(self, basedir, mf_name, file_hash, src_root):
#print "PARSE: %s" % mf_name
remote_files = string.split(
open(os.path.join(basedir, mf_name + "-mkdep.o"), "r").read(),
"\n")
def mkrel(path, basedir=basedir, src_root=src_root):
x = os.path.normpath(os.path.join(basedir, path))
#print x
#print src_root
if string.lower(x[:len(src_root)]) == string.lower(src_root):
return x[len(src_root):]
return None
relpath = mkrel(basedir)
## FIXME
to_do = 1
t = time.time()
fix_makefile = 0
for ofile in remote_files:
file = os.path.join(basedir, ofile)
#print "FILE: %s" % file
if os.path.isdir(file) and os.listdir(file):
continue
nfile = mkrel(ofile)
if nfile:
#print "HASH %s" % file
file_hash[nfile] = 1
if not fix_makefile and os.path.isabs(ofile):
fix_makefile = 1
print "FIXME: %s => %s" % (ofile, nfile)
if fix_makefile:
to = string.join([os.pardir] * len(string.split(relpath, os.sep)),
os.sep) + os.sep
outmsg.verbose(
"Trying to fix absolute paths in %s by replacing %s with %s" %
(mf_name, src_root, to))
mf = open(os.path.join(basedir, mf_name), "r").read()
mf = string.replace(mf, src_root, to)
open(os.path.join(basedir, mf_name), "w").write(mf)
#print "Loop3, %f seconds" % (time.time() - t)
return (to_do, src_root, relpath)
class Compile(Collate):
"""Base compiler class, implements compile functionality common to
all platforms."""
## Default stage unless "set_stage" has been called
stage = 0
def is_builtin_compile_plugin(self):
return 1
def lcb(self, text):
l = len(text)
if l and text[l - 1] == '\n':
l = l - 1
if l and text[l - 1] == '\r':
l = l - 1
outmsg.verbose(text[:l])
def run_cmd(self, cmd):
outmsg.verbose(cmd)
t = time.time()
try:
ret = shell.run(cmd, self.lcb)
except shell.error, se:
e = err.Error()
e.Set(str(se))
raise err.error, e
outmsg.verbose("Time used: %.2f seconds" % (time.time() - t))
return ret
def build(self):
## Makefile Generation: run umake if there is no Makefile
## under the "no_umake_mode_flag",
if (not os.path.isfile("Makefile") or \
not self.settings.get("no_umake_mode_flag")) and \
self.stage in [0,1]:
self.umake()
if chaingang.current_job.Get().refs > -1:
outmsg.verbose("Missing BIF dependencies detected, will return to this module later.")
return
if not self.settings.get("umake_only_flag"):
self.build_common()
def build(self):
## Makefile Generation: run umake if there is no Makefile
## under the "no_umake_mode_flag",
if (not os.path.isfile("Makefile") or \
not self.settings.get("no_umake_mode_flag")) and \
self.stage in [0,1]:
self.umake()
if chaingang.current_job.Get().refs > -1:
outmsg.verbose(
"Missing BIF dependencies detected, will return to this module later."
)
return
if not self.settings.get("umake_only_flag"):
self.build_common()
def parse_makefile(self, basedir, mf_name, file_hash, src_root):
#print "PARSE: %s" % mf_name
remote_files=string.split(open(os.path.join(basedir, mf_name+"-mkdep.o"),"r").read(),"\n")
def mkrel(path, basedir=basedir, src_root=src_root):
x=os.path.normpath(os.path.join(basedir, path))
#print x
#print src_root
if string.lower(x[:len(src_root)]) == string.lower(src_root):
return x[len(src_root):]
return None
relpath=mkrel(basedir)
## FIXME
to_do=1
t=time.time()
fix_makefile=0
for ofile in remote_files:
file=os.path.join(basedir, ofile)
#print "FILE: %s" % file
if os.path.isdir(file) and os.listdir(file):
continue
nfile=mkrel(ofile)
if nfile:
#print "HASH %s" % file
file_hash[nfile]=1
if not fix_makefile and os.path.isabs(ofile):
fix_makefile=1
print "FIXME: %s => %s" % (ofile, nfile)
if fix_makefile:
to=string.join([os.pardir] * len(string.split(relpath,os.sep)),os.sep) + os.sep
outmsg.verbose("Trying to fix absolute paths in %s by replacing %s with %s" % (mf_name, src_root, to))
mf=open(os.path.join(basedir, mf_name),"r").read()
mf=string.replace(mf, src_root, to)
open(os.path.join(basedir, mf_name),"w").write(mf)
#print "Loop3, %f seconds" % (time.time() - t)
return (to_do, src_root, relpath)
def checkout(self,
tag,
module_list,
az=None,
timestamp=None,
nonrecursive=0,
checkout_dir=None):
log.trace(
'entry',
[tag, module_list, az, timestamp, nonrecursive, checkout_dir])
import distributions
if not checkout_dir:
checkout_dir = "."
if az:
tmpdirbase = os.path.join(checkout_dir, os.path.dirname(az), "cvs")
else:
tmpdirbase = os.path.join(checkout_dir, "cvs")
distributions.setup(tmpdirbase)
tmpdir = distributions.tmpdir(tmpdirbase)
if az:
t = os.path.join(tmpdir, "tmp")
self.update(tag, module_list, t, timestamp, nonrecursive)
if os.path.exists(t):
utils.mkdirTree(os.path.dirname(az))
outmsg.verbose("Moving %s to %s" % (t, az))
shell.move(t, az)
else:
out_dirs = self.update(tag, module_list, az, timestamp,
nonrecursive, tmpdir)
#print out_dirs
for d in out_dirs:
rl = os.path.join(checkout_dir, d[len(tmpdir) + 1:])
if os.path.exists(d):
outmsg.verbose("Moving %s to %s" % (d, rl))
utils.mkdirTree(os.path.dirname(rl))
if d[len(tmpdir) + 1:] == ".":
for tmp in os.listdir(d):
outmsg.verbose(
"Moving %s to %s" %
(os.path.join(d, tmp), os.path.join(rl, tmp)))
shell.move(os.path.join(d, tmp),
os.path.join(rl, tmp))
else:
shell.move(d, rl)
else:
print "%s does not exists" % d
distributions.cleanup(tmpdirbase)
log.trace('exit')
def checkout(self, tag, module_list, az = None, timestamp = None, nonrecursive = 0, checkout_dir = None):
log.trace( 'entry' , [ tag , module_list , az , timestamp , nonrecursive , checkout_dir ] )
import distributions
if not checkout_dir:
checkout_dir="."
if az:
tmpdirbase=os.path.join(checkout_dir, os.path.dirname(az), "cvs")
else:
tmpdirbase=os.path.join(checkout_dir, "cvs")
distributions.setup(tmpdirbase)
tmpdir=distributions.tmpdir(tmpdirbase)
if az:
t=os.path.join(tmpdir,"tmp")
self.update( tag, module_list,t , timestamp, nonrecursive)
if os.path.exists(t):
utils.mkdirTree(os.path.dirname(az))
outmsg.verbose("Moving %s to %s" % (t,az))
shell.move(t, az)
else:
out_dirs=self.update( tag, module_list, az, timestamp, nonrecursive, tmpdir)
#print out_dirs
for d in out_dirs:
rl=os.path.join(checkout_dir, d[len(tmpdir)+1:])
if os.path.exists(d):
outmsg.verbose("Moving %s to %s" % (d, rl))
utils.mkdirTree(os.path.dirname(rl))
if d[len(tmpdir)+1:] == ".":
for tmp in os.listdir(d):
outmsg.verbose("Moving %s to %s" % (os.path.join(d,tmp), os.path.join(rl, tmp)))
shell.move(os.path.join(d,tmp), os.path.join(rl, tmp))
else:
shell.move(d, rl)
else:
print "%s does not exists" % d
distributions.cleanup(tmpdirbase)
log.trace( 'exit' )
def line_cb(line):
outmsg.verbose(string.strip(line))
def line_cb(line):
outmsg.verbose(string.strip(line))
except err.error, e:
self.error(e.text)
raise err.error, e
except:
e = err.Error()
e.Set("Error in umake call")
e.SetTraceback(sys.exc_info())
self.error(e.text)
raise err.error, e
if result and len(result):
e = err.Error()
e.Set("umake error=\"%s\"" % (result))
raise err.error, e
outmsg.verbose("Time used: %.2f seconds" % (time.time() - t))
def make(self, arg = ""):
self.error('make needs to be implemented in a subclass')
def make_clean(self):
## don't do a make clean when doing a clobber build
if self.settings.get('clobber_mode_flag'):
return 0, ""
self.output("making clean")
(status, text) = self.make("clean")
return status, text
def make_depend(self):
self.output("making depend")
except err.error, e:
self.error(e.text)
raise err.error, e
except:
e = err.Error()
e.Set("Error in umake call")
e.SetTraceback(sys.exc_info())
self.error(e.text)
raise err.error, e
if result and len(result):
e = err.Error()
e.Set("umake error=\"%s\"" % (result))
raise err.error, e
outmsg.verbose("Time used: %.2f seconds" % (time.time() - t))
def make(self, arg=""):
self.error('make needs to be implemented in a subclass')
def make_clean(self):
## don't do a make clean when doing a clobber build
if self.settings.get('clobber_mode_flag'):
return 0, ""
self.output("making clean")
(status, text) = self.make("clean")
return status, text
def make_depend(self):
self.output("making depend")
def update(self,
tag,
module_list,
az=None,
timestamp=None,
nonrecursive=0,
checkout_dir=None):
# print "TAG = %s" % tag
# print "TIMESTAMP = %s" % timestamp
if checkout_dir == None:
checkout_dir = os.curdir
cmd = "cvs"
if self.root:
if self.shadow:
cmd = "%s -d %s" % (cmd, self.shadow)
else:
cmd = "%s -d %s" % (cmd, self.root)
cmd = "%s checkout" % cmd
if nonrecursive:
cmd = cmd + " -l"
if len(tag):
if tag == "HEAD":
cmd = "%s -A" % cmd
else:
cmd = '%s -r "%s"' % (cmd, tag)
if timestamp:
cmd = '%s -D "%s"' % (cmd, timestamp)
out_dirs = []
for x in module_list:
#print "%s" % repr( [checkout_dir] + string.split(x,"/"))
out_dirs.append(
apply(os.path.join, [checkout_dir] + string.split(x, "/")))
as_arg = ""
if az:
dir, base = os.path.split(az)
if dir:
utils.mkdirTree(dir)
checkout_dir = dir
cmd = "%s -d %s" % (cmd, base)
out_dirs = [az]
cmd = "%s %s" % (cmd, string.join(module_list))
outmsg.verbose(
"running %s in %s + %s (as = %s)" %
(repr(cmd), repr(os.getcwd()), repr(checkout_dir), repr(az)))
def line_cb(line):
outmsg.verbose(string.strip(line))
# Tenacious cvs checkout loop, tries <num_cvs_retries> times before
# failing.
for N in range(0, num_cvs_retries):
retcode, output = shell.run(cmd, line_cb, dir=checkout_dir)
if retcode:
# If we have a fatal error as determined by error output
# matching any of the following strings, then we will abort out
# of the retry loop.
if string.find(output, "cannot find module") != -1:
break
if string.find(output, "cannot expand modules") != -1:
break
if string.find(output,
"cannot open CVS/Entries for reading") != -1:
break
if string.find(output, "Cannot access ") != -1:
break
if string.find(output, "Can't parse date/time") != -1:
break
if string.find(
output,
"is modified but no longer in the repository") != -1:
break
if string.find(output, "\nC ") != -1:
break
# We didn't abort, so we'll assume non-fatal error, log the
# error, and try again.
log.info(
"CVS command failed with error code %d, trying again in a few seconds."
% retcode)
time.sleep(1 + N * 5)
else:
break
dirs = out_dirs[:]
for dir in dirs:
if os.path.isdir(dir):
## Create a timestamp file
timestamp = os.path.join(dir, "CVS", "timestamp")
shell.rm(timestamp)
try:
open(timestamp, "w").write(str(int(time.time())))
except IOError:
continue
if self.shadow:
for subdir in os.listdir(dir):
if string.lower(subdir) != "cvs":
subdir = os.path.join(dir, subdir)
if os.path.isdir(subdir):
dirs.append(subdir)
rootfile = os.path.join(dir, "CVS", "Root")
open(rootfile, "w").write("%s\n" % self.root)
#print "ODIRS: %s" % repr(out_dirs)
return out_dirs
def update(self, tag, module_list, az = None, timestamp = None, nonrecursive = 0, checkout_dir = None):
# print "TAG = %s" % tag
# print "TIMESTAMP = %s" % timestamp
if checkout_dir == None:
checkout_dir = os.curdir
cmd = "cvs"
if self.root:
if self.shadow:
cmd = "%s -d %s" % (cmd, self.shadow)
else:
cmd = "%s -d %s" % (cmd, self.root)
cmd = "%s checkout" % cmd
if nonrecursive:
cmd = cmd + " -l"
if len(tag):
if tag == "HEAD":
cmd = "%s -A" % cmd
else:
cmd = '%s -r "%s"' % (cmd, tag)
if timestamp:
cmd = '%s -D "%s"' % (cmd, timestamp)
out_dirs = []
for x in module_list:
#print "%s" % repr( [checkout_dir] + string.split(x,"/"))
out_dirs.append(apply(os.path.join, [checkout_dir] +
string.split(x,"/")))
as_arg = ""
if az:
dir, base = os.path.split(az)
if dir:
utils.mkdirTree(dir)
checkout_dir = dir
cmd = "%s -d %s" % (cmd , base)
out_dirs = [az]
cmd = "%s %s" % (cmd, string.join(module_list))
outmsg.verbose("running %s in %s + %s (as = %s)" % (repr(cmd), repr(os.getcwd()), repr(checkout_dir), repr(az)))
def line_cb(line):
outmsg.verbose(string.strip(line))
# Tenacious cvs checkout loop, tries <num_cvs_retries> times before
# failing.
for N in range(0, num_cvs_retries):
retcode, output = shell.run(cmd, line_cb, dir = checkout_dir)
if retcode:
# If we have a fatal error as determined by error output
# matching any of the following strings, then we will abort out
# of the retry loop.
if string.find(output,"cannot find module") != -1:
break
if string.find(output,"cannot expand modules")!=-1:
break
if string.find(output,"cannot open CVS/Entries for reading")!=-1:
break
if string.find(output,"Cannot access ")!=-1:
break
if string.find(output,"Can't parse date/time")!=-1:
break
if string.find(output,"is modified but no longer in the repository")!=-1:
break
if string.find(output,"\nC ")!=-1:
break
# We didn't abort, so we'll assume non-fatal error, log the
# error, and try again.
log.info( "CVS command failed with error code %d, trying again in a few seconds." % retcode )
time.sleep(1 + N*5)
else:
break
dirs = out_dirs[:]
for dir in dirs:
if os.path.isdir(dir):
## Create a timestamp file
timestamp=os.path.join(dir, "CVS", "timestamp")
shell.rm(timestamp)
try:
open(timestamp,"w").write(str(int(time.time())))
except IOError:
continue
if self.shadow:
for subdir in os.listdir(dir):
if string.lower(subdir) != "cvs":
subdir=os.path.join(dir, subdir)
if os.path.isdir(subdir):
dirs.append(subdir)
rootfile = os.path.join(dir, "CVS", "Root")
open(rootfile,"w").write("%s\n" % self.root)
#print "ODIRS: %s" % repr(out_dirs)
return out_dirs