def run(self, context):
"""
Calls the wrapped run method
"""
retries = None
retryNumber = 0 # 1=first retry, etc
self.target.retriesRemaining = int(self.target.options['Target.failureRetries']) # default is 0
backoffSecs = self.target.options['Target.failureRetriesInitialBackoffSecs']
while True:
try:
implicitInputs = self.__getImplicitInputs(context)
if implicitInputs or self.isDirPath:
deleteFile(self.__implicitInputsFile)
self.target.run(context)
if retryNumber > 0: self.target.log.warning('Target %s succeeded on retry #%d', self, retryNumber)
break
except Exception as ex:
if self.target.retriesRemaining == 0:
if retryNumber > 0:
self.target.log.warning('Target %s failed even after %d retries', self, retryNumber)
raise
self.target.retriesRemaining -= 1
retryNumber += 1
time.sleep(backoffSecs)
self.target.log.info('Target %s failed on attempt #%d, will now clean before retry', self, retryNumber)
# hopefully after the backoff time enough file handles will have been removed for the clean to succeed
try:
self.clean(context)
except Exception as ex:
self.target.log.error('Failed to cleanup during retry, after initial failure of %s', self)
raise
# this logic is to prevent CI (e.g. TeamCity) error messages from one retry from causing the whole job to be flagged as a
# failure even if a subsequent retry succeeds
buf = outputBufferingManager.resetStdoutBufferForCurrentThread()
self.target.log.warning('Target %s failed on attempt #%d, will retry after %d seconds backoff (see .log file for details).', self, retryNumber, backoffSecs)
# for now let's just throw away buf - user can look at the .log file to see what happened before the failure if they care
# (can't just re-log it here as that would result in duplication with the .log output)
backoffSecs *= 2
# we expect self.path to NOT be in the fileutils stat cache at this point;
# it's too costly to check this explictly, but unwanted incremental rebuilds
# can be caused if the path does get into the stat cache since it'll have a stale value
# if target built successfully, record what the implicit inputs were to help with the next up to date
# check and ensure incremental build is correct
if implicitInputs or self.isDirPath:
log.debug('writing implicitInputsFile: %s', self.__implicitInputsFile)
mkdir(os.path.dirname(self.__implicitInputsFile))
with openForWrite(self.__implicitInputsFile, 'wb') as f:
f.write(os.linesep.join(implicitInputs).encode('utf-8'))
def updateStampFile(self):
"""
.. private:: Not useful enough to be in the public API.
Assumes self.path is a stamp file that just needs creating / timestamp updating and does so """
path = normLongPath(self.path)
mkdir(os.path.dirname(path))
with openForWrite(path, 'wb') as f:
pass
def jar(path, manifest, sourcedir, options, preserveManifestFormatting=False, update=False, outputHandler=None): """ Create a jar file containing a manifest and some other files @param path: jar file to create. Typically this file does not already exist, but if it does then the specified files or manifest will be merged into it. @param manifest: path to the manifest.mf file (or None to disable manifest entirely) @param sourcedir: the directory to pack everything from (this method may add extra files to this dir) @param options: options map. jar.options is a list of additional arguments @param preserveManifestFormatting: an advanced option that prevents that jar executable from reformatting the specified manifest file to comply with Java conventions (also prevents manifest merging if jar already exists) """ # work out if we need to create a parent directory dir = os.path.dirname(path) if dir and not os.path.exists(dir): mkdir(dir) # location of jar if options['java.home']: binary = os.path.join(options['java.home'], "bin/jar") else: binary = "jar" # build up arguments args = [binary] args.extend(options['jar.options']) if update: mode='-u' else: mode='-c' if not manifest: args.extend([mode+"fM", path]) elif preserveManifestFormatting: mkdir(sourcedir+'/META-INF') srcf = normLongPath(sourcedir+'/META-INF/manifest.mf') with open(manifest, 'rb') as s: with openForWrite(srcf, 'wb') as d: d.write(s.read()) args.extend([mode+"f", path]) else: args.extend([mode+"fm", path, manifest]) if sourcedir: args.extend(["-C", sourcedir, "."]) # actually call jar call(args, outputHandler=outputHandler, timeout=options['process.timeout'])
def create_manifest(path, properties, options):
""" Create a manifest file in path from the map properties.
@param path: The absolute path in which to create a manifest file. If None is specified,
no file is written but a byte string containing the file contents is returned.
@param properties: A map of manifest keys to values (unicode character strings).
Must follow java jar file format specification, e.g. headers cannot contain spaces.
@param options: The options to use for creating the manifest (prefix: jar.manifest)
"""
# merge in the defaults to the map (properties will already have been expanded
fullmap = {}
for source in [options['jar.manifest.defaults'], properties]:
for key in source:
fullmap[key] = source[key]
# build up the list of lines
lines = []
for key in sorted(fullmap.keys()): # select a deterministic order
# strip whitespace since it isn't needed, and could confuse the continuation character logic
line = ("%s: %s") % (key.strip(), fullmap[key].strip())
assert ' ' not in key.strip(), 'manifest.mf key "%s" does not confirm to jar file specification - cannot contain spaces'%(key.strip())
assert '\n' not in line, repr(line)
# assume character strings as input; must convert to utf8 before applying continuation char logic
line = line.encode('utf-8')
# spec says: No line may be longer than 72 bytes (not characters), in its UTF8-encoded form
# If a value would make the initial line longer than this, it should be continued on extra lines (each starting with a single SPACE).
maxbytes = 72-1 # remove one byte for the newline
while len(line) > maxbytes:
lines.append(line[:maxbytes])
line = b" %s" % line[maxbytes:]
assert line
lines.append(line)
lines.append(b'') # need a trailing newline else Java ignores last line!
lines = b'\n'.join(lines) # don't use os.linesep for simplicity, and because spec says only \r\n, \n, \r are valid not \n\r
# write out the file
if path:
with openForWrite(path, 'wb') as f:
f.write(lines)
else: # this case for unit testing
return lines
def javadoc(path, sources, classpath, options, workDir, outputHandler):
""" Create javadoc from sources and a set of options
@param path: The directory under which to create the javadoc
@param sources: a list of source files
@param classpath: a list of jars for the classpath
@param options: the current set of options to use
@param workDir: where temporary files are stored
@param outputHandler: the output handler (optional)
"""
deleteDir(path)
mkdir(path)
# location of javadoc
if options['java.home']:
binary = os.path.join(options['java.home'], "bin/javadoc")
else:
binary = "javadoc"
# store the list of files in a temporary file, then build from that.
mkdir(workDir)
inputlistfile = os.path.join(workDir, "javadoc.inputs")
with openForWrite(inputlistfile, 'w', encoding=locale.getpreferredencoding()) as f:
f.writelines('"'+x.replace('\\','\\\\')+'"'+'\n' for x in sources)
# build up arguments
args = [binary]
args.extend(options['javadoc.options'])
if options['javadoc.ignoreSourceFilesFromClasspath']:
args.extend(['-sourcepath', path+'/xpybuild_fake_sourcepath'])
args.extend([
"-d", path,
"-classpath", classpath,
"-windowtitle", options['javadoc.title'],
"-doctitle", options['javadoc.title'],
"-%s" % options['javadoc.access'],
"@%s" % inputlistfile
])
# actually call javadoc
call(args, outputHandler=outputHandler, timeout=options['process.timeout'])
def _copyFile(self, context, src, dest):
with open(src, 'rb') as inp:
with openForWrite(dest, 'wb') as out:
shutil.copyfileobj(inp, out)
shutil.copymode(src, dest)
assert os.path.exists(dest)
def javac(output, inputs, classpath, options, logbasename, targetname, workDir):
""" Compile some java files to class files.
Will raise BuildException if compilation fails.
@param output: path to a directory in which to put the class files (will be created)
@param inputs: list of paths (.java files) to be compiled
@param classpath: classpath to compile with, as a string
@param options: options map. javac.options is a list of additional arguments, javac.source is the source version,
javac.target is the target version
@param logbasename: absolute, expanded, path to a directory and filename prefix
to use for files such as .err, .out, etc files
@param targetname: to log appropriate error messages
@param workDir: where temporary files are stored.
"""
assert logbasename and '$' not in logbasename
logbasename = os.path.normpath(logbasename)
# make the output directory
if not os.path.exists(output): mkdir(output)
# location of javac
if options['java.home']:
javacpath = os.path.join(options['java.home'], "bin/javac")
else:
javacpath = "javac" # just get it from the path
# store the list of files in a temporary file, then build from that.
mkdir(workDir)
argsfile = os.path.join(workDir, "javac_args.txt")
# build up the arguments
args = ["-d", output]
if options["javac.source"]: args.extend(["-source", options["javac.source"]])
if options["javac.target"]: args.extend(["-target", options["javac.target"]])
if options["javac.encoding"]: args.extend(["-encoding", options["javac.encoding"]])
if options["javac.debug"]:
args.append('-g')
if options['javac.warningsAsErrors']:
args.append('-Werror')
# TODO: should add -Xlint options here I think
args.extend(getStringList(options['javac.options']))
if classpath: args.extend(['-cp', classpath])
args.extend([x for x in inputs if x.endswith('.java')]) # automatically filter out non-java files
with openForWrite(argsfile, 'w', encoding=locale.getpreferredencoding()) as f:
for a in args:
f.write('"%s"'%a.replace('\\','\\\\')+'\n')
success=False
try:
log.info('Executing javac for %s, writing output to %s: %s', targetname, logbasename+'.out', ''.join(['\n\t"%s"'%x for x in [javacpath]+args]))
# make sure we have no old ones hanging around still
try:
deleteFile(logbasename+'-errors.txt', allowRetry=True)
deleteFile(logbasename+'-warnings.txt', allowRetry=True)
deleteFile(logbasename+'.out', allowRetry=True)
except Exception as e:
log.info('Cleaning up file failed: %s' % e)
outputHandler = options.get('javac.outputHandlerFactory', JavacProcessOutputHandler)(targetname, options=options)
if hasattr(outputHandler, 'setJavacLogBasename'):
outputHandler.setJavacLogBasename(logbasename)
call([javacpath, "@%s" % argsfile], outputHandler=outputHandler, outputEncoding='UTF-8', cwd=output, timeout=options['process.timeout'])
if (not os.listdir(output)): # unlikely, but useful failsafe
raise EnvironmentError('javac command failed to create any target files (but returned no error code); see output at "%s"'%(logbasename+'.out'))
success = True
finally:
if not success and classpath:
log.info('Classpath for failed javac was: \n %s', '\n '.join(classpath.split(os.pathsep)))
def handleEnd(self, returnCode=None):
assert self._logbasename # could make this optional, but for now don't
if self._contents:
with openForWrite(self._logbasename+'.out', 'w', encoding='utf-8') as fo:
fo.write(self._contents)
errs = []
warns = []
errmsg = None
for c in self._chunks:
loc = re.match('(.*\.java:\d+): *(.*)', c[0])
msg = re.match('.*\.java:\d+: *(.*)', c[0])
if msg and loc:
msg, loc = msg.group(1), loc.group(1) # key by message
else:
msg = c[0]
loc = None
# special-case merging for common multi-line messages to give better diagnostics
i = 1
while i < len(c):
if c[i].strip().startswith('symbol'):
msg += ': <'+re.search('symbol *: *(.*)', c[i]).group(1)+'>'
del c[i]
elif c[i].strip().startswith('location'):
msg += ' in <'+re.search('location *: *(.*)', c[i]).group(1)+'>'
del c[i]
else:
i += 1
# check returncode since we must never treat stderr output as an error if it returned success, as annotation processes can output all kinds of stuff
iswarning = returnCode==0 or (re.match('.*\.java:\d+: warning: .*', c[0]) or (len(c) > 1 and 'to suppress this warning' in c[1]))
addto = warns if iswarning else errs
existing = next((x[1] for x in addto if x[0] == msg), None)
if not existing:
existing = []
addto.append( (msg, existing) )
existing.append([loc+': '+msg if loc else msg]+c[1:])
if not iswarning and not errmsg: errmsg = msg+(' at %s'%loc if loc else '')
if errs:
with openForWrite(self._logbasename+'-errors.txt', 'w', encoding='utf-8') as fo:
for x in errs:
# x[0] is the common msg type, x[1] is a list of places where it occured, each of which
# in log, summarize the first error of this type (e.g. if a symbol wasn't found in 20 places just log it once)
filename = None
lineno = None
try:
filename = re.sub("^(.*):([0-9]+): .*",r"\1",x[1][0][0])
lineno = int(re.sub("^(.*):([0-9]+): .*",r"\2",x[1][0][0]))
except:
pass
self._log(logging.ERROR, '\njavac> '.join(x[1][0]),
filename,lineno)
print('- '+x[0], file=fo)
print(file=fo)
i = 0
for x2 in x[1]: # print the detail of later similar ones only at INFO
if i != 0:
self._log(logging.INFO, 'similar error: \n %s'%('\n '.join(x2)))
i+=1
self._errors.append(' / '.join(x2[0]))
for x3 in x2:
fo.write(x3)
print(file=fo)
print(file=fo)
self._log(logging.ERROR, '%d javac ERRORS in %s - see %s'%(sum([len(x[1]) for x in errs]), self._targetName, self._logbasename+'-errors.txt'),
self._logbasename+'-errors.txt')
publishArtifact('javac %s errors'%self._targetName, self._logbasename+'-errors.txt')
if warns:
self._log(logging.WARNING, '%d javac WARNINGS in %s - see %s; first is: %s'%(sum([len(x[1]) for x in warns]), self._targetName, self._logbasename+'-warnings.txt', warns[0][0]),
self._logbasename+'-warnings.txt')
with openForWrite(self._logbasename+'-warnings.txt', 'w', encoding='utf-8') as fo:
for x in warns:
if not errmsg and (returnCode != 0):
errmsg = x[1][0][0]
if len(warns)>1:
errmsg = 'Failed due to %d warnings, first is: %s'%(len(warns), errmsg)
# it IS worth publishing warnings if they caused a failure
publishArtifact('javac %s warnings'%self._targetName, self._logbasename+'-warnings.txt')
print('- '+x[0], file=fo)
print(file=fo)
for x2 in x[1]:
for x3 in x2:
print(x3, file=fo)
# might add an option to publish warnings as artifacts, but don't by default as it happens a lot on some projects
#_publishArtifact(self._logbasename+'-warnings.txt')
if errmsg:
msg = errmsg
if len(self._errors)>1:
msg = '%d errors, first is: %s'%(len(self._errors), errmsg)
elif returnCode:
msg = 'javac failed with return code %s'%(returnCode)
else:
assert not errs
return
publishArtifact('javac %s output'%self._targetName, self._logbasename+'.out')
raise BuildException(msg)
def run(self, context):
self.keystore = context.expandPropertyValues(self.keystore)
options = self.options
mkdir(self.path)
for src, dest in self.jars.resolveWithDestinations(context):
if '..' in dest:
# to avoid people abusing this to copy files outside the dest directory!
raise Exception('This target does not permit destination paths to contain ".." relative path expressions')
try:
with open(src, 'rb') as s:
with openForWrite(os.path.join(self.path, dest), 'wb') as d:
d.write(s.read())
shutil.copystat(src, os.path.join(self.path, dest))
# When we re-jar with the user specified manifest entries, jar will complain
# about duplicate attributes IF the original MANIFEST.MF already has those entries.
# This is happening for latest version of SL where Application-Name, Permission etc
# were already there.
#
# The block of code below will first extract the original MANIFEST.MF from the source
# jar file, read all manifest entry to a list. When constructing the new manifest entries,
# make sure the old MANIFEST.MF doesn't have that entry before putting the new manifest entry
# to the list. This will avoid the duplicate attribute error.
#
if self.manifestDefaults:
lines = []
# read each line of MANIFEST.MF of the original jar and put them in lines
with zipfile.ZipFile(src, 'r') as zf:
lst = zf.infolist()
for zi in lst:
fn = zi.filename
if fn.lower().endswith('manifest.mf'):
try:
manifest_txt = zf.read(zi.filename).decode('utf-8', errors='strict')
except Exception as e:
raise BuildException('Failed reading the manifest file %s with exception:%s' % (fn, e))
# if we have all manifest text, parse and save each line
if manifest_txt:
# CR LF | LF | CR can be there as line feed and hence the code below
lines = manifest_txt.replace('\r\n', '\n').replace('\r','\n').split('\n')
# done
break
original_entries = collections.OrderedDict() # to ensure we don't overwrite/duplicate these
# populate the manifest_entries with original values from original manifest
for l in lines:
if ':' in l and not l.startswith(' '): # ignore continuation lines etc because keys are all we care about
key,value = l.split(':', 1)
original_entries[key] = value.strip()
# build up a list of the new manifest entries (will be merged into any existing manifest by jar)
manifest_entries = collections.OrderedDict()
for i in self.manifestDefaults:
# if entry isn't there yet, add to the list
if i not in original_entries:
manifest_entries[i] = context.expandPropertyValues(self.manifestDefaults[i])
# create the manifest file
# we want to add the manifest entries explicitly specified here but
# NOT the 'default' manifest entries we usually add, since these
# are likely to have been set already, and we do not want duplicates
mkdir(self.workDir)
manifest = os.path.join(self.workDir, "MANIFEST.MF") # manifest file
options = dict(options)
options['jar.manifest.defaults'] = {}
create_manifest(manifest, manifest_entries, options)
# update the EXISTING jar file with the new manifest entries, which will be merged into
# existing manifest by the jar tool
jar(os.path.join(self.path, dest), manifest, None, options, update=True)
signjar(os.path.join(self.path, dest), self.keystore, options, alias=self.alias, storepass=self.storepass,
outputHandler=ProcessOutputHandler.create('signjars', treatStdErrAsErrors=False, options=options))
except BuildException as e:
raise BuildException('Error processing %s: %s'%(os.path.basename(dest), e))
def run(self, context):
options = self.options
# make sure temp dir exists
mkdir(self.workDir)
classes = os.path.join(self.workDir, "classes") # output dir for classes
# create the classpath, sorting within PathSet (for determinism), but retaining original order of
# PathSet elements in the list
classpath = os.pathsep.join(self.classpath.resolve(context))
# compile everything
mkdir(classes) # (need this for assembling other files to package later on, even if we don't do any javac)
if self.compile:
mkdir(self.getOption('javac.logs'))
javac(classes, self.compile.resolve(context), classpath, options=options, logbasename=options.get('javac.logs')+'/'+targetNameToUniqueId(self.name), targetname=self.name, workDir=self.workDir)
manifest = os.path.join(self.workDir, "MANIFEST.MF") # manifest file
if isinstance(self.manifest, str):
manifest = context.getFullPath(self.manifest, self.baseDir)
elif self.manifest == None:
manifest = None
else: # generate one
# rewrite property values in the manifest
manifest_entries = {}
for i in self.manifest:
manifest_entries[i] = context.expandPropertyValues(self.manifest[i])
# determine classpath for manifest
classpath_entries = []
if "Class-path" not in manifest_entries: # assuming it wasn't hardcoded, set it here
for src, dest in self.classpath.resolveWithDestinations(context):
# we definitely do want to support use of ".." in destinations here, it can be very useful
classpath_entries.append(dest)
assert isinstance(options['jar.manifest.classpathAppend'], list), options['jar.manifest.classpathAppend'] # must not be a string
classpath_entries.extend(options['jar.manifest.classpathAppend'] or [])
# need to always use / not \ for these to be valid
classpath_entries = [p.replace(os.path.sep, '/').replace('\\', '/') for p in classpath_entries if p]
if classpath_entries:
manifest_entries["Class-path"] = " ".join(classpath_entries) # include the classpath from here
if not manifest_entries.get('Class-path'): # suppress this element entirely if not needed, otherwise there would be no way to have an empty classpath
manifest_entries.pop('Class-path','')
# create the manifest file
create_manifest(manifest, manifest_entries, options=options)
# copy in the additional things to include
for (src, dest) in self.package.resolveWithDestinations(context):
if '..' in dest: raise Exception('This target does not permit packaged destination paths to contain ".." relative path expressions')
mkdir(os.path.dirname(os.path.join(classes, dest)))
destpath = normLongPath(classes+'/'+dest)
srcpath = normLongPath(src)
if os.path.isdir(srcpath):
mkdir(destpath)
else:
with open(srcpath, 'rb') as s:
with openForWrite(destpath, 'wb') as d:
d.write(s.read())
# create the jar
jar(self.path, manifest, classes, options=options, preserveManifestFormatting=self.preserveManifestFormatting,
outputHandler=ProcessOutputHandler.create('jar', treatStdErrAsErrors=False,options=options))