def run(self, context):
"""
Calls the wrapped run method
"""
retries = None
retryNumber = 0 # 1=first retry, etc
self.target.retriesRemaining = int(self.target.options['Target.failureRetries']) # default is 0
backoffSecs = self.target.options['Target.failureRetriesInitialBackoffSecs']
while True:
try:
implicitInputs = self.__getImplicitInputs(context)
if implicitInputs or self.isDirPath:
deleteFile(self.__implicitInputsFile)
self.target.run(context)
if retryNumber > 0: self.target.log.warning('Target %s succeeded on retry #%d', self, retryNumber)
break
except Exception as ex:
if self.target.retriesRemaining == 0:
if retryNumber > 0:
self.target.log.warning('Target %s failed even after %d retries', self, retryNumber)
raise
self.target.retriesRemaining -= 1
retryNumber += 1
time.sleep(backoffSecs)
self.target.log.info('Target %s failed on attempt #%d, will now clean before retry', self, retryNumber)
# hopefully after the backoff time enough file handles will have been removed for the clean to succeed
try:
self.clean(context)
except Exception as ex:
self.target.log.error('Failed to cleanup during retry, after initial failure of %s', self)
raise
# this logic is to prevent CI (e.g. TeamCity) error messages from one retry from causing the whole job to be flagged as a
# failure even if a subsequent retry succeeds
buf = outputBufferingManager.resetStdoutBufferForCurrentThread()
self.target.log.warning('Target %s failed on attempt #%d, will retry after %d seconds backoff (see .log file for details).', self, retryNumber, backoffSecs)
# for now let's just throw away buf - user can look at the .log file to see what happened before the failure if they care
# (can't just re-log it here as that would result in duplication with the .log output)
backoffSecs *= 2
# we expect self.path to NOT be in the fileutils stat cache at this point;
# it's too costly to check this explictly, but unwanted incremental rebuilds
# can be caused if the path does get into the stat cache since it'll have a stale value
# if target built successfully, record what the implicit inputs were to help with the next up to date
# check and ensure incremental build is correct
if implicitInputs or self.isDirPath:
log.debug('writing implicitInputsFile: %s', self.__implicitInputsFile)
mkdir(os.path.dirname(self.__implicitInputsFile))
with openForWrite(self.__implicitInputsFile, 'wb') as f:
f.write(os.linesep.join(implicitInputs).encode('utf-8'))
def internal_clean(self, context): """ Calls the BaseTarget clean, not the target-specific clean """ try: deleteFile(self.__implicitInputsFile) except Exception: time.sleep(10.0) deleteFile(self.__implicitInputsFile) BaseTarget.clean(self.target, context)
def clean(self, context): """ Calls the wrapped clean method """ try: deleteFile(self.__implicitInputsFile) except Exception: time.sleep(10.0) deleteFile(self.__implicitInputsFile) self.target.clean(context)
def clean(self, context: xpybuild.buildcontext.BuildContext):
"""Called by xpybuild when the target should be deleted (can be overridden if needed).
The default implementation will simply delete the target, and any target
workdir, but can be overridden to delete additional temporary files if
needed (shouldn't be).
"""
try:
if self.workDir:
fileutils.deleteDir(self.workDir)
finally:
if os.path.isdir(self.path):
self.log.info('Target clean is deleting directory: %s',
self.path)
fileutils.deleteDir(self.path)
else:
fileutils.deleteFile(self.path)
def clean(self, context):
deleteFile(self._getMakeDependsFile(context))
BaseTarget.clean(self, context)
def _run_target(self, target):
"""
Run a single target, calling clean or run appropriately and counting the time taken
target - the TargetWrapper holding the target to build
number - how far through the build are we
total - the total number of targets to (potentially) build
Returns a list of error(s) encountered during the build
"""
errors = []
log.info("%s: executing", target.name)
starttime = time.time()
try:
if self.options["clean"]:
try:
target.clean(self.context)
except Exception as e:
errors.extend(
self._handle_error(target.target,
prefix='Target clean FAILED'))
else:
# must always clean before running, in case there's some incorrect junk around
# in output dir or work dir from a previous execution;
# doing this means we don't require a full clean when a target incremental build
# fails
try:
log.debug('%s: Performing pre-execution clean',
target.name)
target.internal_clean(
self.context
) # removes the target and work dir, but doesn't call target-specific clean
except Exception as e:
errors.extend(
self._handle_error(
target.target,
prefix='Target pre-execution clean FAILED'))
# run the target
try:
if not errors:
# if it's enabled, this is the point we want log statements
# to get buffered for writing to the output at the end of
# this target's execution
# (nb: this is after the initial "*** Building XXXX " log message... but that's
# actually useful in case of hanging targets, and anyway would be non-trivial to fix... the target end
# message is however included atomically with the target)
outputBufferingManager.startBufferingForCurrentThread()
log.debug('%s: executing run method for target',
target.name)
target.run(self.context)
except Exception as e:
errors.extend(self._handle_error(target.target))
# if it failed we MUST delete the stamp file so it rebuilds next time, but
# probably useful to not delete the target as it might help tracking
# down the error; and definitely don't want to nuke the work dir which often
# contains invaluable log files - so don't actually call clean here
try:
deleteFile(target.stampfile)
except Exception:
# hopefully won't happen ever
errors.extend(
self._handle_error(
target.target,
prefix=
'ERROR deleting target stampfile after target failure'
))
if self.options["verify"]:
for dep in target.target._resolveUnderlyingDependencies(
self.context, rawdeps=True):
dep = toLongPathSafe(dep)
if not os.path.exists(dep):
self.verificationerrors.append(
"%s verification error: Target dependency was deleted while it was being built: %s"
% (target, dep))
elif not isDirPath(dep):
# check not modified recently obviously must NOT use cached stat values here
# note that precision of this field is not very much on some OSes (hence checking > ceil not >=)
modtime = os.stat(dep).st_mtime
if modtime > math.ceil(starttime):
self.verificationerrors.append(
"%s verification error: Modification date of target dependency %s is %0.1fs after the target's start time (suggests that this target or some other has modified the file without making the dependency explicit)"
% (target, dep, modtime - starttime))
duration = time.time() - starttime
if not self.options["clean"]:
self.targetTimes[target.name] = (target.path, duration)
log.critical(
" %s done in %.1f seconds: %s",
target.name,
duration,
# it's useful to print the actual (relative) paths of them in case user wants to manually inspect them
os.path.relpath(target.path))
return errors
finally:
outputBufferingManager.endBufferingForCurrentThread()
def javac(output, inputs, classpath, options, logbasename, targetname, workDir):
""" Compile some java files to class files.
Will raise BuildException if compilation fails.
@param output: path to a directory in which to put the class files (will be created)
@param inputs: list of paths (.java files) to be compiled
@param classpath: classpath to compile with, as a string
@param options: options map. javac.options is a list of additional arguments, javac.source is the source version,
javac.target is the target version
@param logbasename: absolute, expanded, path to a directory and filename prefix
to use for files such as .err, .out, etc files
@param targetname: to log appropriate error messages
@param workDir: where temporary files are stored.
"""
assert logbasename and '$' not in logbasename
logbasename = os.path.normpath(logbasename)
# make the output directory
if not os.path.exists(output): mkdir(output)
# location of javac
if options['java.home']:
javacpath = os.path.join(options['java.home'], "bin/javac")
else:
javacpath = "javac" # just get it from the path
# store the list of files in a temporary file, then build from that.
mkdir(workDir)
argsfile = os.path.join(workDir, "javac_args.txt")
# build up the arguments
args = ["-d", output]
if options["javac.source"]: args.extend(["-source", options["javac.source"]])
if options["javac.target"]: args.extend(["-target", options["javac.target"]])
if options["javac.encoding"]: args.extend(["-encoding", options["javac.encoding"]])
if options["javac.debug"]:
args.append('-g')
if options['javac.warningsAsErrors']:
args.append('-Werror')
# TODO: should add -Xlint options here I think
args.extend(getStringList(options['javac.options']))
if classpath: args.extend(['-cp', classpath])
args.extend([x for x in inputs if x.endswith('.java')]) # automatically filter out non-java files
with openForWrite(argsfile, 'w', encoding=locale.getpreferredencoding()) as f:
for a in args:
f.write('"%s"'%a.replace('\\','\\\\')+'\n')
success=False
try:
log.info('Executing javac for %s, writing output to %s: %s', targetname, logbasename+'.out', ''.join(['\n\t"%s"'%x for x in [javacpath]+args]))
# make sure we have no old ones hanging around still
try:
deleteFile(logbasename+'-errors.txt', allowRetry=True)
deleteFile(logbasename+'-warnings.txt', allowRetry=True)
deleteFile(logbasename+'.out', allowRetry=True)
except Exception as e:
log.info('Cleaning up file failed: %s' % e)
outputHandler = options.get('javac.outputHandlerFactory', JavacProcessOutputHandler)(targetname, options=options)
if hasattr(outputHandler, 'setJavacLogBasename'):
outputHandler.setJavacLogBasename(logbasename)
call([javacpath, "@%s" % argsfile], outputHandler=outputHandler, outputEncoding='UTF-8', cwd=output, timeout=options['process.timeout'])
if (not os.listdir(output)): # unlikely, but useful failsafe
raise EnvironmentError('javac command failed to create any target files (but returned no error code); see output at "%s"'%(logbasename+'.out'))
success = True
finally:
if not success and classpath:
log.info('Classpath for failed javac was: \n %s', '\n '.join(classpath.split(os.pathsep)))
def run(self, context):
if self.cwd: self.cwd = context.getFullPath(self.cwd, self.baseDir)
if isDirPath(self.path):
mkdir(self.path)
cwd = self.cwd or self.path
else:
mkdir(os.path.dirname(self.path))
cwd = self.cwd or self.workDir
mkdir(self.workDir)
commands = self._resolveCommands(context)
assert len(
commands) > 0, 'No commands were specified to run in this target!'
if len(commands) > 1:
assert not (
self.redirectStdOutToTarget or self.stdout or self.stderr
), 'Invalid argument was specified for multiple commands mode'
cmdindex = 0
for cmd in commands:
cmdindex += 1
# this location is a lot easier to find than the target's workdir
logbasename = os.path.normpath(
context.getPropertyValue('BUILD_WORK_DIR') +
'/CustomCommandOutput/' + os.path.basename(cmd[0]) + "." +
targetNameToUniqueId(self.name))
if cmdindex > 1:
logbasename = logbasename + ".%d" % cmdindex # make this unique
cmdDisplaySuffix = ' #%d' % (cmdindex) if len(commands) > 1 else ''
stdoutPath = context.getFullPath(
self.path if self.redirectStdOutToTarget else
(self.stdout or logbasename + '.out'),
defaultDir='${BUILD_WORK_DIR}/CustomCommandOutput/')
stderrPath = context.getFullPath(
self.stderr or logbasename + '.err',
defaultDir='${BUILD_WORK_DIR}/CustomCommandOutput/')
self.log.info('Building %s by executing command%s: %s', self.name,
cmdDisplaySuffix,
''.join(['\n\t"%s"' % x for x in cmd]))
if self.cwd and cmdindex == 1:
self.log.info(' building %s from working directory: %s',
self.name, self.cwd) # only print if overridden
env = self.env or {}
if env:
if callable(env):
env = env(context)
else:
env = {
k: None if None == env[k] else self._resolveItem(
env[k], context)
for k in env
}
if cmdindex == 1:
self.log.info(
' environment overrides for %s are: %s', self.name,
''.join(['\n\t"%s=%s"' % (k, env[k]) for k in env]))
for k in os.environ:
if k not in env: env[k] = os.getenv(k)
for k in list(env.keys()):
if None == env[k]:
del env[k]
self.log.info(
' output from %s will be written to "%s" and "%s"',
self.name + cmdDisplaySuffix, stdoutPath, stderrPath)
if not os.path.exists(cmd[0]) and not (
IS_WINDOWS and os.path.exists(cmd[0] + '.exe')):
raise BuildException(
'Cannot run command because the executable does not exist: "%s"'
% (cmd[0]),
location=self.location)
encoding = self.options['common.processOutputEncodingDecider'](
context, cmd[0])
handler = self.options['CustomCommand.outputHandlerFactory']
if handler: # create a new handler for each command
handler = handler(str(self), options=self.options)
success = False
rc = None
try:
# maybe send output to a file instead
mkdir(os.path.dirname(logbasename))
with open(
stderrPath,
'wb') as fe: # can't use openForWrite with subprocess
with open(stdoutPath, 'wb') as fo:
process = subprocess.Popen(cmd,
stderr=fe,
stdout=fo,
cwd=cwd,
env=env)
rc = _wait_with_timeout(
process,
'%s(%s)' % (self.name, os.path.basename(cmd[0])),
self.options['process.timeout'], False)
success = rc == 0
finally:
try:
if os.path.getsize(stderrPath) == 0 and not self.stderr:
deleteFile(stderrPath, allowRetry=True)
if not self.redirectStdOutToTarget and os.path.getsize(
stdoutPath) == 0 and not self.stdout:
deleteFile(stdoutPath, allowRetry=True)
except Exception as e:
# stupid windows, it passes understanding
self.log.info(
'Failed to delete empty .out/.err files (ignoring error as it is not critical): %s',
e)
#if not os.listdir(self.workDir): deleteDir(self.workDir) # don't leave empty work dirs around
mainlog = '<command did not write any stdout/stderr>'
logMethod = self.log.info if success else self.log.error
if (handler
or not self.redirectStdOutToTarget) and os.path.isfile(
stdoutPath) and os.path.getsize(stdoutPath) > 0:
if handler:
with open(stdoutPath,
'r',
encoding=encoding,
errors='replace') as f:
for l in f:
handler.handleLine(l, isstderr=False)
elif os.path.getsize(stdoutPath) < 15 * 1024:
logMethod(
' stdout from %s is: \n%s',
self.name + cmdDisplaySuffix,
open(stdoutPath,
'r',
encoding=encoding,
errors='replace').read().replace(
'\n', '\n\t'))
mainlog = stdoutPath
if not success:
context.publishArtifact(
'%s%s stdout' % (self, cmdDisplaySuffix),
stdoutPath)
if os.path.isfile(
stderrPath) and os.path.getsize(stderrPath) > 0:
if handler:
with open(stderrPath,
'r',
encoding=encoding,
errors='replace') as f:
for l in f:
handler.handleLine(l, isstderr=True)
elif os.path.getsize(stderrPath) < 15 * 1024:
logMethod(
' stderr from %s is: \n%s',
self.name + cmdDisplaySuffix,
open(stderrPath,
'r',
encoding=encoding,
errors='replace').read().replace(
'\n', '\n\t'))
mainlog = stderrPath # take precedence over stdout
if not success:
context.publishArtifact(
'%s%s stderr' % (self, cmdDisplaySuffix),
stderrPath)
if handler:
handler.handleEnd(returnCode=rc)
elif rc != None and rc != 0 and not handler:
if IS_WINDOWS:
quotearg = lambda c: '"%s"' % c if ' ' in c else c
else:
quotearg = shlex.quote
# having it in this format makes it easier for people to re-run the command manually
self.log.info(' full command line is: %s',
' '.join(quotearg(c) for c in cmd))
raise BuildException(
'%s command%s failed with error code %s; see output at "%s" or look under %s'
% (os.path.basename(
cmd[0]), cmdDisplaySuffix, rc, mainlog, cwd),
location=self.location)
# final sanity check
if not os.path.exists(self.path):
raise BuildException(
'%s returned no error code but did not create the output file/dir; see output at "%s" or look under %s'
% (self, mainlog, cwd),
location=self.location)
if (not isDirPath(self.path)) and (not os.path.isfile(self.path)):
raise BuildException(
'%s did not create a file as expected (please check that trailing "/" is used if and only if a directory output is intended)'
% self,
location=self.location)
if isDirPath(self.path) and not os.listdir(self.path):
raise BuildException('%s created an empty directory' % self,
location=self.location)