def emit(self, record):
if record.pathname == '<process>':
entry = '%s_%s.%03d' % (record.name,
time.strftime(
'%Y-%m-%d_%H:%M:%S',
time.localtime(record.created)),
int(record.msecs))
files = record.files
files['info'] = 'call=%s\nexit=%s\n' % (repr(
record.proc.get_call()), record.proc.status(0))
files['stdout'] = record.proc.stdout.read_log()
files['stderr'] = record.proc.stderr.read_log()
files['stdin'] = record.proc.stdin.read_log()
try:
tar = tarfile.TarFile.open(self._fn, 'a')
for key, value in record.additional.items():
if os.path.exists(value):
value = open(value, 'r').read()
fileObj = VirtualFile(os.path.join(entry, key), [value])
info, handle = fileObj.getTarInfo()
tar.addfile(info, handle)
handle.close()
tar.close()
except Exception:
raise GCError(
'Unable to log results of external call "%s" to "%s"' %
(record.proc.get_call(), self._fn))
self._log.warning('All logfiles were moved to %s', self._fn)
def logError(self, target, brief=False, **kwargs): # Can also log content of additional files via kwargs
now = time.time()
entry = '%s.%s' % (time.strftime('%Y-%m-%d_%H:%M:%S', time.localtime(now)), ('%.5f' % (now - int(now)))[2:])
self._logger.log_time(logging.WARNING, '%s failed with code %d', self.niceCmd, self.wait())
if not brief:
self._logger.log_time(logging.WARNING, '\n%s', self.getError())
try:
tar = tarfile.TarFile.open(target, 'a')
data = {'retCode': self.wait(), 'exec': self.cmd, 'args': self.args}
files = [VirtualFile(os.path.join(entry, 'info'), DictFormat().format(data))]
kwargs.update({'stdout': self.getOutput(), 'stderr': self.getError()})
for key, value in kwargs.items():
try:
content = open(value, 'r').readlines()
except Exception:
content = [value]
files.append(VirtualFile(os.path.join(entry, key), content))
for fileObj in files:
info, handle = fileObj.getTarInfo()
tar.addfile(info, handle)
handle.close()
tar.close()
except Exception:
raise GCError('Unable to log errors of external process "%s" to "%s"' % (self.niceCmd, target))
self._logger.info('All logfiles were moved to %s', target)
def _getSandboxFiles(self, module, monitor, smList):
files = BasicWMS._getSandboxFiles(self, module, monitor, smList)
for idx, authFile in enumerate(self._token.getAuthFiles()):
files.append(
VirtualFile(('_proxy.dat.%d' % idx).replace('.0', ''),
open(authFile, 'r').read()))
return files
def _getSandboxFiles(self, task, monitor, smList):
# Prepare all input files
depList = set(
ichain(imap(lambda x: x.getDependencies(), [task] + smList)))
depPaths = lmap(lambda pkg: utils.pathShare('', pkg=pkg),
os.listdir(utils.pathPKG()))
depFiles = lmap(
lambda dep: utils.resolvePath('env.%s.sh' % dep, depPaths),
depList)
taskEnv = utils.mergeDicts(
imap(lambda x: x.getTaskConfig(), [monitor, task] + smList))
taskEnv.update({
'GC_DEPFILES': str.join(' ', depList),
'GC_USERNAME': self._token.getUsername(),
'GC_WMS_NAME': self.wmsName
})
taskConfig = sorted(
utils.DictFormat(escapeString=True).format(
taskEnv, format='export %s%s%s\n'))
varMappingDict = dict(
izip(monitor.getTaskConfig().keys(),
monitor.getTaskConfig().keys()))
varMappingDict.update(task.getVarMapping())
varMapping = sorted(
utils.DictFormat(delimeter=' ').format(varMappingDict,
format='%s%s%s\n'))
# Resolve wildcards in task input files
def getTaskFiles():
for f in task.getSBInFiles():
matched = glob.glob(f.pathAbs)
if matched != []:
for match in matched:
yield match
else:
yield f.pathAbs
return lchain([
monitor.getFiles(), depFiles,
getTaskFiles(),
[
VirtualFile('_config.sh', taskConfig),
VirtualFile('_varmap.dat', varMapping)
]
])
def _write_process_log(self, record):
entry = '%s_%s.%03d' % (record.name, time.strftime('%Y-%m-%d_%H:%M:%S', time.localtime(record.created)), int(record.msecs))
files = record.files
files['info'] = 'call=%s\nexit=%s\n' % (repr(record.proc.get_call()), record.proc.status(0))
files['stdout'] = record.proc.stdout.read_log()
files['stderr'] = record.proc.stderr.read_log()
files['stdin'] = record.proc.stdin.read_log()
try:
tar = tarfile.TarFile.open(self._fn, 'a')
for key, value in record.files.items():
if os.path.exists(value):
value = SafeFile(value).read()
fileObj = VirtualFile(os.path.join(entry, key), [value])
info, handle = fileObj.getTarInfo()
tar.addfile(info, handle)
handle.close()
tar.close()
except Exception:
raise GCError('Unable to log results of external call "%s" to "%s"' % (record.proc.get_call(), self._fn))
def _addToTar(self, tar, name, data): info, fileObj = VirtualFile(name, data).getTarInfo() tar.addfile(info, fileObj) fileObj.close()