def main():
""" Entry point when called as an executable """
parser = argparse.ArgumentParser(
description='Run check on given submit wcl')
parser.add_argument('--verbose', action='store', default=True)
parser.add_argument('--des_db_section', action='store')
parser.add_argument('--des_services', action='store')
parser.add_argument(
'--expandwcl',
action='store',
default=True,
help='set to False if running on an uberctrl/config.des')
parser.add_argument('wclfile', action='store')
args = vars(parser.parse_args()) # convert dict
args['verbose'] = miscutils.convertBool(args['verbose'])
args['usePFWconfig'] = miscutils.convertBool(args['expandwcl'])
args['get_db_config'] = miscutils.convertBool(args['expandwcl'])
# usePFWconfig and get_db_config set to True because dessubmit does
# (work only done at submit time)
# use_db_in=False in submit wcl overrides get_db_config
print "Gathering wcl..."
config = pfwconfig.PfwConfig(args)
config[pfwdefs.ATTNUM] = '0' # must be string as if read from wcl file
testcnts = pfwcheck.check(config, '')
print "\nTest Summary"
print "\tErrors: %d" % testcnts[0]
print "\tWarnings: %d" % testcnts[1]
print "\tItems fixed: %d" % testcnts[2]
def check_module(config, modname, indent=''):
""" Check module """
cnts = [0] * NUMCNTS
print "%sChecking module %s..." % (indent, modname)
moddict = config[pfwdefs.SW_MODULESECT][modname]
dataobjs = {pfwdefs.SW_INPUTS: {}, pfwdefs.SW_OUTPUTS: {}}
# check that have wrappername (required)
if pfwdefs.SW_WRAPPERNAME not in moddict and \
not miscutils.convertBool(moddict[pfwdefs.PF_NOOP]):
error(indent+' ', "module %s - missing %s value" % (modname, pfwdefs.SW_WRAPPERNAME))
cnts[ERRCNT_POS] += 1
# check that have at least 1 exec section (required)
execsects = intgmisc.get_exec_sections(moddict, pfwdefs.SW_EXECPREFIX)
if not execsects and not miscutils.convertBool(moddict[pfwdefs.PF_NOOP]):
error(indent+' ', "module %s - 0 exec sections (%s*)" % (modname, pfwdefs.SW_EXECPREFIX))
cnts[ERRCNT_POS] += 1
else:
# check exec sections
for xsectname in execsects:
xsectdict = moddict[xsectname]
cnts2 = check_exec(config, modname, dataobjs, xsectname, xsectdict, indent+" ")
cnts = [x + y for x, y in zip(cnts, cnts2)] # increment counts
# check file/list sections
cnts2 = check_dataobjs(config, modname, moddict, dataobjs, indent+" ")
cnts = [x + y for x, y in zip(cnts, cnts2)] # increment counts
return cnts
def stagefiles(self, opts=None):
"""Return whether to save stage files to target archive.
"""
retval = True
notarget_exists, notarget = self.search(pfwdefs.PF_DRYRUN, opts)
if notarget_exists and miscutils.convertBool(notarget):
print("Do not stage file due to dry run\n")
retval = False
else:
stagefiles_exists, stagefiles = self.search(
pfwdefs.STAGE_FILES, opts)
if stagefiles_exists:
#print "checking stagefiles (%s)" % stagefiles
results = replfuncs.replace_vars_single(stagefiles, self, opts)
retval = miscutils.convertBool(results)
#print "after interpolation stagefiles (%s)" % retval
else:
envkey = 'DESDM_%s' % pfwdefs.STAGE_FILES.upper()
if envkey in os.environ and not miscutils.convertBool(
os.environ[envkey]):
retval = False
#print "stagefiles retval = %s" % retval
return retval
def stagefile(self, opts):
""" Determine whether should stage files or not """
retval = True
(dryrun_exists, dryrun) = self.search(pfwdefs.PF_DRYRUN, opts)
if dryrun_exists and miscutils.convertBool(dryrun):
retval = False
(stagefiles_exists, stagefiles) = self.search(pfwdefs.STAGE_FILES, opts)
if stagefiles_exists and not miscutils.convertBool(stagefiles):
retval = False
return retval
def check_file_valid_input(config, modname, fname, fdict, indent=''):
""" Check if given input file is valid """
cnts = [0] * NUMCNTS
# check that any given filename pattern has a definition
if pfwdefs.SW_FILEPAT in fdict:
cnts2 = check_filepat_valid(config, fdict[pfwdefs.SW_FILEPAT], modname, fname, indent+' ')
cnts = [x + y for x, y in zip(cnts, cnts2)] # increment counts
# check that it has filepat, filename, depends, or query wcl (required)
# if filename is a pattern, can I check that all needed values exist?
# todo check depends happens in same block previous to this module
if (('listonly' not in fdict or not miscutils.convertBool(fdict['listonly'])) and
pfwdefs.SW_FILEPAT not in fdict and pfwdefs.FILENAME not in fdict and
'fullname' not in fdict and 'query_fields' not in fdict and pfwdefs.DATA_DEPENDS not in fdict):
error(indent, "module %s, %s, %s - Missing terms needed to determine input filename" % (modname, pfwdefs.SW_INPUTS, fname))
cnts[ERRCNT_POS] += 1
# check that it has pfwdefs.DIRPAT : err
# can I check that all values for pfwdefs.DIRPAT exist?
if pfwdefs.DIRPAT not in fdict:
error(indent, "module %s, %s, %s - Missing %s" % (modname, pfwdefs.SW_INPUTS, fname, pfwdefs.DIRPAT))
cnts[ERRCNT_POS] += 1
return cnts
def getfull(self, key, opts=None, default=None):
""" Return with variables replaced and expanded if string(s) """
if miscutils.fwdebug_check(9, "WCL_DEBUG"):
miscutils.fwdebug_print(f"BEG - key={key}")
miscutils.fwdebug_print(f"default - {default}")
miscutils.fwdebug_print(f"opts - {opts}")
(found, value) = self.search(key, opts)
if not found:
value = default
elif isinstance(value, str):
if opts is None:
newopts = {'expand': True, intgdefs.REPLACE_VARS: True}
else:
newopts = copy.deepcopy(opts)
if intgdefs.REPLACE_VARS not in newopts or \
miscutils.convertBool(newopts[intgdefs.REPLACE_VARS]):
newopts['expand'] = True
if miscutils.fwdebug_check(9, "WCL_DEBUG"):
miscutils.fwdebug_print(
f"calling replace_vars value={value}, opts={newopts}")
(value, _) = replfuncs.replace_vars(value, self, newopts)
if len(value) == 1:
value = value[0]
return value
def check_proxy(config):
"""Check for proxy.
Check if any block will submit to remote machine needing proxy,
if so check for proxy.
"""
if miscutils.fwdebug_check(3, 'PFWSUBMIT_DEBUG'):
miscutils.fwdebug_print("Beg")
config.reset_blknum()
blocklist = miscutils.fwsplit(config[pfwdefs.SW_BLOCKLIST].lower(), ',')
for blockname in blocklist:
if miscutils.fwdebug_check(3, 'PFWSUBMIT_DEBUG'):
miscutils.fwdebug_print("Checking block %s..." % (blockname))
config.set_block_info()
(exists, check_proxy) = config.search(pfwdefs.SW_CHECK_PROXY,
{intgdefs.REPLACE_VARS: True})
if exists and miscutils.convertBool(check_proxy):
timeleft = pfwcondor.get_grid_proxy_timeleft()
assert timeleft > 0
if timeleft < 21600: # 5 * 60 * 60
print("Warning: Proxy expires in less than 5 hours")
break
config.inc_blknum()
config.reset_blknum()
if miscutils.fwdebug_check(3, 'PFWSUBMIT_DEBUG'):
miscutils.fwdebug_print("End")
def compress_files(listfullnames, compresssuffix, execname, argsorig, max_try_cnt=3, cleanup=True):
""" Compress given files """
if miscutils.fwdebug_check(3, 'PFWCOMPRESS_DEBUG'):
miscutils.fwdebug_print("BEG num files to compress = %s" % (len(listfullnames)))
results = {}
tot_bytes_before = 0
tot_bytes_after = 0
for fname in listfullnames:
errstr = None
cmd = None
fname_compressed = None
returncode = 1
try:
if not os.path.exists(fname):
errstr = "Error: Uncompressed file does not exist (%s)" % fname
returncode = 1
else:
tot_bytes_before += os.path.getsize(fname)
fname_compressed = fname + compresssuffix
# create command
args = copy.deepcopy(argsorig)
args = replfuncs.replace_vars_single(args,
{'__UCFILE__': fname,
'__CFILE__': fname_compressed},
None)
cmd = '%s %s' % (execname, args)
if miscutils.fwdebug_check(3, 'PFWCOMPRESS_DEBUG'):
miscutils.fwdebug_print("compression command: %s" % cmd)
returncode = run_compression_command(cmd, fname_compressed, max_try_cnt)
except IOError as exc:
errstr = "I/O error({0}): {1}".format(exc.errno, exc.strerror)
returncode = 1
if returncode != 0:
errstr = "Compression failed with exit code %i" % returncode
# check for partial compressed output and remove
if os.path.exists(fname_compressed):
miscutils.fwdebug_print("Compression failed. Removing compressed file.")
os.unlink(fname_compressed)
elif miscutils.convertBool(cleanup): # if successful, remove uncompressed if requested
os.unlink(fname)
if returncode == 0:
tot_bytes_after += os.path.getsize(fname_compressed)
else:
tot_bytes_after += os.path.getsize(fname)
# save exit code, cmd and new name
results[fname] = {'status': returncode,
'outname': fname_compressed,
'err': errstr,
'cmd': cmd}
if miscutils.fwdebug_check(3, 'PFWCOMPRESS_DEBUG'):
miscutils.fwdebug_print("END bytes %s => %s" % (tot_bytes_before, tot_bytes_after))
return (results, tot_bytes_before, tot_bytes_after)
def check_block(config, indent=''):
"""Check blocks level defs.
"""
cnts = [0] * NUMCNTS
blocklist = miscutils.fwsplit(config[pfwdefs.SW_BLOCKLIST].lower(), ',')
for blockname in blocklist:
print("%sChecking block %s..." % (indent, blockname))
config.set_block_info()
for key in [pfwdefs.PF_USE_DB_IN, pfwdefs.PF_USE_DB_OUT]:
if key in config and miscutils.convertBool(config.getfull(key)):
(found, val) = config.search('target_des_db_section')
if not found:
error(
indent + ' ',
"using DB (%s), but missing target_des_db_section" %
(key))
cnts[ERRCNT_POS] += 1
(found, val) = config.search('target_des_services')
if not found:
error(
indent + ' ',
"using DB (%s), but missing target_des_services" %
(key))
cnts[ERRCNT_POS] += 1
# check modules
block = config[pfwdefs.SW_BLOCKSECT][blockname]
if pfwdefs.SW_MODULELIST in block:
modulelist = miscutils.fwsplit(
block[pfwdefs.SW_MODULELIST].lower(), ',')
for modname in modulelist:
if modname not in config[pfwdefs.SW_MODULESECT]:
error(
indent + ' ', "block %s - invalid %s" %
(blockname, pfwdefs.SW_MODULELIST))
print("%s (bad module name: %s, list: %s)" %
(indent, modname, modulelist))
cnts[ERRCNT_POS] += 1
else:
cnts2 = check_module(config, blockname, modname,
indent + ' ')
cnts = [x + y
for x, y in zip(cnts, cnts2)] # increment counts
else:
error(
indent + ' ', "block %s - missing %s value" %
(blockname, pfwdefs.SW_MODULESECT))
cnts[ERRCNT_POS] += 1
config.inc_blknum()
config.reset_blknum()
return cnts
def endrun(configfile):
"""Program entry point.
"""
miscutils.fwdebug_print("BEG")
config = pfwconfig.PfwConfig({'wclfile': configfile})
os.chdir('../uberctrl')
retval = pfwdefs.PF_EXIT_SUCCESS
if pfwdefs.USE_HOME_ARCHIVE_OUTPUT in config and \
config[pfwdefs.USE_HOME_ARCHIVE_OUTPUT].lower() == 'run':
if pfwdefs.ATTEMPT_ARCHIVE_PATH not in config:
print("Error: Cannot find %s in config" % pfwdefs.ATTEMPT_ARCHIVE_PATH)
print("\tIt is needed for the mass copy of the run back to the " \
"home archive at the end of the run")
return pfwdefs.PF_EXIT_FAILURE
archpath = config.getfull(config[pfwdefs.ATTEMPT_ARCHIVE_PATH])
print("archpath =", archpath)
# call archive transfer for target archive to home archive
# check if using target archive
target_info = None
if pfwdefs.USE_TARGET_ARCHIVE_OUTPUT in config and \
config.getfull(pfwdefs.USE_TARGET_ARCHIVE_OUTPUT).lower() != 'never':
if pfwdefs.TARGET_ARCHIVE in config and \
config.getfull(pfwdefs.TARGET_ARCHIVE) in config[pfwdefs.SW_ARCHIVESECT]:
target_info = config[pfwdefs.SW_ARCHIVESECT][config.getfull(pfwdefs.TARGET_ARCHIVE)]
else:
print("Error: cannot determine info for target archive")
return pfwdefs.PF_EXIT_FAILURE
else:
print("Error: Asked to transfer outputs at end of run, but not using target archive")
return pfwdefs.PF_EXIT_FAILURE
home_info = None
print(config[pfwdefs.HOME_ARCHIVE])
if pfwdefs.HOME_ARCHIVE in config and \
config[pfwdefs.HOME_ARCHIVE] in config[pfwdefs.SW_ARCHIVESECT]:
home_info = config[pfwdefs.SW_ARCHIVESECT][config.getfull(pfwdefs.HOME_ARCHIVE)]
# call transfer
archive_transfer_utils.archive_copy_dir(target_info, home_info,
config.getfull('archive_transfer'),
archpath, config)
if miscutils.convertBool(config[pfwdefs.PF_USE_DB_OUT]):
miscutils.fwdebug_print("Calling update_attempt_end: retval = %s" % retval)
dbh = pfwdb.PFWDB(config.getfull('submit_des_services'),
config.getfull('submit_des_db_section'))
dbh.end_task(config['task_id']['attempt'], retval, True)
dbh.commit()
dbh.close()
miscutils.fwdebug_print("END - exiting with code %s" % retval)
return retval
def get_optout(self, sect):
""" Return whether file(s) are optional outputs """
optout = False
sectkeys = sect.split('.')
if sectkeys[0] == intgdefs.IW_FILE_SECT:
if intgdefs.IW_OUTPUT_OPTIONAL in self.inputwcl.get(sect):
optout = miscutils.convertBool(
self.inputwcl.get(sect)[intgdefs.IW_OUTPUT_OPTIONAL])
elif sectkeys[0] == intgdefs.IW_LIST_SECT:
if intgdefs.IW_OUTPUT_OPTIONAL in self.inputwcl.get(
f"{intgdefs.IW_FILE_SECT}.{sectkeys[2]}"):
optout = miscutils.convertBool(
self.inputwcl.get(f"{intgdefs.IW_FILE_SECT}.{sectkeys[2]}")
[intgdefs.IW_OUTPUT_OPTIONAL])
else:
raise KeyError(f"Unknown data section {sectkeys[0]}")
return optout
def __init__(self, config):
if 'transfer_stats_per_file' in config:
self.transfer_stats_per_file = miscutils.convertBool(
config['transfer_stats_per_file'])
else:
self.transfer_stats_per_file = False
self.batchvals = {}
self.filevals = {}
self.__initialize_values__()
def savefiles(self, opts=None):
""" Return whether to save files from job """
retval = True
savefiles_exists, savefiles = self.search(pfwdefs.SAVE_FILE_ARCHIVE, opts)
if savefiles_exists:
if miscutils.fwdebug_check(3, 'PFWCONFIG_DEBUG'):
miscutils.fwdebug_print("checking savefiles (%s)" % savefiles)
results = replfuncs.replace_vars_single(savefiles, self, opts)
retval = miscutils.convertBool(results)
if miscutils.fwdebug_check(3, 'PFWCONFIG_DEBUG'):
miscutils.fwdebug_print("after interpolation savefiles (%s)" % retval)
else:
envkey = 'DESDM_%s' % pfwdefs.SAVE_FILE_ARCHIVE.upper()
if envkey in os.environ and not miscutils.convertBool(os.environ[envkey]):
retval = False
if miscutils.fwdebug_check(3, 'PFWCONFIG_DEBUG'):
miscutils.fwdebug_print("savefiles retval = %s" % retval)
return retval
def should_save_file(mastersave, filesave, exitcode):
""" Determine whether should save the file """
msave = mastersave.lower()
fsave = miscutils.convertBool(filesave)
if msave == 'failure':
if exitcode != 0:
msave = 'always'
else:
msave = 'file'
return (msave == 'always') or (msave == 'file' and fsave)
def endblock(configfile):
"""Program entry point.
"""
miscutils.fwdebug_print("BEG")
config = pfwconfig.PfwConfig({'wclfile': configfile})
blkdir = config.getfull('block_dir')
os.chdir(blkdir)
if pfwdefs.USE_HOME_ARCHIVE_OUTPUT in config and \
config[pfwdefs.USE_HOME_ARCHIVE_OUTPUT].lower() == 'block':
# check if using target archive
target_info = None
if pfwdefs.USE_TARGET_ARCHIVE_OUTPUT in config and \
config[pfwdefs.USE_TARGET_ARCHIVE_OUTPUT].lower() != 'never':
print(config[pfwdefs.TARGET_ARCHIVE])
if pfwdefs.TARGET_ARCHIVE in config and \
config[pfwdefs.TARGET_ARCHIVE] in config.getfull(pfwdefs.SW_ARCHIVESECT):
target_info = config.getfull(pfwdefs.SW_ARCHIVESECT)[config.getfull(pfwdefs.TARGET_ARCHIVE)]
else:
print("Error: cannot determine info for target archive")
return pfwdefs.PF_EXIT_FAILURE
else:
print("Error: Asked to transfer outputs at end of block, but not using target archive")
return pfwdefs.PF_EXIT_FAILURE
home_info = None
print(config.getfull(pfwdefs.HOME_ARCHIVE))
if pfwdefs.HOME_ARCHIVE in config and \
config.getfull(pfwdefs.HOME_ARCHIVE) in config[pfwdefs.SW_ARCHIVESECT]:
home_info = config[pfwdefs.SW_ARCHIVESECT][config.getfull(pfwdefs.HOME_ARCHIVE)]
# get file list of files to transfer
if pfwdefs.PF_USE_DB_OUT in config and miscutils.convertBool(config[pfwdefs.PF_USE_DB_OUT]):
dbh = pfwdb.PFWDB()
filelist = dbh.get_run_filelist(config.getfull(pfwdefs.REQNUM),
config.getfull(pfwdefs.UNITNAME),
config.getfull(pfwdefs.ATTNUM),
config.getfull(pfwdefs.PF_BLKNUM),
config.getfull(pfwdefs.TARGET_ARCHIVE))
else:
print("Error: Asked to transfer outputs at end of block, but not using database.")
print(" Currently not supported.")
return pfwdefs.PF_EXIT_FAILURE
# call transfer
archive_transfer_utils.archive_copy(target_info, home_info,
config.getfull('archive_transfer'), filelist, config)
miscutils.fwdebug_print("END - exiting with code %s" % pfwdefs.PF_EXIT_SUCCESS)
return pfwdefs.PF_EXIT_SUCCESS
def get_optout(self, sect):
""" Return whether file(s) are optional outputs
Parameters
----------
sect : str
The section of the WCL to use for finding file data
Returns
-------
bool
Whether or not the files in the specified section are optional.
Raises
------
KeyError
If the specified `sect` does not exist.
"""
optout = False
sectkeys = sect.split('.')
if sectkeys[0] == intgdefs.IW_FILE_SECT:
if intgdefs.IW_OUTPUT_OPTIONAL in self.inputwcl.get(sect):
optout = miscutils.convertBool(
self.inputwcl.get(sect)[intgdefs.IW_OUTPUT_OPTIONAL])
elif sectkeys[0] == intgdefs.IW_LIST_SECT:
if intgdefs.IW_OUTPUT_OPTIONAL in self.inputwcl.get(
"%s.%s" % (intgdefs.IW_FILE_SECT, sectkeys[2])):
optout = miscutils.convertBool(
self.inputwcl.get(
"%s.%s" % (intgdefs.IW_FILE_SECT,
sectkeys[2]))[intgdefs.IW_OUTPUT_OPTIONAL])
else:
raise KeyError("Unknown data section %s" % sectkeys[0])
return optout
def getfull(self, key, opts=None, default=None):
""" Return with variables replaced and expanded if string(s)
Parameters
----------
key : str
The key whose value is to be returned.
opts : dict, optional
Options for processing the value. Default is ``None``.
default : various, optional
The default value to return. Default is ``None``.
"""
if miscutils.fwdebug_check(9, "WCL_DEBUG"):
miscutils.fwdebug_print("BEG - key=%s" % key)
miscutils.fwdebug_print("default - %s" % default)
miscutils.fwdebug_print("opts - %s" % opts)
(found, value) = self.search(key, opts)
if not found:
value = default
elif isinstance(value, (str, unicode)):
if opts is None:
newopts = {'expand': True, intgdefs.REPLACE_VARS: True}
else:
newopts = copy.deepcopy(opts)
if intgdefs.REPLACE_VARS not in newopts or \
miscutils.convertBool(newopts[intgdefs.REPLACE_VARS]):
newopts['expand'] = True
if miscutils.fwdebug_check(9, "WCL_DEBUG"):
miscutils.fwdebug_print("calling replace_vars value=%s, opts=%s" % \
(value, newopts))
(value, _) = replfuncs.replace_vars(value, self, newopts)
if len(value) == 1:
value = value[0]
return value
def should_compress_file(mastercompress, filecompress, exitcode):
""" Determine whether should compress the file """
if miscutils.fwdebug_check(6, "PFWUTILS_DEBUG"):
miscutils.fwdebug_print(f"BEG: master={mastercompress}, file={filecompress}, exitcode={exitcode}")
mcompress = mastercompress
if isinstance(mastercompress, str):
mcompress = mastercompress.lower()
fcompress = miscutils.convertBool(filecompress)
if mcompress == 'success':
if exitcode != 0:
mcompress = 'never'
else:
mcompress = 'file'
retval = (mcompress == 'file' and fcompress)
if miscutils.fwdebug_check(6, "PFWUTILS_DEBUG"):
miscutils.fwdebug_print(f"END - retval = {retval}")
return retval
def __init__(self, config, fullcfg=None):
if not miscutils.use_db(config):
miscutils.fwdie("Error: TransferStatsDB class requires DB "\
" but was told not to use DB", 1)
self.currvals = {}
self.__initialize_values__()
self.desservices = None
if config is not None and 'des_services' in config:
self.desservices = config['des_services']
self.section = None
if config is not None and 'des_db_section' in config:
self.section = config['des_db_section']
have_connect = False
if 'connection' in config:
try:
desdmdbi.DesDmDbi.__init__(self, connection=config['connection'])
have_connect = True
except:
miscutils.fwdebug_print('Could not connect to DB using transferred connection, falling back to new connection.')
if not have_connect:
try:
desdmdbi.DesDmDbi.__init__(self, self.desservices, self.section)
except (configparser.NoSectionError, IOError) as err:
miscutils.fwdie(f"Error: problem connecting to database: {err}\n" \
"\tCheck desservices file and environment variables", 1)
self.parent_task_id = config['parent_task_id']
self.root_task_id = config['root_task_id']
if 'transfer_stats_per_file' in config:
self.transfer_stats_per_file = miscutils.convertBool(config['transfer_stats_per_file'])
else:
self.transfer_stats_per_file = False
def begrun(argv):
"""TODO: short summary.
Performs steps executed on submit machine at beginning of processing
attempt.
"""
pfw_dbh = None
try:
configfile = argv[0]
config = pfwconfig.PfwConfig({'wclfile': configfile})
if miscutils.fwdebug_check(6, 'BEGRUN_DEBUG'):
miscutils.fwdebug_print(
'use_home_archive_output = %s' %
config.getfull(pfwdefs.USE_HOME_ARCHIVE_OUTPUT))
if miscutils.convertBool(config.getfull(pfwdefs.PF_USE_DB_OUT)):
import processingfw.pfwdb as pfwdb
pfw_dbh = pfwdb.PFWDB(config.getfull('submit_des_services'),
config.getfull('submit_des_db_section'))
pfw_dbh.begin_task(config['task_id']['attempt'], True)
# the three wcl files to copy to the home archive
origwcl = config['origwcl']
expwcl = config['expwcl']
fullwcl = config['fullwcl']
# if not a dryrun and using a home archive for output
if (config.getfull(pfwdefs.USE_HOME_ARCHIVE_OUTPUT) != 'never'
and 'submit_files_mvmt' in config and
(pfwdefs.PF_DRYRUN not in config
or not miscutils.convertBool(config.getfull(pfwdefs.PF_DRYRUN)))):
# get home archive info
home_archive = config.getfull('home_archive')
archive_info = config[pfwdefs.SW_ARCHIVESECT][home_archive]
# load filemgmt class
attempt_tid = config['task_id']['attempt']
filemgmt = pfwutils.pfw_dynam_load_class(pfw_dbh, config,
attempt_tid, attempt_tid,
"filemgmt",
archive_info['filemgmt'],
archive_info)
# save file information
filemgmt.register_file_data('wcl', [origwcl, expwcl, fullwcl],
config['pfw_attempt_id'], attempt_tid,
False, None, None)
copy_files_home(config, archive_info, filemgmt)
filemgmt.commit()
if pfw_dbh is not None:
print("Saving attempt's archive path into PFW tables...", end=' ')
pfw_dbh.update_attempt_archive_path(config)
pfw_dbh.commit()
except Exception as exc:
msg = "begrun: %s: %s" % (exc.__class__.__name__, str(exc))
if pfw_dbh is not None:
Messaging.pfw_message(pfw_dbh, config['pfw_attempt_id'],
config['task_id']['attempt'], msg,
pfw_utils.PFW_DB_WARN, 'begrun.out', 0)
send_failed_email(config, msg)
raise
except SystemExit as exc:
msg = "begrun: SysExit=%s" % str(exc)
if pfw_dbh is not None:
Messaging.pfw_message(pfw_dbh, config['pfw_attempt_id'],
config['task_id']['attempt'], msg,
pfw_utils.PFW_DB_WARN, 'begrun.out', 0)
send_failed_email(config, msg)
raise
def create_master_list(config, configfile, modname, moddict,
search_name, search_dict, search_type):
"""Create master data list for a module's list or file def.
"""
miscutils.fwdebug_print("BEG")
if 'qouttype' in search_dict:
qouttype = search_dict['qouttype']
else:
qouttype = intgdefs.DEFAULT_QUERY_OUTPUT_FORMAT
qoutfile = config.get_filename('qoutput',
{pfwdefs.PF_CURRVALS: {'modulename': modname,
'searchname': search_name,
'suffix': qouttype}})
qlog = config.get_filename('qoutput',
{pfwdefs.PF_CURRVALS: {'modulename': modname,
'searchname': search_name,
'suffix': 'out'}})
prog = None
if 'exec' in search_dict:
prog = search_dict['exec']
if 'args' not in search_dict:
print("\t\tWarning: %s in module %s does not have args defined\n" % \
(search_name, modname))
args = ""
else:
args = search_dict['args']
elif 'query_fields' in search_dict:
if 'processingfw_dir' in config:
dirgenquery = config['processingfw_dir']
elif 'PROCESSINGFW_DIR' in os.environ:
dirgenquery = os.environ['PROCESSINGFW_DIR']
else:
miscutils.fwdie("Error: Could not determine base path for genquerydb.py",
pfwdefs.PF_EXIT_FAILURE)
prog = "%s/libexec/genquerydb.py" % (dirgenquery)
args = "--qoutfile %s --qouttype %s --config %s --module %s --search %s" % \
(qoutfile, qouttype, configfile, modname, search_name)
if not prog:
print("\tWarning: %s in module %s does not have exec or %s defined" % \
(search_name, modname, pfwdefs.SW_QUERYFIELDS))
return
search_dict['qoutfile'] = qoutfile
search_dict['qlog'] = qlog
prog = replfuncs.replace_vars_single(prog, config,
{pfwdefs.PF_CURRVALS: {pfwdefs.SW_MODULESECT: modname},
'searchobj': search_dict})
# handle both outputxml and outputfile args
args = replfuncs.replace_vars_single(args, config,
{pfwdefs.PF_CURRVALS: {pfwdefs.SW_MODULESECT: modname,
'outputxml': qoutfile,
'outputfile': qoutfile,
'qoutfile': qoutfile},
#intgdefs.REPLACE_VARS: True,
'searchobj': search_dict})
# get version for query code
query_version = None
if prog in config[pfwdefs.SW_EXEC_DEF]:
query_version = pfwutils.get_version(prog, config[pfwdefs.SW_EXEC_DEF])
if search_type == pfwdefs.SW_LISTSECT:
datatype = 'L'
elif search_type == pfwdefs.SW_FILESECT:
datatype = 'F'
else:
datatype = search_type[0].upper()
# call code
query_tid = None
if miscutils.convertBool(config.getfull(pfwdefs.PF_USE_DB_OUT)):
pfw_dbh = pfwdb.PFWDB()
query_tid = pfw_dbh.insert_data_query(config, modname, datatype, search_name,
prog, args, query_version)
#pfw_dbh.close()
else:
pfw_dbh = None
cwd = os.getcwd()
print("\t\tCalling code to create master list for obj %s in module %s" % \
(search_name, modname))
print("\t\t", prog, args)
print("\t\tSee output in %s/%s" % (cwd, qlog))
print("\t\tSee master list will be in %s/%s" % (cwd, qoutfile))
print("\t\tCreating master list - start ", time.time())
cmd = "%s %s" % (prog, args)
exitcode = None
try:
exitcode = pfwutils.run_cmd_qcf(cmd, qlog, query_tid, os.path.basename(prog),
config.getfull(pfwdefs.PF_USE_QCF), pfw_dbh,
config['pfw_attempt_id'])
#exitcode = pfwutils.run_cmd_qcf(cmd, qlog, query_tid, os.path.basename(prog),
# 5000, config.getfull(pfwdefs.PF_USE_QCF))
except:
print("******************************")
print("Error: ")
(extype, exvalue, trback) = sys.exc_info()
print("******************************")
traceback.print_exception(extype, exvalue, trback, file=sys.stdout)
exitcode = pfwdefs.PF_EXIT_FAILURE
print("\t\tCreating master list - end ", time.time())
sys.stdout.flush()
if miscutils.convertBool(config.getfull(pfwdefs.PF_USE_DB_OUT)):
pfw_dbh = pfwdb.PFWDB()
pfw_dbh.end_task(query_tid, exitcode, True)
pfw_dbh.close()
if exitcode != 0:
miscutils.fwdie("Error: problem creating master list (exitcode = %s)" %
(exitcode), exitcode)
miscutils.fwdebug_print("END")
def begblock(argv):
"""Program entry point.
"""
if argv == None:
argv = sys.argv
configfile = argv[0]
config = pfwconfig.PfwConfig({'wclfile': configfile})
config.set_block_info()
blknum = config[pfwdefs.PF_BLKNUM]
blkdir = config.getfull('block_dir')
os.chdir(blkdir)
(exists, submit_des_services) = config.search('submit_des_services')
if exists and submit_des_services is not None:
os.environ['DES_SERVICES'] = submit_des_services
(exists, submit_des_db_section) = config.search('submit_des_db_section')
if exists and submit_des_db_section is not None:
os.environ['DES_DB_SECTION'] = submit_des_db_section
dbh = None
blktid = -1
if miscutils.fwdebug_check(3, 'PFWBLOCK_DEBUG'):
miscutils.fwdebug_print("blknum = %s" % (config[pfwdefs.PF_BLKNUM]))
if miscutils.convertBool(config.getfull(pfwdefs.PF_USE_DB_OUT)):
dbh = pfwdb.PFWDB(submit_des_services, submit_des_db_section)
dbh.insert_block(config)
blktid = config['task_id']['block'][str(blknum)]
config['task_id']['begblock'] = dbh.create_task(
name='begblock',
info_table=None,
parent_task_id=blktid,
root_task_id=int(config['task_id']['attempt']),
label=None,
do_begin=True,
do_commit=True)
try:
modulelist = miscutils.fwsplit(
config.getfull(pfwdefs.SW_MODULELIST).lower())
modules_prev_in_list = {}
joblist = {}
parlist = OrderedDict()
masterdata = OrderedDict()
filelist = {'infiles': {}, 'outfiles': {}}
for num, modname in enumerate(modulelist):
print("XXXXXXXXXXXXXXXXXXXX %s XXXXXXXXXXXXXXXXXXXX" % modname)
if modname not in config[pfwdefs.SW_MODULESECT]:
miscutils.fwdie(
"Error: Could not find module description for module %s\n"
% (modname), pfwdefs.PF_EXIT_FAILURE)
moddict = config[pfwdefs.SW_MODULESECT][modname]
runqueries(config, configfile, modname, modules_prev_in_list)
pfwblock.read_master_lists(config, modname, masterdata,
modules_prev_in_list)
(infsect, outfsect) = pfwblock.get_datasect_types(config, modname)
pfwblock.fix_master_lists(config, modname, masterdata, outfsect)
if pfwdefs.PF_NOOP not in moddict or not miscutils.convertBool(
moddict[pfwdefs.PF_NOOP]):
pfwblock.create_fullnames(config, modname, masterdata)
if miscutils.fwdebug_check(
9, 'PFWBLOCK_DEBUG') and modname in masterdata:
with open('%s-masterdata.txt' % modname, 'w') as fh:
miscutils.pretty_print_dict(masterdata[modname], fh)
pfwblock.add_file_metadata(config, modname)
sublists = pfwblock.create_sublists(config, modname,
masterdata)
if sublists is not None:
if miscutils.fwdebug_check(3, 'PFWBLOCK_DEBUG'):
miscutils.fwdebug_print("sublists.keys() = %s" %
(list(sublists.keys())))
loopvals = pfwblock.get_wrapper_loopvals(config, modname)
wrapinst = pfwblock.create_wrapper_inst(
config, modname, loopvals)
wcnt = 1
for winst in list(wrapinst.values()):
if miscutils.fwdebug_check(6, 'PFWBLOCK_DEBUG'):
miscutils.fwdebug_print("winst %d - BEG" % wcnt)
pfwblock.assign_data_wrapper_inst(config, modname, winst,
masterdata, sublists,
infsect, outfsect)
pfwblock.finish_wrapper_inst(config, modname, winst,
outfsect)
tempfiles = pfwblock.create_module_wrapper_wcl(
config, modname, winst)
for fl in tempfiles['infiles']:
if fl not in list(filelist['infiles'].keys()):
filelist['infiles'][fl] = num
for fl in tempfiles['outfiles']:
filelist['outfiles'][fl] = num
#filelist['infiles'] += tempfiles['infiles']
#filelist['outfiles'] += tempfiles['outfiles']
pfwblock.divide_into_jobs(config, modname, winst, joblist,
parlist)
if miscutils.fwdebug_check(6, 'PFWBLOCK_DEBUG'):
miscutils.fwdebug_print("winst %d - %s - END" %
(wcnt, etime - stime))
wcnt += 1
modules_prev_in_list[modname] = True
if miscutils.fwdebug_check(
9, 'PFWBLOCK_DEBUG') and modname in masterdata:
with open('%s-masterdata.txt' % modname, 'w') as fh:
miscutils.pretty_print_dict(masterdata[modname], fh)
scriptfile = pfwblock.write_runjob_script(config)
intersect = list(
set(filelist['infiles'].keys()) & set(filelist['outfiles'].keys()))
finallist = []
for fl in list(filelist['infiles'].keys()):
if fl not in intersect:
finallist.append(fl)
else:
if filelist['infiles'][fl] <= filelist['outfiles'][fl]:
raise Exception(
'Input file %s requested before it is generated.' %
(fl))
if miscutils.convertBool(config.getfull(pfwdefs.PF_USE_DB_OUT)):
missingfiles = dbh.check_files(config, finallist)
if len(missingfiles) > 0:
raise Exception(
"The following input files cannot be found in the archive:"
+ ",".join(missingfiles))
miscutils.fwdebug_print("Creating job files - BEG")
for jobkey, jobdict in sorted(joblist.items()):
jobdict['jobnum'] = pfwutils.pad_jobnum(config.inc_jobnum())
jobdict['jobkeys'] = jobkey
jobdict['numexpwrap'] = len(jobdict['tasks'])
if miscutils.fwdebug_check(6, 'PFWBLOCK_DEBUG'):
miscutils.fwdebug_print("jobnum = %s, jobkey = %s:" %
(jobkey, jobdict['jobnum']))
jobdict['tasksfile'] = write_workflow_taskfile(
config, jobdict['jobnum'], jobdict['tasks'])
if (len(jobdict['inlist']) > 0 and
config.getfull(pfwdefs.USE_HOME_ARCHIVE_OUTPUT) != 'never'
and 'submit_files_mvmt' in config and
(pfwdefs.PF_DRYRUN not in config or not miscutils.convertBool(
config.getfull(pfwdefs.PF_DRYRUN)))):
# get home archive info
home_archive = config.getfull('home_archive')
archive_info = config[pfwdefs.SW_ARCHIVESECT][home_archive]
# load filemgmt class
attempt_tid = config['task_id']['attempt']
filemgmt = pfwutils.pfw_dynam_load_class(
dbh, config, attempt_tid, attempt_tid, "filemgmt",
archive_info['filemgmt'], archive_info)
# save file information
filemgmt.register_file_data('list', jobdict['inlist'],
config['pfw_attempt_id'],
attempt_tid, False, None, None)
pfwblock.copy_input_lists_home_archive(config, filemgmt,
archive_info,
jobdict['inlist'])
filemgmt.commit()
jobdict['inputwcltar'] = pfwblock.tar_inputfiles(
config, jobdict['jobnum'],
jobdict['inwcl'] + jobdict['inlist'])
if miscutils.convertBool(config.getfull(pfwdefs.PF_USE_DB_OUT)):
dbh.insert_job(config, jobdict)
pfwblock.write_jobwcl(config, jobkey, jobdict)
if ('glidein_use_wall' in config and miscutils.convertBool(
config.getfull('glidein_use_wall'))
and 'jobwalltime' in config):
jobdict['wall'] = config['jobwalltime']
miscutils.fwdebug_print("Creating job files - END")
numjobs = len(joblist)
if miscutils.convertBool(config.getfull(pfwdefs.PF_USE_DB_OUT)):
dbh.update_block_numexpjobs(config, numjobs)
#if miscutils.fwdebug_check(6, 'PFWBLOCK_DEBUG'):
# miscutils.fwdebug_print("inputfiles: %s, %s" % (type(inputfiles), inputfiles))
# miscutils.fwdebug_print("outputfiles: %s, %s" % (type(outputfiles), outputfiles))
#files2stage = set(inputfiles) - set(outputfiles)
#pfwblock.stage_inputs(config, files2stage)
#if pfwdefs.USE_HOME_ARCHIVE_OUTPUT in config and \
# config.getfull(pfwdefs.USE_HOME_ARCHIVE_OUTPUT).lower() == 'block':
# config['block_outputlist'] = 'potential_outputfiles.list'
# pfwblock.write_output_list(config, outputfiles)
dagfile = config.get_filename('jobdag')
pfwblock.create_jobmngr_dag(config, dagfile, scriptfile, joblist)
except:
retval = pfwdefs.PF_EXIT_FAILURE
with open(configfile, 'w') as cfgfh:
config.write(
cfgfh) # save config, have updated jobnum, wrapnum, etc
if miscutils.convertBool(config.getfull(pfwdefs.PF_USE_DB_OUT)):
dbh.end_task(config['task_id']['begblock'], retval, True)
dbh.end_task(blktid, retval, True)
raise
# save config, have updated jobnum, wrapnum, etc
with open(configfile, 'w') as cfgfh:
config.write(cfgfh)
(exists, dryrun) = config.search(pfwdefs.PF_DRYRUN)
if exists and miscutils.convertBool(dryrun):
retval = pfwdefs.PF_EXIT_DRYRUN
else:
retval = pfwdefs.PF_EXIT_SUCCESS
if miscutils.convertBool(config.getfull(pfwdefs.PF_USE_DB_OUT)):
dbh.end_task(config['task_id']['begblock'], retval, True)
miscutils.fwdebug_print("END - exiting with code %s" % retval)
return retval
def run_cmd_qcf(cmd, logfilename, wid, execnames, use_qcf=False, dbh=None, pfwattid=0, patterns={}, threaded=False):
""" Execute the command piping stdout/stderr to log and QCF """
bufsize = 1024 * 10
lasttime = time.time()
if miscutils.fwdebug_check(3, "PFWUTILS_DEBUG"):
miscutils.fwdebug_print("BEG")
miscutils.fwdebug_print(f"working dir = {os.getcwd()}")
miscutils.fwdebug_print(f"cmd = {cmd}")
miscutils.fwdebug_print(f"logfilename = {logfilename}")
miscutils.fwdebug_print(f"wid = {wid}")
miscutils.fwdebug_print(f"execnames = {execnames}")
miscutils.fwdebug_print(f"use_qcf = {use_qcf}")
use_qcf = miscutils.convertBool(use_qcf)
sys.stdout.flush()
try:
messaging = Messaging.Messaging(logfilename, execnames, pfwattid=pfwattid, taskid=wid,
dbh=dbh, usedb=use_qcf, qcf_patterns=patterns, threaded=threaded)
process_wrap = subprocess.Popen(shlex.split(cmd),
shell=False,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
except:
(extype, exvalue, _) = sys.exc_info()
print("********************")
print(f"Unexpected error: {extype} - {exvalue}")
print(f"cmd> {cmd}")
print(f"Probably could not find {cmd.split()[0]} in path")
print("Check for mispelled execname in submit wcl or")
print(" make sure that the corresponding eups package is in the metapackage ")
print(" and it sets up the path correctly")
raise
try:
buf = os.read(process_wrap.stdout.fileno(), bufsize)
while process_wrap.poll() is None or buf:
if dbh is not None:
now = time.time()
if now - lasttime > 30.*60.:
if not dbh.ping():
dbh.reconnect()
lasttime = now
messaging.write(buf)
#print buf
buf = os.read(process_wrap.stdout.fileno(), bufsize)
# brief sleep
if process_wrap.poll() is None:
time.sleep(0.1)
except IOError as exc:
print(f"\tI/O error({exc.errno}): {exc.strerror}")
except:
(extype, exvalue, _) = sys.exc_info()
print(f"\tError: Unexpected error: {extype} - {exvalue}")
raise
sys.stdout.flush()
if miscutils.fwdebug_check(3, "PFWUTILS_DEBUG"):
if process_wrap.returncode != 0:
miscutils.fwdebug_print(f"\tInfo: cmd exited with non-zero exit code = {process_wrap.returncode}")
miscutils.fwdebug_print(f"\tInfo: failed cmd = {cmd}")
else:
miscutils.fwdebug_print("\tInfo: cmd exited with exit code = 0")
if miscutils.fwdebug_check(3, "PFWUTILS_DEBUG"):
miscutils.fwdebug_print("END")
return process_wrap.returncode
def blockpost(argv=None):
"""Program entry point.
"""
if argv is None:
argv = sys.argv
# open file to catch error messages about command line
debugfh = open('blockpost.out', 'w')
sys.stdout = debugfh
sys.stderr = debugfh
print(' '.join(argv)) # print command line for debugging
print("running on %s" % (socket.gethostname()))
if len(argv) != 3:
print('Usage: blockpost.py configfile retval')
debugfh.close()
return pfwdefs.PF_EXIT_FAILURE
configfile = argv[1]
retval = int(argv[2])
if miscutils.fwdebug_check(3, 'PFWPOST_DEBUG'):
miscutils.fwdebug_print("configfile = %s" % configfile)
miscutils.fwdebug_print("retval = %s" % retval)
# read sysinfo file
config = pfwconfig.PfwConfig({'wclfile': configfile})
if miscutils.fwdebug_check(3, 'PFWPOST_DEBUG'):
miscutils.fwdebug_print("done reading config file")
blockname = config.getfull('blockname')
blkdir = config.getfull('block_dir')
# now that have more information, can rename output file
miscutils.fwdebug_print("getting new_log_name")
new_log_name = config.get_filename(
'block',
{pfwdefs.PF_CURRVALS: {
'flabel': 'blockpost',
'fsuffix': 'out'
}})
new_log_name = "%s/%s" % (blkdir, new_log_name)
miscutils.fwdebug_print("new_log_name = %s" % new_log_name)
debugfh.close()
os.chmod('blockpost.out', 0o666)
os.rename('blockpost.out', new_log_name)
debugfh = open(new_log_name, 'a+')
sys.stdout = debugfh
sys.stderr = debugfh
os.chdir(blkdir)
log_pfw_event(config, blockname, 'blockpost', 'j', ['posttask', retval])
dryrun = config.getfull(pfwdefs.PF_DRYRUN)
run = config.getfull('run')
attid = config['pfw_attempt_id']
reqnum = config.getfull(pfwdefs.REQNUM)
unitname = config.getfull(pfwdefs.UNITNAME)
attnum = config.getfull(pfwdefs.ATTNUM)
blknum = int(config.getfull(pfwdefs.PF_BLKNUM))
blktid = None
msg2 = ""
dbh = None
job_byblk = {}
wrap_byjob = {}
wrap_bymod = {}
wrapinfo = {}
jobinfo = {}
failedwraps = {}
whyfailwraps = {} # mod failures for other modname, shouldn't happen
usedb = miscutils.convertBool(config.getfull(pfwdefs.PF_USE_DB_OUT))
verify_files = miscutils.convertBool(config.getfull('verify_files'))
verify_status = 0
if verify_files and not usedb:
print('Skipping file verification due to lack of database connection')
if miscutils.convertBool(config.getfull(pfwdefs.PF_USE_DB_OUT)):
sem = None
try:
miscutils.fwdebug_print("Connecting to DB")
dbh = pfwdb.PFWDB(config.getfull('submit_des_services'),
config.getfull('submit_des_db_section'))
if verify_files:
curs = dbh.cursor()
curs.execute("select root from ops_archive where name='%s'" %
(config.getfull('home_archive')))
rows = curs.fetchall()
if rows is None or len(rows) != 1:
raise Exception(
"Invalid archive name (%s). Found %s rows in ops_archive"
% (config.getfull('home_archive'), len(rows)))
root = rows[0][0]
if not os.path.isdir(root):
print(
"Cannot read archive root directory:%s This program must be run on an NCSA machine with access to the archive storage system."
% (config.getfull('home_archive')))
sem = dbsem.DBSemaphore(
'verify_files_10', None,
config.getfull('submit_des_services'),
config.getfull('submit_des_db_section'))
print(
"\n\nVerifying archive file sizes on disk (0 is success)")
verify_status = cu.compare(
dbh=dbh,
archive=config.getfull('home_archive'),
pfwid=attid,
filesize=True,
md5sum=False,
quick=True,
debug=False,
script=False,
verbose=False,
silent=True)
if sem is not None:
del sem
print(" Verification of files returned status %i" %
(verify_status))
if verify_status != 0:
print(
" This indicates that one or more files do not have the correct file size (based on DB entries). Run"
)
print(
"\n compare_db.py --des_services %s --section %s --archive %s --pfwid %i --filesize --verbose"
% (config.getfull('submit_des_services'),
config.getfull('submit_des_db_section'),
config.getfull('home_archive'), int(attid)))
print("\n to see the details.")
if miscutils.convertBool(config.getfull(pfwdefs.PF_USE_QCF)):
import qcframework.qcfdb as qcfdb
qdbh = qcfdb.QCFDB(config.getfull('submit_des_services'),
config.getfull('submit_des_db_section'))
print("\n\nChecking non-job block task status from task table in DB (%s is success)" % \
pfwdefs.PF_EXIT_SUCCESS)
num_bltasks_failed = 0
bltasks = {}
blktid = None
if ('block' in config['task_id']
and str(blknum) in config['task_id']['block']):
blktid = int(config['task_id']['block'][str(blknum)])
miscutils.fwdebug_print("Getting block task info from DB")
start_time = time.time()
bltasks = dbh.get_block_task_info(blktid)
end_time = time.time()
miscutils.fwdebug_print(
"Done getting block task info from DB (%s secs)" %
(end_time - start_time))
for bltdict in list(bltasks.values()):
print("Block status = ", bltdict['status'])
if bltdict['status'] == pfwdefs.PF_EXIT_DRYRUN:
print("setting return value to dryrun")
retval = bltdict['status']
elif bltdict['status'] != pfwdefs.PF_EXIT_SUCCESS:
num_bltasks_failed += 1
msg2 += "\t%s" % (bltdict['name'])
if bltdict['label'] is not None:
msg2 += " - %s" % (bltdict['label'])
msg2 += " failed\n"
if bltdict['name'] == 'begblock':
# try to read the begblock.out and begblock.err files
print(
"Trying to get begblock.out and begblock.err")
msg2 += get_subblock_output("begblock")
# try to get QCF messages (especially from query codes)
begblock_tid = int(config['task_id']['begblock'])
sql = "select id from task where parent_task_id=%i and status!=0" % (
begblock_tid)
curs = dbh.cursor()
curs.execute(sql)
res = curs.fetchall()
msg2 += "\n===== QCF Messages =====\n"
msg2 += "\n begblock\n"
wrapids = [blktid, begblock_tid]
for r in res:
wrapids.append(r[0])
wrapmsg = {}
if qdbh is not None:
miscutils.fwdebug_print(
"Querying QCF messages")
start_time = time.time()
wrapmsg = qdbh.get_qcf_messages_for_wrappers(
wrapids)
end_time = time.time()
miscutils.fwdebug_print(
"Done querying QCF messages (%s secs)" %
(end_time - start_time))
miscutils.fwdebug_print("wrapmsg = %s" %
wrapmsg)
if len(wrapmsg) == 0:
msg2 += " No QCF messages\n"
else:
for msgs in list(wrapmsg.values()):
for m in msgs:
msg2 += " " + m['message'] + "\n"
retval = pfwdefs.PF_EXIT_FAILURE
if retval != pfwdefs.PF_EXIT_DRYRUN:
print("\n\nChecking job status from pfw_job table in DB (%s is success)" % \
pfwdefs.PF_EXIT_SUCCESS)
miscutils.fwdebug_print("Getting job info from DB")
start_time = time.time()
jobinfo = dbh.get_job_info({'pfw_block_task_id': blktid})
end_time = time.time()
miscutils.fwdebug_print(
"Done getting job info from DB (%s secs)" %
(end_time - start_time))
miscutils.fwdebug_print("Getting wrapper info from DB")
start_time = time.time()
wrapinfo = dbh.get_wrapper_info(pfw_attempt_id=attid,
pfw_block_task_id=blktid)
end_time = time.time()
miscutils.fwdebug_print(
"Done getting wrapper info from DB (%s secs)" %
(end_time - start_time))
else:
msg = "Could not find task id for block %s in config.des" % blockname
print("Error:", msg)
if 'attempt' in config['task_id']:
miscutils.fwdebug_print("Saving pfw message")
start_time = time.time()
Messaging.pfw_message(dbh, attid,
config['task_id']['attempt'], msg,
pfw_utils.PFW_DB_INFO,
'blockpost.out', 0)
end_time = time.time()
miscutils.fwdebug_print(
"Done saving pfw message (%s secs)" %
(end_time - start_time))
print("all the task ids:", config['task_id'])
archive = None
if pfwdefs.HOME_ARCHIVE in config:
archive = config.getfull(pfwdefs.HOME_ARCHIVE)
logfullnames = dbh.get_fail_log_fullnames(attid, archive)
dbh.close()
print("len(jobinfo) = ", len(jobinfo))
print("len(wrapinfo) = ", len(wrapinfo))
job_byblk = pfwutils.index_job_info(jobinfo)
print("blktid: ", blktid)
print("job_byblk:", job_byblk)
if blktid not in job_byblk:
print("Warn: could not find jobs for block %s" % blknum)
print(" This is ok if attempt died before jobs ran")
print(" block task_ids in job_byblk:" %
list(job_byblk.keys()))
else:
wrap_byjob, wrap_bymod = pfwutils.index_wrapper_info(wrapinfo)
#print "wrap_byjob:", wrap_byjob
#print "wrap_bymod:", wrap_bymod
for jobtid, jobdict in sorted(job_byblk[blktid].items()):
failedwraps[jobtid] = []
whyfailwraps[jobtid] = []
jobkeys = ""
# don't print out successful wrappers
if jobtid in wrap_byjob and jobdict[
'status'] == pfwdefs.PF_EXIT_SUCCESS:
continue
if jobdict['jobkeys'] is not None:
jobkeys = jobdict['jobkeys']
#print "jobkeys = ", jobkeys, type(jobkeys)
submit_job_path = "%s/B%02d-%s/%04d" % (
config.getfull('work_dir'),
int(config.getfull('blknum')),
config.getfull('blockname'), int(jobdict['jobnum']))
msg2 += "\n\t%s (%s) " % (pfwutils.pad_jobnum(
jobdict['jobnum']), jobkeys)
if jobtid not in wrap_byjob:
msg2 += "\tNo wrapper instances"
else:
#print "wrapnum in job =", wrap_byjob[jobtid].keys()
maxwrap = max(wrap_byjob[jobtid].keys())
#print "maxwrap =", maxwrap
modname = wrap_byjob[jobtid][maxwrap]['modname']
#print "modname =", modname
msg2 += "%d/%s %s" % (len(
wrap_byjob[jobtid]), jobdict['expect_num_wrap'],
modname)
# determine wrappers for this job without success exit
for wrapnum, wdict in list(wrap_byjob[jobtid].items()):
if wdict['status'] is None or wdict[
'status'] != pfwdefs.PF_EXIT_SUCCESS:
if wdict['modname'] == modname:
failedwraps[jobtid].append(wrapnum)
else:
whyfailwraps[jobtid].append(wrapnum)
if jobdict['status'] == pfwdefs.PF_EXIT_EUPS_FAILURE:
msg2 += " - FAIL - EUPS setup failure"
retval = jobdict['status']
elif jobdict['status'] == pfwdefs.PF_EXIT_CONDOR:
msg2 += " - FAIL - Condor/Globus failure"
retval = jobdict['status']
elif jobdict['status'] is None:
msg2 += " - FAIL - NULL status"
retval = pfwdefs.PF_EXIT_FAILURE
elif jobdict['status'] != pfwdefs.PF_EXIT_SUCCESS:
msg2 += " - FAIL - Non-zero status"
retval = jobdict['status']
if jobdict['status'] != pfwdefs.PF_EXIT_SUCCESS:
msg2 += "\n\t\t%s/runjob.out " % (submit_job_path)
msg2 += '\n'
# print pfw_messages
if 'message' in jobdict:
print(jobdict['message'])
for msgdict in sorted(jobdict['message'],
key=lambda k: k['message_time']):
level = int(msgdict['message_lvl'])
levelstr = 'info'
if level == pfwdefs.PFWDB_MSG_WARN:
levelstr = 'WARN'
elif level == pfwdefs.PFWDB_MSG_ERROR:
levelstr = 'ERROR'
msg2 += "\t\t%s - %s\n" % (
levelstr, msgdict['message'].replace(
'\n', '\n\t\t\t'))
if jobtid in wrap_byjob:
# print log file name for failed/unfinished wrappers
for wrapnum in failedwraps[jobtid]:
wrapdict = wrap_byjob[jobtid][wrapnum]
if wrapdict['log'] in logfullnames:
msg2 += "\t\t%s - %s\n" % (
wrapnum, logfullnames[wrapdict['log']])
else:
msg2 += "\t\t%s - Could not find log in archive (%s)\n" % (
wrapnum, wrapdict['log'])
wrapmsg = get_qcf_messages(qdbh, config,
[wrapdict['task_id']])
msg2 = print_qcf_messages(config, wrapdict,
wrapmsg, msg2)
msg2 += '\n'
# If weirdness happened in run, print a message
if len(whyfailwraps[jobtid]) > 0:
msg2 += "\n*** Contact framework developers. Wrappers ran after at least 1 wrapper from a previous module that doesn't have success status.\n"
msg2 += "\t%s\n" % ','.join(whyfailwraps[jobtid])
except Exception as exc:
if sem is not None:
del sem
msg2 += "\n\nEncountered error trying to gather status information for email."
msg2 += "\nCheck output for blockpost for further details."
print(
"\n\nEncountered error trying to gather status information for email"
)
print("%s: %s" % (exc.__class__.__name__, str(exc)))
(extype, exvalue, trback) = sys.exc_info()
traceback.print_exception(extype, exvalue, trback, file=sys.stdout)
retval = pfwdefs.PF_EXIT_FAILURE
retval = int(retval) + verify_status
print("before email retval =", retval)
when_to_email = 'run'
if 'when_to_email' in config:
when_to_email = config.getfull('when_to_email').lower()
if miscutils.convertBool(dryrun):
if when_to_email != 'never':
print("dryrun = ", dryrun)
print("Sending dryrun email")
if retval == pfwdefs.PF_EXIT_DRYRUN:
msg1 = "%s: In dryrun mode, block %s has finished successfully." % (
run, blockname)
else:
msg1 = "%s: In dryrun mode, block %s has failed." % (
run, blockname)
send_email(config, blockname, retval, "", msg1, msg2)
else:
print("Not sending dryrun email")
print("retval = ", retval)
retval = pfwdefs.PF_EXIT_DRYRUN
elif retval:
if when_to_email != 'never':
print("Sending block failed email\n")
msg1 = "%s: block %s has failed." % (run, blockname)
send_email(config, blockname, retval, "", msg1, msg2)
else:
print("Not sending failed email")
print("retval = ", retval)
elif retval == pfwdefs.PF_EXIT_SUCCESS:
if when_to_email == 'block':
msg1 = "%s: block %s has finished successfully." % (run,
blockname)
msg2 = ""
print("Sending success email\n")
send_email(config, blockname, retval, "", msg1, msg2)
elif when_to_email == 'run':
numblocks = len(
miscutils.fwsplit(config[pfwdefs.SW_BLOCKLIST], ','))
if int(config[pfwdefs.PF_BLKNUM]) == numblocks:
msg1 = "%s: run has finished successfully." % (run)
msg2 = ""
print("Sending success email\n")
send_email(config, blockname, retval, "", msg1, msg2)
else:
print("Not sending run email because not last block")
print("retval = ", retval)
else:
print("Not sending success email")
print("retval = ", retval)
else:
print("Not sending email")
print("retval = ", retval)
# Store values in DB and hist file
dbh = None
if miscutils.convertBool(config[pfwdefs.PF_USE_DB_OUT]):
dbh = pfwdb.PFWDB(config.getfull('submit_des_services'),
config.getfull('submit_des_db_section'))
if blktid is not None:
print("Updating end of block task", blktid)
dbh.end_task(blktid, retval, True)
else:
print("Could not update end of block task without block task id")
if retval != pfwdefs.PF_EXIT_SUCCESS:
print("Updating end of attempt", config['task_id']['attempt'])
dbh.end_task(config['task_id']['attempt'], retval, True)
dbh.commit()
dbh.close()
print("before next block retval = ", retval)
if retval == pfwdefs.PF_EXIT_SUCCESS:
# Get ready for next block
config.inc_blknum()
with open(configfile, 'w') as cfgfh:
config.write(cfgfh)
print("new blknum = ", config[pfwdefs.PF_BLKNUM])
print("number of blocks = ",
len(miscutils.fwsplit(config[pfwdefs.SW_BLOCKLIST], ',')))
miscutils.fwdebug_print("Returning retval = %s (%s)" %
(retval, type(retval)))
miscutils.fwdebug_print("END")
debugfh.close()
return int(retval)
def summary(argv=None):
""" Create and send summary email """
if argv is None:
argv = sys.argv
debugfh = open('summary.out', 'w')
sys.stdout = debugfh
sys.stderr = debugfh
print(' '.join(argv))
if len(argv) < 2:
print("Usage: summary configfile status")
debugfh.close()
return pfwdefs.PF_EXIT_FAILURE
if len(argv) == 3:
status = argv[2]
# dagman always exits with 0 or 1
if status == 1:
status = pfwdefs.PF_EXIT_FAILURE
else:
print("summary: Missing status value")
status = None
# read sysinfo file
config = pfwconfig.PfwConfig({'wclfile': argv[1]})
log_pfw_event(config, 'process', 'mngr', 'j', ['posttask', status])
msgstr = ""
msg1 = ""
subject = ""
if not status:
msg1 = f"Processing finished with unknown results.\n{msgstr}"
elif pfwdefs.PF_DRYRUN in config and miscutils.convertBool(
config.getfull(pfwdefs.PF_DRYRUN)):
msg1 = f"Processing ended after DRYRUN\n{msgstr}"
if int(status) == pfwdefs.PF_EXIT_SUCCESS:
msg1 = "Processing has successfully completed.\n"
subject = ""
else:
print(f"status = '{status}'")
print("type(status) =", type(status))
print(f"SUCCESS = '{pfwdefs.PF_EXIT_SUCCESS}'")
print("type(SUCCESS) =", type(pfwdefs.PF_EXIT_SUCCESS))
msg1 = f"Processing aborted with status {status}.\n"
subject = ""
pfwemail.send_email(config, "processing", status, subject, msg1, '')
if miscutils.convertBool(config.getfull(pfwdefs.PF_USE_DB_OUT)):
dbh = pfwdb.PFWDB(config.getfull('submit_des_services'),
config.getfull('submit_des_db_section'))
dbh.update_attempt_end_vals(config['pfw_attempt_id'], status)
print(f"summary: status = '{status}'")
print("summary:", msg1)
print("summary: End")
debugfh.close()
return status
def check_dataobjs(config, modname, moddict, dataobjs, indent=''):
""" calls functions to check files have all needed info as well as note extra file defs """
cnts = [0] * NUMCNTS
# check every file
if pfwdefs.SW_FILESECT in moddict:
print "%sChecking %s section..." % (indent, pfwdefs.SW_FILESECT)
for fname, fdict in moddict[pfwdefs.SW_FILESECT].items():
key = '%s.%s' % (pfwdefs.SW_FILESECT, fname)
if key not in dataobjs[pfwdefs.SW_INPUTS] and \
key not in dataobjs[pfwdefs.SW_OUTPUTS] and \
('listonly' not in fdict or not miscutils.convertBool(fdict['listonly'])):
warning(indent + ' ', "%s.%s does not appear in provenance lines" % \
(pfwdefs.SW_FILESECT, fname))
cnts[WARNCNT_POS] += 1
if key in dataobjs[pfwdefs.SW_INPUTS]:
cnts2 = check_file_valid_input(config, modname,
fname, fdict, indent+' ')
cnts = [x + y for x, y in zip(cnts, cnts2)] # increment counts
if key in dataobjs[pfwdefs.SW_OUTPUTS]:
cnts2 = check_file_valid_output(config, modname,
fname, fdict, indent+' ')
cnts = [x + y for x, y in zip(cnts, cnts2)] # increment counts
# check every list
if pfwdefs.SW_LISTSECT in moddict:
print "%sChecking %s section..." % (indent, pfwdefs.SW_LISTSECT)
for lname, ldict in moddict[pfwdefs.SW_LISTSECT].items():
key = '%s.%s' % (pfwdefs.SW_LISTSECT, lname)
if key not in dataobjs[pfwdefs.SW_INPUTS] and \
key not in dataobjs[pfwdefs.SW_OUTPUTS]:
found = False
if 'columns' in ldict:
for col in ldict['columns'].split(','):
nkey = key + "." + col
nkey = nkey.replace('.fullname', '')
if nkey in dataobjs[pfwdefs.SW_INPUTS] or \
nkey in dataobjs[pfwdefs.SW_OUTPUTS]:
found = True
# check to see if list def has file name
if not found:
nkey = col
nkey = 'file.' + nkey.replace('.fullname', '')
if nkey in dataobjs[pfwdefs.SW_INPUTS] or \
nkey in dataobjs[pfwdefs.SW_OUTPUTS]:
found = True
if not found:
warning(indent + ' ', "%s.%s does not appear in provenance lines" % \
(pfwdefs.SW_LISTSECT, lname))
cnts[WARNCNT_POS] += 1
if key in dataobjs[pfwdefs.SW_INPUTS]:
cnts2 = check_list_valid_input(modname,
lname, ldict, indent+' ')
cnts = [x + y for x, y in zip(cnts, cnts2)] # increment counts
return cnts
def check_exec_inputs(config, modname, dataobjs, xsectname, xsectdict, indent=''):
""" Check exec input definition is valid """
cnts = [0] * NUMCNTS
moddict = config[pfwdefs.SW_MODULESECT][modname]
if pfwdefs.SW_INPUTS in xsectdict:
print "%sChecking %s %s..." % (indent, xsectname, pfwdefs.SW_INPUTS)
indent += ' '
#print "%sxsectdict[pfwdefs.SW_INPUTS] = %s" % (indent, xsectdict[pfwdefs.SW_INPUTS])
# for each entry in inputs
for objname in miscutils.fwsplit(xsectdict[pfwdefs.SW_INPUTS], ','):
objname = objname.lower()
(sect, name, subname) = parse_wcl_objname(objname)
if sect is None:
error(indent+' ', "module %s, %s, %s - Invalid entry (%s). Missing section label" % (modname, xsectname, pfwdefs.SW_INPUTS, objname))
cnts[ERRCNT_POS] += 1
else:
# check that appears in [file/list]sect : error
bad = False
if sect not in moddict or name not in moddict[sect]:
found = False
if 'loopobj' in moddict and moddict['loopobj'].startswith(sect) and sect in moddict:
temp = moddict['loopobj'].split('.')[1:]
d = moddict[sect]
for t in temp:
if t in d:
d = d[t]
if name in d:
found = True
else:
if 'div_list_by_col' in d:
if name in d['div_list_by_col']:
found = True
moddict[sect][name] = d['div_list_by_col'][name]
if not found:
bad = True
error(indent+' ', "module %s, %s, %s - Invalid entry (%s). Cannot find definition." % (modname, xsectname, pfwdefs.SW_INPUTS, objname))
cnts[ERRCNT_POS] += 1
if not bad:
if subname is None: # file
dataobjs[pfwdefs.SW_INPUTS][objname] = True
elif sect != pfwdefs.SW_LISTSECT: # only lists can have subname
error(indent+' ', "module %s, %s, %s, %s - Too many sections/periods for a %s." % (modname, xsectname, pfwdefs.SW_INPUTS, objname, sect))
cnts[ERRCNT_POS] += 1
elif subname not in moddict[pfwdefs.SW_FILESECT]:
error(indent+' ', "module %s, %s, %s, %s - Cannot find definition for %s" % (modname, xsectname, pfwdefs.SW_INPUTS, objname, subname))
cnts[ERRCNT_POS] += 1
else:
dataobjs[pfwdefs.SW_INPUTS]["%s.%s" % (pfwdefs.SW_LISTSECT, name)] = True
dataobjs[pfwdefs.SW_INPUTS]["%s.%s" % (pfwdefs.SW_FILESECT, subname)] = True
dataobjs[pfwdefs.SW_INPUTS][objname] = True
fdict = moddict[pfwdefs.SW_FILESECT][subname]
if ('listonly' not in fdict or not miscutils.convertBool(fdict['listonly'])):
warning(indent, "module %s, %s, %s, %s - File in list does not have listonly=True" % (modname, xsectname, pfwdefs.SW_INPUTS, objname))
cnts[WARNCNT_POS] += 1
return cnts
def run_cmd_qcf(cmd,
logfilename,
wid,
execnames,
use_qcf=False,
dbh=None,
pfwattid=0,
patterns={}):
"""Execute the command piping stdout/stderr to log and QCF.
"""
bufsize = 1024 * 10
if miscutils.fwdebug_check(3, "PFWUTILS_DEBUG"):
miscutils.fwdebug_print("BEG")
miscutils.fwdebug_print("working dir = %s" % (os.getcwd()))
miscutils.fwdebug_print("cmd = %s" % cmd)
miscutils.fwdebug_print("logfilename = %s" % logfilename)
miscutils.fwdebug_print("wid = %s" % wid)
miscutils.fwdebug_print("execnames = %s" % execnames)
miscutils.fwdebug_print("use_qcf = %s" % use_qcf)
use_qcf = miscutils.convertBool(use_qcf)
sys.stdout.flush()
try:
messaging = Messaging.Messaging(logfilename,
execnames,
pfwattid=pfwattid,
taskid=wid,
dbh=dbh,
usedb=use_qcf,
qcf_patterns=patterns)
process_wrap = subprocess.Popen(shlex.split(cmd),
shell=False,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
except:
(extype, exvalue, _) = sys.exc_info()
print("********************")
print("Unexpected error: %s - %s" % (extype, exvalue))
print("cmd> %s" % cmd)
print("Probably could not find %s in path" % cmd.split()[0])
print("Check for mispelled execname in submit wcl or")
print(
" make sure that the corresponding eups package is in the metapackage "
)
print(" and it sets up the path correctly")
raise
try:
buf = os.read(process_wrap.stdout.fileno(), bufsize).decode()
while process_wrap.poll() == None or len(buf) != 0:
messaging.write(buf)
buf = os.read(process_wrap.stdout.fileno(), bufsize).decode()
except IOError as exc:
print("\tI/O error({0}): {1}".format(exc.errno, exc.strerror))
except:
(extype, exvalue, _) = sys.exc_info()
print("\tError: Unexpected error: %s - %s" % (extype, exvalue))
raise
sys.stdout.flush()
if miscutils.fwdebug_check(3, "PFWUTILS_DEBUG"):
if process_wrap.returncode != 0:
miscutils.fwdebug_print(
"\tInfo: cmd exited with non-zero exit code = %s" %
process_wrap.returncode)
miscutils.fwdebug_print("\tInfo: failed cmd = %s" % cmd)
else:
miscutils.fwdebug_print("\tInfo: cmd exited with exit code = 0")
if miscutils.fwdebug_check(3, "PFWUTILS_DEBUG"):
miscutils.fwdebug_print("END")
return process_wrap.returncode
def jobpost(argv=None):
"""Performs steps needed after a pipeline job.
"""
condor2db = {'jobid': 'condor_job_id',
'csubmittime': 'condor_submit_time',
'gsubmittime': 'target_submit_time',
'starttime': 'condor_start_time',
'endtime': 'condor_end_time'}
if argv is None:
argv = sys.argv
debugfh = tempfile.NamedTemporaryFile(mode='w+', prefix='jobpost_', dir='.', delete=False)
tmpfn = debugfh.name
sys.stdout = debugfh
sys.stderr = debugfh
miscutils.fwdebug_print("temp log name = %s" % tmpfn)
print('cmd>', ' '.join(argv)) # print command line for debugging
if len(argv) < 7:
# open file to catch error messages about command line
print('Usage: jobpost.py configfile block jobnum inputtar outputtar retval')
debugfh.close()
return pfwdefs.PF_EXIT_FAILURE
configfile = argv[1]
blockname = argv[2]
jobnum = argv[3]
inputtar = argv[4]
outputtar = argv[5]
retval = pfwdefs.PF_EXIT_FAILURE
if len(argv) == 7:
retval = int(sys.argv[6])
if miscutils.fwdebug_check(3, 'PFWPOST_DEBUG'):
miscutils.fwdebug_print("configfile = %s" % configfile)
miscutils.fwdebug_print("block = %s" % blockname)
miscutils.fwdebug_print("jobnum = %s" % jobnum)
miscutils.fwdebug_print("inputtar = %s" % inputtar)
miscutils.fwdebug_print("outputtar = %s" % outputtar)
miscutils.fwdebug_print("retval = %s" % retval)
# read sysinfo file
config = pfwconfig.PfwConfig({'wclfile': configfile})
if miscutils.fwdebug_check(3, 'PFWPOST_DEBUG'):
miscutils.fwdebug_print("done reading config file")
# now that have more information, rename output file
if miscutils.fwdebug_check(3, 'PFWPOST_DEBUG'):
miscutils.fwdebug_print("before get_filename")
blockname = config.getfull('blockname')
blkdir = config.getfull('block_dir')
tjpad = pfwutils.pad_jobnum(jobnum)
os.chdir("%s/%s" % (blkdir, tjpad))
new_log_name = config.get_filename('job', {pfwdefs.PF_CURRVALS: {pfwdefs.PF_JOBNUM: jobnum,
'flabel': 'jobpost',
'fsuffix': 'out'}})
new_log_name = "%s" % (new_log_name)
miscutils.fwdebug_print("new_log_name = %s" % new_log_name)
debugfh.close()
os.chmod(tmpfn, 0o666)
os.rename(tmpfn, new_log_name)
debugfh = open(new_log_name, 'a+')
sys.stdout = debugfh
sys.stderr = debugfh
dbh = None
if miscutils.convertBool(config.getfull(pfwdefs.PF_USE_DB_OUT)):
dbh = pfwdb.PFWDB(config.getfull('submit_des_services'),
config.getfull('submit_des_db_section'))
# get job information from the job stdout if exists
(tjobinfo, tjobinfo_task) = parse_job_output(config, jobnum, dbh, retval)
if dbh and len(tjobinfo) > 0:
print("tjobinfo: ", tjobinfo)
dbh.update_tjob_info(config['task_id']['job'][jobnum], tjobinfo)
# get job information from the condor job log
logfilename = 'runjob.log'
if os.path.exists(logfilename) and os.path.getsize(logfilename) > 0: # if made it to submitting/running jobs
try:
# update job info in DB from condor log
print("Updating job info in DB from condor log")
condorjobinfo = pfwcondor.parse_condor_user_log(logfilename)
if len(list(condorjobinfo.keys())) > 1:
print("More than single job in job log")
j = list(condorjobinfo.keys())[0]
cjobinfo = condorjobinfo[j]
djobinfo = {}
for ckey, dkey in list(condor2db.items()):
if ckey in cjobinfo:
djobinfo[dkey] = cjobinfo[ckey]
print(djobinfo)
dbh.update_job_info(config, cjobinfo['jobname'], djobinfo)
if 'holdreason' in cjobinfo and cjobinfo['holdreason'] is not None:
msg = "Condor HoldReason: %s" % cjobinfo['holdreason']
print(msg)
if dbh:
Messaging.pfw_message(dbh, config['pfw_attempt_id'],
config['task_id']['job'][jobnum],
msg, pfwdefs.PFWDB_MSG_WARN)
if 'abortreason' in cjobinfo and cjobinfo['abortreason'] is not None:
tjobinfo_task['start_time'] = cjobinfo['starttime']
tjobinfo_task['end_time'] = cjobinfo['endtime']
if 'condor_rm' in cjobinfo['abortreason']:
tjobinfo_task['status'] = pfwdefs.PF_EXIT_OPDELETE
else:
tjobinfo_task['status'] = pfwdefs.PF_EXIT_CONDOR
else:
pass
except Exception:
(extype, exvalue, trback) = sys.exc_info()
traceback.print_exception(extype, exvalue, trback, file=sys.stdout)
else:
print("Warning: no job condor log file")
if dbh:
# update job task
if 'status' not in tjobinfo_task:
tjobinfo_task['status'] = pfwdefs.PF_EXIT_CONDOR
if 'end_time' not in tjobinfo_task:
tjobinfo_task['end_time'] = datetime.now()
wherevals = {'id': config['task_id']['job'][jobnum]}
dbh.basic_update_row('task', tjobinfo_task, wherevals)
dbh.commit()
log_pfw_event(config, blockname, jobnum, 'j', ['posttask', retval])
# input wcl should already exist in untar form
if os.path.exists(inputtar):
print("found inputtar: %s" % inputtar)
os.unlink(inputtar)
else:
print("Could not find inputtar: %s" % inputtar)
# untar output wcl tar and delete tar
if os.path.exists(outputtar):
print("Size of output wcl tar:", os.path.getsize(outputtar))
if os.path.getsize(outputtar) > 0:
print("found outputtar: %s" % outputtar)
pfwutils.untar_dir(outputtar, '..')
os.unlink(outputtar)
else:
msg = "Warn: outputwcl tarball (%s) is 0 bytes." % outputtar
print(msg)
if dbh:
Messaging.pfw_message(dbh, config['pfw_attempt_id'],
config['task_id']['job'][jobnum],
msg, pfwdefs.PFWDB_MSG_WARN)
else:
msg = "Warn: outputwcl tarball (%s) does not exist." % outputtar
print(msg)
if dbh:
Messaging.pfw_message(dbh, config['pfw_attempt_id'],
config['task_id']['job'][jobnum],
msg, pfwdefs.PFWDB_MSG_WARN)
if retval != pfwdefs.PF_EXIT_SUCCESS:
miscutils.fwdebug_print("Setting failure retval")
retval = pfwdefs.PF_EXIT_FAILURE
miscutils.fwdebug_print("Returning retval = %s" % retval)
miscutils.fwdebug_print("jobpost done")
debugfh.close()
return int(retval)
def jobpre(argv=None):
""" Program entry point """
if argv is None:
argv = sys.argv
#debugfh = tempfile.NamedTemporaryFile(prefix='jobpre_', dir='.', delete=False)
default_log = f"jobpre_{random.randint(1,10000000):08d}.out"
debugfh = open(default_log, 'w')
tmpfn = debugfh.name
outorig = sys.stdout
errorig = sys.stderr
sys.stdout = debugfh
sys.stderr = debugfh
print(' '.join(argv)) # command line for debugging
print(os.getcwd())
if len(argv) < 3:
print("Usage: jobpre configfile jobnum")
debugfh.close()
return pfwdefs.PF_EXIT_FAILURE
configfile = sys.argv[1]
jobnum = sys.argv[2] # could also be uberctrl
# read wcl file
config = pfwconfig.PfwConfig({'wclfile': configfile})
blockname = config.getfull('blockname')
blkdir = config.get('block_dir')
tjpad = pfwutils.pad_jobnum(jobnum)
# now that have more information, can rename output file
miscutils.fwdebug_print("getting new_log_name")
new_log_name = config.get_filename('job', {pfwdefs.PF_CURRVALS: {pfwdefs.PF_JOBNUM:jobnum,
'flabel': 'jobpre',
'fsuffix':'out'}})
new_log_name = f"{blkdir}/{tjpad}/{new_log_name}"
miscutils.fwdebug_print(f"new_log_name = {new_log_name}")
debugfh.close()
sys.stdout = outorig
sys.stderr = errorig
os.chmod(tmpfn, 0o666)
os.rename(tmpfn, new_log_name)
dbh = None
if miscutils.convertBool(config.getfull(pfwdefs.PF_USE_DB_OUT)):
if config.dbh is None:
dbh = pfwdb.PFWDB(config.getfull('submit_des_services'),
config.getfull('submit_des_db_section'))
else:
dbh = config.dbh
if 'use_qcf' in config and config['use_qcf']:
debugfh = Messaging.Messaging(new_log_name, 'jobpre.py', config['pfw_attempt_id'], dbh=dbh, mode='a+', usedb=dbh is not None)
else:
debugfh = open(new_log_name, 'a+')
sys.stdout = debugfh
sys.stderr = debugfh
if miscutils.convertBool(config.getfull(pfwdefs.PF_USE_DB_OUT)):
ctstr = dbh.get_current_timestamp_str()
dbh.update_job_info(config, tjpad, {'condor_submit_time': ctstr,
'target_submit_time': ctstr})
log_pfw_event(config, blockname, tjpad, 'j', ['pretask'])
miscutils.fwdebug_print("jobpre done")
debugfh.close()
sys.stdout = outorig
sys.stderr = errorig
return pfwdefs.PF_EXIT_SUCCESS