def _get_update_values_explicit(self, update_info):
""" include values explicitly set by operator/framework """
upinfo2 = collections.OrderedDict()
# for each set of header updates
for updset in update_info.values():
headers = ['0'] # default to primary header
if 'headers' in updset:
headers = miscutils.fwsplit(update_info[updset], ',')
hdu_updset = collections.OrderedDict()
for key, val in updset.items():
if key != 'headers':
uval = ucomment = udatatype = None
header_info = miscutils.fwsplit(val, '/')
uval = header_info[0]
if len(header_info) == 3:
ucomment = header_info[1]
udatatype = header_info[2]
hdu_updset[key] = (uval, ucomment, udatatype)
for hdname in headers:
if hdname not in update_info:
upinfo2[hdname] = collections.OrderedDict()
upinfo2[hdname].update(hdu_updset)
return upinfo2
def read_fullnames_from_listfile(listfile, linefmt, colstr):
""" Read a list file returning fullnames from the list """
if miscutils.fwdebug_check(3, 'INTGMISC_DEBUG'):
miscutils.fwdebug_print('colstr=%s' % colstr)
columns = convert_col_string_to_list(colstr, False)
if miscutils.fwdebug_check(3, 'INTGMISC_DEBUG'):
miscutils.fwdebug_print('columns=%s' % columns)
fullnames = {}
pos2fsect = {}
for pos in range(0, len(columns)):
lcol = columns[pos].lower()
if lcol.endswith('.fullname'):
filesect = lcol[:-9]
pos2fsect[pos] = filesect
fullnames[filesect] = []
# else a data column instead of a filename
if miscutils.fwdebug_check(3, 'INTGMISC_DEBUG'):
miscutils.fwdebug_print('pos2fsect=%s' % pos2fsect)
if linefmt == 'config' or linefmt == 'wcl':
miscutils.fwdie(
'Error: wcl list format not currently supported (%s)' % listfile,
1)
else:
with open(listfile, 'r') as listfh:
for line in listfh:
line = line.strip()
# convert line into python list
lineinfo = []
if linefmt == 'textcsv':
lineinfo = miscutils.fwsplit(line, ',')
elif linefmt == 'texttab':
lineinfo = miscutils.fwsplit(line, '\t')
elif linefmt == 'textsp':
lineinfo = miscutils.fwsplit(line, ' ')
else:
miscutils.fwdie('Error: unknown linefmt (%s)' % linefmt,
1)
# save each fullname in line
for pos in pos2fsect:
# use common routine to parse actual fullname (e.g., remove [0])
parsemask = miscutils.CU_PARSE_PATH | miscutils.CU_PARSE_FILENAME | \
miscutils.CU_PARSE_COMPRESSION
(path, filename, compression) = miscutils.parse_fullname(
lineinfo[pos], parsemask)
fname = "%s/%s" % (path, filename)
if compression is not None:
fname += compression
fullnames[pos2fsect[pos]].append(fname)
if miscutils.fwdebug_check(6, 'INTGMISC_DEBUG'):
miscutils.fwdebug_print('fullnames = %s' % fullnames)
return fullnames
def process_csv(fncsv, delim, version, fnskymap, ocurs):
datacnt = 0
with open(fncsv) as infh:
# first line has headers
line = infh.readline()
headers = miscutils.fwsplit(line, delim)
print headers
# read data
line = infh.readline()
while line:
line = line.split('#')[0]
line = line.strip()
data = miscutils.fwsplit(line, delim)
svals = dict(zip(headers, data))
newpatch = re.sub(oldpatchsep, newpatchsep, svals['patch'])
ovals = {
'tract': svals['tract'],
'patch': newpatch,
'visit': svals['visit'],
'ccd': svals['ccd'],
'version': version,
'skymap': fnskymap
}
datacnt += 1
insert_row_oracle(ocurs, ovals)
line = infh.readline()
print datacnt, "lines from csv"
def check_block(config, indent=''):
""" check blocks level defs """
cnts = [0] * NUMCNTS
blocklist = miscutils.fwsplit(config[pfwdefs.SW_BLOCKLIST].lower(), ',')
for blockname in blocklist:
print "%sChecking block %s..." % (indent, blockname)
config.set_block_info()
# check modules
block = config[pfwdefs.SW_BLOCKSECT][blockname]
if pfwdefs.SW_MODULELIST in block:
modulelist = miscutils.fwsplit(block[pfwdefs.SW_MODULELIST].lower(), ',')
for modname in modulelist:
if modname not in config[pfwdefs.SW_MODULESECT]:
error(indent+' ', "block %s - invalid %s" % (blockname, pfwdefs.SW_MODULELIST))
print "%s (bad module name: %s, list: %s)" % (indent, modname, modulelist)
cnts[ERRCNT_POS] += 1
else:
cnts2 = check_module(config, blockname, modname, indent+' ')
cnts = [x + y for x, y in zip(cnts, cnts2)] # increment counts
else:
error(indent+' ', "block %s - missing %s value" % (blockname, pfwdefs.SW_MODULESECT))
cnts[ERRCNT_POS] += 1
config.inc_blknum()
config.reset_blknum()
return cnts
def check_block(config, indent=''):
"""Check blocks level defs.
"""
cnts = [0] * NUMCNTS
blocklist = miscutils.fwsplit(config[pfwdefs.SW_BLOCKLIST].lower(), ',')
for blockname in blocklist:
print("%sChecking block %s..." % (indent, blockname))
config.set_block_info()
for key in [pfwdefs.PF_USE_DB_IN, pfwdefs.PF_USE_DB_OUT]:
if key in config and miscutils.convertBool(config.getfull(key)):
(found, val) = config.search('target_des_db_section')
if not found:
error(
indent + ' ',
"using DB (%s), but missing target_des_db_section" %
(key))
cnts[ERRCNT_POS] += 1
(found, val) = config.search('target_des_services')
if not found:
error(
indent + ' ',
"using DB (%s), but missing target_des_services" %
(key))
cnts[ERRCNT_POS] += 1
# check modules
block = config[pfwdefs.SW_BLOCKSECT][blockname]
if pfwdefs.SW_MODULELIST in block:
modulelist = miscutils.fwsplit(
block[pfwdefs.SW_MODULELIST].lower(), ',')
for modname in modulelist:
if modname not in config[pfwdefs.SW_MODULESECT]:
error(
indent + ' ', "block %s - invalid %s" %
(blockname, pfwdefs.SW_MODULELIST))
print("%s (bad module name: %s, list: %s)" %
(indent, modname, modulelist))
cnts[ERRCNT_POS] += 1
else:
cnts2 = check_module(config, blockname, modname,
indent + ' ')
cnts = [x + y
for x, y in zip(cnts, cnts2)] # increment counts
else:
error(
indent + ' ', "block %s - missing %s value" %
(blockname, pfwdefs.SW_MODULESECT))
cnts[ERRCNT_POS] += 1
config.inc_blknum()
config.reset_blknum()
return cnts
def check_exec_parentchild(modname, dataobjs, xsectname, xsectdict, indent=''):
#pylint: disable=unbalanced-tuple-unpacking
""" Check that parent and children appear in inputs and outputs """
# assumes check_exec_input and check_exec_output have already been executed so there are entries in dataobjs
cnts = [0] * NUMCNTS
if pfwdefs.SW_PARENTCHILD in xsectdict:
print "%sChecking %s %s..." % (indent, xsectname,
pfwdefs.SW_PARENTCHILD)
indent += ' '
#print "%sxsectdict[pfwdefs.SW_PARENTCHILD] = %s" % (indent, xsectdict[pfwdefs.SW_PARENTCHILD])
#print "%sdataobjs[pfwdefs.SW_INPUTS] = %s" % (indent, dataobjs[pfwdefs.SW_INPUTS])
#print "%sdataobjs[pfwdefs.SW_OUTPUTS] = %s" % (indent, dataobjs[pfwdefs.SW_OUTPUTS])
#print "%sfsplit = %s" % (indent, miscutils.fwsplit(xsectdict[pfwdefs.SW_PARENTCHILD], ',') )
msginfo = "module %s, %s, %s" % \
(modname, xsectname, pfwdefs.SW_PARENTCHILD)
for pair in miscutils.fwsplit(xsectdict[pfwdefs.SW_PARENTCHILD], ','):
pair = pair.lower()
if ':' in pair:
(parent, child) = miscutils.fwsplit(pair, ':')
if '.' in parent:
if parent not in dataobjs[pfwdefs.SW_INPUTS]:
error(indent, "%s - parent %s not listed in %s" % \
(msginfo, parent, pfwdefs.SW_INPUTS))
cnts[ERRCNT_POS] += 1
else:
error(indent, "%s - parent %s missing section label" % \
(msginfo, parent))
cnts[ERRCNT_POS] += 1
if '.' in child:
if child not in dataobjs[pfwdefs.SW_OUTPUTS]:
error(indent, "%s - child %s not listed in %s" % \
(msginfo, child, pfwdefs.SW_OUTPUTS))
cnts[ERRCNT_POS] += 1
else:
error(indent, "%s - child %s missing section label" % \
(msginfo, child))
cnts[ERRCNT_POS] += 1
else:
error(indent, "%s - Invalid parent/child pair (%s). Missing colon." % \
(msginfo, pair))
cnts[ERRCNT_POS] += 1
elif pfwdefs.SW_INPUTS in xsectdict and pfwdefs.SW_OUTPUTS in xsectdict:
msginfo = "module %s, %s" % \
(modname, xsectname)
warning(indent, "%s - has %s and %s, but not %s" % \
(msginfo, pfwdefs.SW_INPUTS, pfwdefs.SW_OUTPUTS, pfwdefs.SW_PARENTCHILD))
cnts[WARNCNT_POS] += 1
return cnts
def check_input_files(sect, filewcl):
""" Check that the files for a single input file section exist """
sectkeys = sect.split('.')
fnames = miscutils.fwsplit(filewcl[sectkeys[1]]['fullname'], ',')
(exists1, missing1) = check_files(fnames)
return (exists1, missing1)
def main(args):
""" run the splitter """
if args.verbose:
print args
#
# Make sure that required arguments are present and appear OK.
#
if args.infile is None:
print "Input filename (-i) is required"
print "Aborting!"
return False
if not os.path.isfile(args.infile):
print "Missing input file: {:s}".format(args.infile)
print "Aborting!"
return False
if args.outfile is None:
print "Output filename (-o) is required"
print "Aborting!"
return False
if args.ccdlist == "All":
ccd_list = range(1, 63)
else:
ccd_list = fwsplit(args.ccdlist)
ccd_list = [int(x) for x in ccd_list]
status = scamputil.split_ahead_by_ccd(args.infile, args.outfile, ccd_list)
return status
def parse_provided_list(listname):
""" create dictionary of files from list in file """
#cwd = os.getcwd()
cwd = os.getenv('PWD') # don't use getcwd as it canonicallizes path
# which is not what we want for links internal to archive
uniqfiles = {}
filelist = {}
try:
with open(listname, "r") as listfh:
for line in listfh:
(fullname, filetype) = miscutils.fwsplit(line, ',')
if fullname[0] != '/':
fullname = cwd + '/' + fullname
if not os.path.exists(fullname):
miscutils.fwdie(
f"Error: could not find file on disk: {fullname}",
1)
(_, fname) = os.path.split(fullname)
if fname in uniqfiles:
miscutils.fwdie(
f"Error: Found duplicate filenames in list: {fname}",
1)
uniqfiles[fname] = True
if filetype not in filelist:
filelist[filetype] = []
filelist[filetype].append(fullname)
except IOError as err:
miscutils.fwdie(f"Error: Problems reading file '{listname}': {err}", 1)
return filelist
def check_exec_outputs(config, modname, dataobjs, xsectname, xsectdict, indent=''):
""" Check if exec output definition is valid """
# initialize
cnts = [0] * NUMCNTS
moddict = config[pfwdefs.SW_MODULESECT][modname]
if pfwdefs.SW_OUTPUTS in xsectdict:
# for each entry in inputs
print "%sChecking %s %s..." % (indent, xsectname, pfwdefs.SW_OUTPUTS)
indent += ' '
#print "%sxsectdict[pfwdefs.SW_OUTPUTS] = %s" % (indent, xsectdict[pfwdefs.SW_OUTPUTS])
for objname in miscutils.fwsplit(xsectdict[pfwdefs.SW_OUTPUTS], ','):
objname = objname.lower()
#print '%sobjname=%s' % (indent, objname)
(sect, name, _) = parse_wcl_objname(objname)
#print '%s(sect, name, subname) = (%s, %s, %s)' % (indent, sect, name, subname)
if sect is None:
error(indent+' ', "module %s, %s, %s - Invalid entry (%s). Missing section label" % (modname, xsectname, pfwdefs.SW_OUTPUTS, objname))
cnts[ERRCNT_POS] += 1
else:
# check that appears in [file/list]sect : err
if sect not in moddict or name not in moddict[sect]:
error(indent+' ', "module %s, %s, %s - Invalid entry (%s). Cannot find definition." % (modname, xsectname, pfwdefs.SW_OUTPUTS, objname))
cnts[ERRCNT_POS] += 1
else:
dataobjs[pfwdefs.SW_OUTPUTS][objname] = True
return cnts
def parse_args(argv):
"""Parse command line arguments.
"""
parser = argparse.ArgumentParser(
description='Print task information for a processing attempt')
parser.add_argument('--des_services', action='store', help='')
parser.add_argument(
'--section',
'-s',
action='store',
help='Must be specified if DES_DB_SECTION is not set in environment')
parser.add_argument('attempt_str', nargs='?', action='store')
parser.add_argument('-r', '--reqnum', action='store')
parser.add_argument('-u', '--unitname', action='store')
parser.add_argument('-a', '--attnum', action='store')
parser.add_argument('--verbose', action='store_true')
args = vars(parser.parse_args(argv)) # convert to dict
if args['attempt_str'] is None:
if args['reqnum'] is None:
print("Error: Must specify attempt_str or r,u,a")
sys.exit(1)
else:
args['attempt_str'] = '%s_r%sp%02d' % (
args['unitname'], args['reqnum'], int(args['attnum']))
args['runs'] = miscutils.fwsplit(args['attempt_str'], ',')
return args
def check_proxy(config):
"""Check for proxy.
Check if any block will submit to remote machine needing proxy,
if so check for proxy.
"""
if miscutils.fwdebug_check(3, 'PFWSUBMIT_DEBUG'):
miscutils.fwdebug_print("Beg")
config.reset_blknum()
blocklist = miscutils.fwsplit(config[pfwdefs.SW_BLOCKLIST].lower(), ',')
for blockname in blocklist:
if miscutils.fwdebug_check(3, 'PFWSUBMIT_DEBUG'):
miscutils.fwdebug_print("Checking block %s..." % (blockname))
config.set_block_info()
(exists, check_proxy) = config.search(pfwdefs.SW_CHECK_PROXY,
{intgdefs.REPLACE_VARS: True})
if exists and miscutils.convertBool(check_proxy):
timeleft = pfwcondor.get_grid_proxy_timeleft()
assert timeleft > 0
if timeleft < 21600: # 5 * 60 * 60
print("Warning: Proxy expires in less than 5 hours")
break
config.inc_blknum()
config.reset_blknum()
if miscutils.fwdebug_check(3, 'PFWSUBMIT_DEBUG'):
miscutils.fwdebug_print("End")
def min_wcl_checks(config):
""" execute minimal submit wcl checks """
max_label_length = 30 # todo: figure out how to get length from DB
msg = "ERROR\nError: Missing %s in submit wcl. Make sure submitting correct file. "
msg += "Aborting submission."
# check that reqnum and unitname exist
(exists, _) = config.search(pfwdefs.REQNUM, {intgdefs.REPLACE_VARS: True})
if not exists:
miscutils.fwdie(msg % pfwdefs.REQNUM, pfwdefs.PF_EXIT_FAILURE)
(exists, _) = config.search(pfwdefs.UNITNAME,
{intgdefs.REPLACE_VARS: True})
if not exists:
miscutils.fwdie(msg % pfwdefs.UNITNAME, pfwdefs.PF_EXIT_FAILURE)
# check that any given labels are short enough
(exists, labelstr) = config.search(pfwdefs.SW_LABEL,
{intgdefs.REPLACE_VARS: True})
if exists:
labels = miscutils.fwsplit(labelstr, ',')
for lab in labels:
if len(lab) > max_label_length:
miscutils.fwdie("ERROR\nError: label %s is longer (%s) than allowed (%s). " \
"Aborting submission." % \
(lab, len(lab), max_label_length), pfwdefs.PF_EXIT_FAILURE)
def main(argv=None):
"""Program entry point.
"""
if argv is None:
argv = sys.argv
if len(argv) != 3:
miscutils.fwdie("Usage: runqueries.pl configfile condorjobid\n", pfwdefs.PF_EXIT_FAILURE)
configfile = argv[1]
condorid = argv[2]
config = pfwconfig.PfwConfig({'wclfile': configfile})
# log condor jobid
log_pfw_event(config, config['curr_block'], 'runqueries', 'j', ['cid', condorid])
if pfwdefs.SW_MODULELIST not in config:
miscutils.fwdie("Error: No modules to run.", pfwdefs.PF_EXIT_FAILURE)
### Get master lists and files calling external codes when needed
modulelist = miscutils.fwsplit(config[pfwdefs.SW_MODULELIST].lower())
modules_prev_in_list = {}
for modname in modulelist:
if modname not in config[pfwdefs.SW_MODULESECT]:
miscutils.fwdie("Error: Could not find module description for module %s\n" %
(modname), pfwdefs.PF_EXIT_FAILURE)
runqueries(config, configfile, modname, modules_prev_in_list)
modules_prev_in_list[modname] = True
return 0
def insert_attempt_val(self, config):
"""Insert key/val pairs into the pfw_attempt_val table.
Insert key/val pairs of information about an attempt into the
pfw_attempt_val table.
"""
if miscutils.fwdebug_check(3, 'PFWDB_DEBUG'):
miscutils.fwdebug_print("Inserting into pfw_attempt_val table\n")
row = {}
row['pfw_attempt_id'] = config['pfw_attempt_id']
if pfwdefs.SW_SAVE_RUN_VALS in config:
keys2save = config.getfull(pfwdefs.SW_SAVE_RUN_VALS)
keys = miscutils.fwsplit(keys2save, ',')
for key in keys:
row['key'] = key
val = config.getfull(key)
if isinstance(val, list):
for v in val:
row['val'] = v
self.insert_PFW_row('PFW_ATTEMPT_VAL', row)
else:
row['val'] = val
self.insert_PFW_row('PFW_ATTEMPT_VAL', row)
def get_file_fullnames(sect, filewcl, fullwcl):
""" Get the full name of the files in the specified section.
Parameters
----------
sect : str
The WCL section to use
filewcl : WCL
The WCl to use
fullwcl : WCL
The full WCL, used to generate the full names
Returns
-------
set
The full file names
"""
sectkeys = sect.split('.')
sectname = sectkeys[1]
if miscutils.fwdebug_check(3, 'INTGMISC_DEBUG'):
miscutils.fwdebug_print("INFO: Beg sectname=%s" % sectname)
fnames = []
if sectname in filewcl:
filesect = filewcl[sectname]
if 'fullname' in filesect:
fnames = replfuncs.replace_vars(filesect['fullname'], fullwcl)[0]
fnames = miscutils.fwsplit(fnames, ',')
if miscutils.fwdebug_check(3, 'INTGMISC_DEBUG'):
miscutils.fwdebug_print("INFO: fullname = %s" % fnames)
return set(fnames)
def runqueries(config, configfile, modname, modules_prev_in_list):
"""Run any queries for a particular module.
"""
moddict = config[pfwdefs.SW_MODULESECT][modname]
# process each "list" in each module
if pfwdefs.SW_LISTSECT in moddict:
uber_list_dict = moddict[pfwdefs.SW_LISTSECT]
if 'list_order' in moddict:
listorder = miscutils.fwsplit(moddict['list_order'].lower())
else:
listorder = list(uber_list_dict.keys())
for listname in listorder:
list_dict = uber_list_dict[listname]
if 'depends' not in list_dict or \
list_dict['depends'] not in modules_prev_in_list:
print("\t%s-%s: creating master list\n" % \
(modname, listname))
create_master_list(config, configfile, modname,
moddict, listname, list_dict, pfwdefs.SW_LISTSECT)
# process each "file" in each module
if pfwdefs.SW_FILESECT in moddict:
for filename, file_dict in list(moddict[pfwdefs.SW_FILESECT].items()):
if 'depends' not in file_dict or \
not file_dict['depends'] not in modules_prev_in_list:
print("\t%s-%s: creating master list\n" % \
(modname, filename))
create_master_list(config, configfile, modname,
moddict, filename, file_dict, pfwdefs.SW_FILESECT)
def write_main_dag(config, maindag):
""" Writes main manager dag input file """
pfwdir = config.getfull('processingfw_dir')
print(f"maindag = '{maindag}', type={type(maindag)}")
dagfh = open(maindag, 'w')
dagfh.write(f"""
JOB begrun {pfwdir}/share/condor/runtask.condor
VARS begrun exec="$(pfwdir)/libexec/begrun.py"
VARS begrun arguments="../uberctrl/config.des"
""")
varstr = create_common_vars(config, 'begrun')
dagfh.write(f"{varstr}\n")
blocklist = miscutils.fwsplit(config[pfwdefs.SW_BLOCKLIST].lower(), ',')
for i, blockname in enumerate(blocklist):
blockdir = f"../B{i + 1:02d}-{blockname}"
cjobname = f"B{i + 1:02d}-{blockname}"
blockdag = write_block_dag(config, blockdir, blockname)
dagfh.write(f"""
JOB {cjobname} {blockdir}/{blockdag}.condor.sub
SCRIPT pre {cjobname} {pfwdir}/libexec/blockpre.py ../uberctrl/config.des
SCRIPT post {cjobname} {blockdir}/blockpost.sh $RETURN
""")
varstr = create_common_vars(config, cjobname)
dagfh.write(f"{varstr}\n")
#SCRIPT post %(cjob)s %(pdir)s/libexec/blockpost.py ../uberctrl/config.des $RETURN
with open(f"{blockdir}/blockpost.sh", 'w') as bpostfh:
bpostfh.write("#!/usr/bin/env sh\n")
bpostfh.write("sem --record-env\n")
bpostfh.write(
f"sem --fg --id blockpost -j 20 {pfwdir}/libexec/blockpost.py ../uberctrl/config.des $1\n"
)
os.chmod(f"{blockdir}/blockpost.sh", stat.S_IRWXU | stat.S_IRWXG)
dagfh.write(f"""
JOB endrun {pfwdir}/share/condor/runtask.condor
VARS endrun exec="$(pfwdir)/libexec/endrun.py"
VARS endrun arguments="../uberctrl/config.des"
""")
varstr = create_common_vars(config, 'endrun')
dagfh.write(f"{varstr}\n")
child = f"B{1:02d}-{blocklist[0]}"
dagfh.write(f"PARENT begrun CHILD {child}\n")
for i in range(1, len(blocklist)):
parent = child
child = f"B{i + 1:02d}-{blocklist[i]}"
dagfh.write(f"PARENT {parent} CHILD {child}\n")
dagfh.write(f"PARENT {child} CHILD endrun\n")
dagfh.close()
pfwcondor.add2dag(maindag, config.get_dag_cmd_opts(),
config.get_condor_attributes('uberctrl', 'mainmngr'),
None, sys.stdout)
def get_block_name(self, blknum):
""" Return block name based upon given block num """
blknum = int(blknum) # read in from file as string
blockname = ''
blockarray = miscutils.fwsplit(self[pfwdefs.SW_BLOCKLIST], ',')
if 1 <= blknum <= len(blockarray):
blockname = blockarray[blknum - 1]
return blockname
def write_main_dag(config, maindag):
"""Writes main manager dag input file.
"""
pfwdir = config.getfull('processingfw_dir')
print("maindag = '%s', type=%s" % (maindag, type(maindag)))
dagfh = open(maindag, 'w')
dagfh.write("""
JOB begrun %s/share/condor/runtask.condor
VARS begrun exec="$(pfwdir)/libexec/begrun.py"
VARS begrun arguments="../uberctrl/config.des"
""" % (pfwdir))
varstr = create_common_vars(config, 'begrun')
dagfh.write('%s\n' % varstr)
blocklist = miscutils.fwsplit(config[pfwdefs.SW_BLOCKLIST].lower(), ',')
for i in range(len(blocklist)):
blockname = blocklist[i]
blockdir = "../B%02d-%s" % (i + 1, blockname)
cjobname = "B%02d-%s" % (i + 1, blockname)
blockdag = write_block_dag(config, blockdir, blockname)
dagfh.write("""
JOB %(cjob)s %(bdir)s/%(bdag)s.condor.sub
SCRIPT pre %(cjob)s %(pdir)s/libexec/blockpre.py ../uberctrl/config.des
SCRIPT post %(cjob)s %(pdir)s/libexec/blockpost.py ../uberctrl/config.des $RETURN
""" % {
'cjob': cjobname,
'bdir': blockdir,
'bdag': blockdag,
'pdir': pfwdir
})
varstr = create_common_vars(config, cjobname)
dagfh.write('%s\n' % varstr)
dagfh.write("""
JOB endrun %s/share/condor/runtask.condor
VARS endrun exec="$(pfwdir)/libexec/endrun.py"
VARS endrun arguments="../uberctrl/config.des"
""" % (pfwdir))
varstr = create_common_vars(config, 'endrun')
dagfh.write('%s\n' % varstr)
child = "B%02d-%s" % (1, blocklist[0])
dagfh.write("PARENT begrun CHILD %s\n" % child)
for i in range(1, len(blocklist)):
parent = child
child = "B%02d-%s" % (i + 1, blocklist[i])
dagfh.write("PARENT %s CHILD %s\n" % (parent, child))
dagfh.write("PARENT %s CHILD endrun\n" % child)
dagfh.close()
pfwcondor.add2dag(maindag, config.get_dag_cmd_opts(),
config.get_condor_attributes('uberctrl', 'mainmngr'),
None, sys.stdout)
def check_target_archive(config, indent=''):
""" check info related to target archive """
cnts = [0] * NUMCNTS
print "%sChecking target archive..." % (indent)
blocklist = miscutils.fwsplit(config[pfwdefs.SW_BLOCKLIST].lower(), ',')
for blockname in blocklist:
config.set_block_info()
(found_input, use_target_archive_input) = config.search(pfwdefs.USE_TARGET_ARCHIVE_INPUT,
{pfwdefs.PF_CURRVALS: {'curr_block': blockname}})
(found_output, use_target_archive_output) = config.search(pfwdefs.USE_TARGET_ARCHIVE_OUTPUT, {pfwdefs.PF_CURRVALS: {'curr_block': blockname}})
(found_archive, _) = config.search(pfwdefs.TARGET_ARCHIVE, {pfwdefs.PF_CURRVALS: {'curr_block': blockname}})
if not found_input:
error(indent+' ', "Could not determine %s" % (pfwdefs.USE_TARGET_ARCHIVE_INPUT))
cnts[ERRCNT_POS] += 1
elif use_target_archive_input.lower() not in pfwdefs.VALID_TARGET_ARCHIVE_INPUT:
error(indent+' ', "Invalid %s value" % (pfwdefs.USE_TARGET_ARCHIVE_INPUT))
cnts[ERRCNT_POS] += 1
if not found_output:
error(indent+' ', "Could not determine %s" % (pfwdefs.USE_TARGET_ARCHIVE_OUTPUT))
cnts[ERRCNT_POS] += 1
elif use_target_archive_output.lower() not in pfwdefs.VALID_TARGET_ARCHIVE_OUTPUT:
error(indent+' ', "Invalid %s value" % (pfwdefs.USE_TARGET_ARCHIVE_OUTPUT))
cnts[ERRCNT_POS] += 1
# if need to use a target_archive for this block
if ((found_input and use_target_archive_input.lower() != 'never') or
(found_output and use_target_archive_output.lower() != 'never')):
if not found_archive:
error(indent+' ', "Missing %s value" % (pfwdefs.TARGET_ARCHIVE))
cnts[ERRCNT_POS] += 1
elif pfwdefs.SW_ARCHIVESECT not in config:
error(indent+' ', "Needs archive section which doesn't exist")
cnts[ERRCNT_POS] += 1
elif pfwdefs.TARGET_ARCHIVE not in config[pfwdefs.SW_ARCHIVESECT]:
error(indent+' ', "Invalid %s value" % (pfwdefs.TARGET_ARCHIVE))
cnts[ERRCNT_POS] += 1
else:
# check that we have all archive req values exist
pass
config.inc_blknum()
config.reset_blknum()
return cnts
def insert_attempt_label(self, config):
""" Insert label for an attempt into pfw_attempt_label table """
if miscutils.fwdebug_check(3, 'PFWDB_DEBUG'):
miscutils.fwdebug_print("Inserting into pfw_attempt_label table\n")
row = {}
row['pfw_attempt_id'] = config['pfw_attempt_id']
if pfwdefs.SW_LABEL in config:
labels = config.getfull(pfwdefs.SW_LABEL)
labels = miscutils.fwsplit(labels, ',')
for label in labels:
row['label'] = label
self.insert_PFW_row('PFW_ATTEMPT_LABEL', row)
def get_file_fullnames(sect, filewcl, fullwcl):
sectkeys = sect.split('.')
sectname = sectkeys[1]
if miscutils.fwdebug_check(3, 'INTGMISC_DEBUG'):
miscutils.fwdebug_print("INFO: Beg sectname=%s" % sectname)
fnames = []
if sectname in filewcl:
filesect = filewcl[sectname]
if 'fullname' in filesect:
fnames = replfuncs.replace_vars(filesect['fullname'], fullwcl)[0]
fnames = miscutils.fwsplit(fnames, ',')
if miscutils.fwdebug_check(3, 'INTGMISC_DEBUG'):
miscutils.fwdebug_print("INFO: fullname = %s" % fnames)
return set(fnames)
def parse_wcl_objname(objname):
#pylint: disable=unbalanced-tuple-unpacking
""" Parse WCL object name into parts """
sect = name = subname = None
parts = miscutils.fwsplit(objname, '.')
#print 'parts=', parts
if len(parts) == 3: # lists have 3 parts
(sect, name, subname) = parts
elif len(parts) == 2: # files have 2 parts
(sect, name) = parts
elif len(parts) == 1:
name = parts[0]
else:
error('', "cannot parse objname %s (too many sections/periods)" % (objname))
return sect, name, subname
def check_globals(config, indent=''):
""" Check global settings """
print "%sChecking globals..." % (indent)
# initialize counters
cnts = [0] * NUMCNTS
# always required
for key in [
'pipeline', 'pipeprod', 'pipever', 'project', 'target_site',
pfwdefs.SW_SITESECT, 'filename_pattern', 'directory_pattern',
'job_file_mvmt', pfwdefs.ATTEMPT_ARCHIVE_PATH,
pfwdefs.SW_MODULESECT, 'create_junk_tarball', 'campaign'
]:
try:
if key not in config:
error(indent + ' ',
"missing %s global key or section" % (key))
cnts[ERRCNT_POS] += 1
except:
error(indent + ' ', "missing %s global key or section" % (key))
cnts[ERRCNT_POS] += 1
print '%s Checking %s...' % (indent, pfwdefs.SW_SAVE_RUN_VALS)
if pfwdefs.SW_SAVE_RUN_VALS in config:
keys2save = config.getfull(pfwdefs.SW_SAVE_RUN_VALS)
keys = miscutils.fwsplit(keys2save, ',')
for key in keys:
exists = False
try:
(exists, _) = config.search(key, {
intgdefs.REPLACE_VARS: True,
'expand': True
})
except SystemExit:
pass
if not exists:
error(indent + ' ', 'Cannot determine %s value (%s).' % \
(pfwdefs.SW_SAVE_RUN_VALS, key))
cnts[ERRCNT_POS] += 1
return cnts
def __init__(self, dbh, config):
self.config = config
self.dbh = dbh
self.verbose = 0
keyfile = self.config['keywords_file']
self.keywords = {'pri':{}, 'ext':{}}
with open(keyfile, 'r') as fh:
for line in fh:
line = line.upper()
(keyname, pri, ext) = miscutils.fwsplit(line, ',')
if pri != 'Y' and pri != 'N' and pri != 'R':
raise ValueError('Error: Invalid primary entry in keyword file (%s)' % line)
if ext != 'Y' and ext != 'N' and ext != 'R':
raise ValueError('Error: Invalid extenstion entry in keyword file (%s)' % line)
self.keywords['pri'][keyname] = pri
self.keywords['ext'][keyname] = ext
def check_input_files(sect, filewcl):
""" Check that the files for a single input file section exist
Parameters
----------
sect : str
The WCL section to look in.
filewcl : WCL
The WCl to use
Returns
-------
tuple
Two element tuple of lists containing the existing and missing files.
"""
sectkeys = sect.split('.')
fnames = miscutils.fwsplit(filewcl[sectkeys[1]]['fullname'], ',')
(exists1, missing1) = check_files(fnames)
return (exists1, missing1)
def check_valid(self, listfullnames):
""" Check whether the given files are valid raw files """
assert isinstance(listfullnames, list)
results = {}
for fname in listfullnames:
results[fname] = False
keyfile = None
if 'raw_keywords_file' in self.config:
keyfile = self.config['raw_keywords_file']
elif 'FILEMGMT_DIR' in os.environ:
keyfile = '%s/etc/decam_src_keywords.txt' % os.environ[
'FILEMGMT_DIR']
miscutils.fwdebug_print("keyfile = %s" % keyfile)
if keyfile is not None and os.path.exists(keyfile):
keywords = {'pri': {}, 'ext': {}}
with open(keyfile, 'r') as keyfh:
for line in keyfh:
line = line.upper()
[keyname, pri, ext] = miscutils.fwsplit(line, ',')[0:3]
if pri != 'Y' and pri != 'N' and pri != 'R':
raise ValueError(
'Invalid primary entry in keyword file (%s)' %
line)
if ext != 'Y' and ext != 'N' and ext != 'R':
raise ValueError(
'Invalid extenstion entry in keyword file (%s)' %
line)
keywords['pri'][keyname] = pri
keywords['ext'][keyname] = ext
for fname in listfullnames:
results[fname] = check_single_valid(keywords, fname, 0)
else:
raise OSError('Error: Could not find keywords file')
return results
print args
#
# Make sure that required arguments are present and appear OK.
#
if (args.infile is None):
print("Input filename (-i) is required")
print("Aborting!")
exit(1)
if (not(os.path.isfile(args.infile))):
print("Missing input file: {:s}".format(args.infile))
print("Aborting!")
exit(1)
if (args.outfile is None):
print("Output filename (-o) is required")
print("Aborting!")
exit(1)
if (args.ccdlist == "All"):
ccd_list=range(1,63)
else:
ccd_list=fwsplit(args.ccdlist)
ccd_list=[int(x) for x in ccd_list]
status = scamputil.split_ahead_by_ccd(args.infile,args.outfile,ccd_list)
if (not(status)):
exit(1)
else:
exit(0)
def begblock(argv):
"""Program entry point.
"""
if argv == None:
argv = sys.argv
configfile = argv[0]
config = pfwconfig.PfwConfig({'wclfile': configfile})
config.set_block_info()
blknum = config[pfwdefs.PF_BLKNUM]
blkdir = config.getfull('block_dir')
os.chdir(blkdir)
(exists, submit_des_services) = config.search('submit_des_services')
if exists and submit_des_services is not None:
os.environ['DES_SERVICES'] = submit_des_services
(exists, submit_des_db_section) = config.search('submit_des_db_section')
if exists and submit_des_db_section is not None:
os.environ['DES_DB_SECTION'] = submit_des_db_section
dbh = None
blktid = -1
if miscutils.fwdebug_check(3, 'PFWBLOCK_DEBUG'):
miscutils.fwdebug_print("blknum = %s" % (config[pfwdefs.PF_BLKNUM]))
if miscutils.convertBool(config.getfull(pfwdefs.PF_USE_DB_OUT)):
dbh = pfwdb.PFWDB(submit_des_services, submit_des_db_section)
dbh.insert_block(config)
blktid = config['task_id']['block'][str(blknum)]
config['task_id']['begblock'] = dbh.create_task(
name='begblock',
info_table=None,
parent_task_id=blktid,
root_task_id=int(config['task_id']['attempt']),
label=None,
do_begin=True,
do_commit=True)
try:
modulelist = miscutils.fwsplit(
config.getfull(pfwdefs.SW_MODULELIST).lower())
modules_prev_in_list = {}
joblist = {}
parlist = OrderedDict()
masterdata = OrderedDict()
filelist = {'infiles': {}, 'outfiles': {}}
for num, modname in enumerate(modulelist):
print("XXXXXXXXXXXXXXXXXXXX %s XXXXXXXXXXXXXXXXXXXX" % modname)
if modname not in config[pfwdefs.SW_MODULESECT]:
miscutils.fwdie(
"Error: Could not find module description for module %s\n"
% (modname), pfwdefs.PF_EXIT_FAILURE)
moddict = config[pfwdefs.SW_MODULESECT][modname]
runqueries(config, configfile, modname, modules_prev_in_list)
pfwblock.read_master_lists(config, modname, masterdata,
modules_prev_in_list)
(infsect, outfsect) = pfwblock.get_datasect_types(config, modname)
pfwblock.fix_master_lists(config, modname, masterdata, outfsect)
if pfwdefs.PF_NOOP not in moddict or not miscutils.convertBool(
moddict[pfwdefs.PF_NOOP]):
pfwblock.create_fullnames(config, modname, masterdata)
if miscutils.fwdebug_check(
9, 'PFWBLOCK_DEBUG') and modname in masterdata:
with open('%s-masterdata.txt' % modname, 'w') as fh:
miscutils.pretty_print_dict(masterdata[modname], fh)
pfwblock.add_file_metadata(config, modname)
sublists = pfwblock.create_sublists(config, modname,
masterdata)
if sublists is not None:
if miscutils.fwdebug_check(3, 'PFWBLOCK_DEBUG'):
miscutils.fwdebug_print("sublists.keys() = %s" %
(list(sublists.keys())))
loopvals = pfwblock.get_wrapper_loopvals(config, modname)
wrapinst = pfwblock.create_wrapper_inst(
config, modname, loopvals)
wcnt = 1
for winst in list(wrapinst.values()):
if miscutils.fwdebug_check(6, 'PFWBLOCK_DEBUG'):
miscutils.fwdebug_print("winst %d - BEG" % wcnt)
pfwblock.assign_data_wrapper_inst(config, modname, winst,
masterdata, sublists,
infsect, outfsect)
pfwblock.finish_wrapper_inst(config, modname, winst,
outfsect)
tempfiles = pfwblock.create_module_wrapper_wcl(
config, modname, winst)
for fl in tempfiles['infiles']:
if fl not in list(filelist['infiles'].keys()):
filelist['infiles'][fl] = num
for fl in tempfiles['outfiles']:
filelist['outfiles'][fl] = num
#filelist['infiles'] += tempfiles['infiles']
#filelist['outfiles'] += tempfiles['outfiles']
pfwblock.divide_into_jobs(config, modname, winst, joblist,
parlist)
if miscutils.fwdebug_check(6, 'PFWBLOCK_DEBUG'):
miscutils.fwdebug_print("winst %d - %s - END" %
(wcnt, etime - stime))
wcnt += 1
modules_prev_in_list[modname] = True
if miscutils.fwdebug_check(
9, 'PFWBLOCK_DEBUG') and modname in masterdata:
with open('%s-masterdata.txt' % modname, 'w') as fh:
miscutils.pretty_print_dict(masterdata[modname], fh)
scriptfile = pfwblock.write_runjob_script(config)
intersect = list(
set(filelist['infiles'].keys()) & set(filelist['outfiles'].keys()))
finallist = []
for fl in list(filelist['infiles'].keys()):
if fl not in intersect:
finallist.append(fl)
else:
if filelist['infiles'][fl] <= filelist['outfiles'][fl]:
raise Exception(
'Input file %s requested before it is generated.' %
(fl))
if miscutils.convertBool(config.getfull(pfwdefs.PF_USE_DB_OUT)):
missingfiles = dbh.check_files(config, finallist)
if len(missingfiles) > 0:
raise Exception(
"The following input files cannot be found in the archive:"
+ ",".join(missingfiles))
miscutils.fwdebug_print("Creating job files - BEG")
for jobkey, jobdict in sorted(joblist.items()):
jobdict['jobnum'] = pfwutils.pad_jobnum(config.inc_jobnum())
jobdict['jobkeys'] = jobkey
jobdict['numexpwrap'] = len(jobdict['tasks'])
if miscutils.fwdebug_check(6, 'PFWBLOCK_DEBUG'):
miscutils.fwdebug_print("jobnum = %s, jobkey = %s:" %
(jobkey, jobdict['jobnum']))
jobdict['tasksfile'] = write_workflow_taskfile(
config, jobdict['jobnum'], jobdict['tasks'])
if (len(jobdict['inlist']) > 0 and
config.getfull(pfwdefs.USE_HOME_ARCHIVE_OUTPUT) != 'never'
and 'submit_files_mvmt' in config and
(pfwdefs.PF_DRYRUN not in config or not miscutils.convertBool(
config.getfull(pfwdefs.PF_DRYRUN)))):
# get home archive info
home_archive = config.getfull('home_archive')
archive_info = config[pfwdefs.SW_ARCHIVESECT][home_archive]
# load filemgmt class
attempt_tid = config['task_id']['attempt']
filemgmt = pfwutils.pfw_dynam_load_class(
dbh, config, attempt_tid, attempt_tid, "filemgmt",
archive_info['filemgmt'], archive_info)
# save file information
filemgmt.register_file_data('list', jobdict['inlist'],
config['pfw_attempt_id'],
attempt_tid, False, None, None)
pfwblock.copy_input_lists_home_archive(config, filemgmt,
archive_info,
jobdict['inlist'])
filemgmt.commit()
jobdict['inputwcltar'] = pfwblock.tar_inputfiles(
config, jobdict['jobnum'],
jobdict['inwcl'] + jobdict['inlist'])
if miscutils.convertBool(config.getfull(pfwdefs.PF_USE_DB_OUT)):
dbh.insert_job(config, jobdict)
pfwblock.write_jobwcl(config, jobkey, jobdict)
if ('glidein_use_wall' in config and miscutils.convertBool(
config.getfull('glidein_use_wall'))
and 'jobwalltime' in config):
jobdict['wall'] = config['jobwalltime']
miscutils.fwdebug_print("Creating job files - END")
numjobs = len(joblist)
if miscutils.convertBool(config.getfull(pfwdefs.PF_USE_DB_OUT)):
dbh.update_block_numexpjobs(config, numjobs)
#if miscutils.fwdebug_check(6, 'PFWBLOCK_DEBUG'):
# miscutils.fwdebug_print("inputfiles: %s, %s" % (type(inputfiles), inputfiles))
# miscutils.fwdebug_print("outputfiles: %s, %s" % (type(outputfiles), outputfiles))
#files2stage = set(inputfiles) - set(outputfiles)
#pfwblock.stage_inputs(config, files2stage)
#if pfwdefs.USE_HOME_ARCHIVE_OUTPUT in config and \
# config.getfull(pfwdefs.USE_HOME_ARCHIVE_OUTPUT).lower() == 'block':
# config['block_outputlist'] = 'potential_outputfiles.list'
# pfwblock.write_output_list(config, outputfiles)
dagfile = config.get_filename('jobdag')
pfwblock.create_jobmngr_dag(config, dagfile, scriptfile, joblist)
except:
retval = pfwdefs.PF_EXIT_FAILURE
with open(configfile, 'w') as cfgfh:
config.write(
cfgfh) # save config, have updated jobnum, wrapnum, etc
if miscutils.convertBool(config.getfull(pfwdefs.PF_USE_DB_OUT)):
dbh.end_task(config['task_id']['begblock'], retval, True)
dbh.end_task(blktid, retval, True)
raise
# save config, have updated jobnum, wrapnum, etc
with open(configfile, 'w') as cfgfh:
config.write(cfgfh)
(exists, dryrun) = config.search(pfwdefs.PF_DRYRUN)
if exists and miscutils.convertBool(dryrun):
retval = pfwdefs.PF_EXIT_DRYRUN
else:
retval = pfwdefs.PF_EXIT_SUCCESS
if miscutils.convertBool(config.getfull(pfwdefs.PF_USE_DB_OUT)):
dbh.end_task(config['task_id']['begblock'], retval, True)
miscutils.fwdebug_print("END - exiting with code %s" % retval)
return retval
def blockpost(argv=None):
"""Program entry point.
"""
if argv is None:
argv = sys.argv
# open file to catch error messages about command line
debugfh = open('blockpost.out', 'w')
sys.stdout = debugfh
sys.stderr = debugfh
print(' '.join(argv)) # print command line for debugging
print("running on %s" % (socket.gethostname()))
if len(argv) != 3:
print('Usage: blockpost.py configfile retval')
debugfh.close()
return pfwdefs.PF_EXIT_FAILURE
configfile = argv[1]
retval = int(argv[2])
if miscutils.fwdebug_check(3, 'PFWPOST_DEBUG'):
miscutils.fwdebug_print("configfile = %s" % configfile)
miscutils.fwdebug_print("retval = %s" % retval)
# read sysinfo file
config = pfwconfig.PfwConfig({'wclfile': configfile})
if miscutils.fwdebug_check(3, 'PFWPOST_DEBUG'):
miscutils.fwdebug_print("done reading config file")
blockname = config.getfull('blockname')
blkdir = config.getfull('block_dir')
# now that have more information, can rename output file
miscutils.fwdebug_print("getting new_log_name")
new_log_name = config.get_filename(
'block',
{pfwdefs.PF_CURRVALS: {
'flabel': 'blockpost',
'fsuffix': 'out'
}})
new_log_name = "%s/%s" % (blkdir, new_log_name)
miscutils.fwdebug_print("new_log_name = %s" % new_log_name)
debugfh.close()
os.chmod('blockpost.out', 0o666)
os.rename('blockpost.out', new_log_name)
debugfh = open(new_log_name, 'a+')
sys.stdout = debugfh
sys.stderr = debugfh
os.chdir(blkdir)
log_pfw_event(config, blockname, 'blockpost', 'j', ['posttask', retval])
dryrun = config.getfull(pfwdefs.PF_DRYRUN)
run = config.getfull('run')
attid = config['pfw_attempt_id']
reqnum = config.getfull(pfwdefs.REQNUM)
unitname = config.getfull(pfwdefs.UNITNAME)
attnum = config.getfull(pfwdefs.ATTNUM)
blknum = int(config.getfull(pfwdefs.PF_BLKNUM))
blktid = None
msg2 = ""
dbh = None
job_byblk = {}
wrap_byjob = {}
wrap_bymod = {}
wrapinfo = {}
jobinfo = {}
failedwraps = {}
whyfailwraps = {} # mod failures for other modname, shouldn't happen
usedb = miscutils.convertBool(config.getfull(pfwdefs.PF_USE_DB_OUT))
verify_files = miscutils.convertBool(config.getfull('verify_files'))
verify_status = 0
if verify_files and not usedb:
print('Skipping file verification due to lack of database connection')
if miscutils.convertBool(config.getfull(pfwdefs.PF_USE_DB_OUT)):
sem = None
try:
miscutils.fwdebug_print("Connecting to DB")
dbh = pfwdb.PFWDB(config.getfull('submit_des_services'),
config.getfull('submit_des_db_section'))
if verify_files:
curs = dbh.cursor()
curs.execute("select root from ops_archive where name='%s'" %
(config.getfull('home_archive')))
rows = curs.fetchall()
if rows is None or len(rows) != 1:
raise Exception(
"Invalid archive name (%s). Found %s rows in ops_archive"
% (config.getfull('home_archive'), len(rows)))
root = rows[0][0]
if not os.path.isdir(root):
print(
"Cannot read archive root directory:%s This program must be run on an NCSA machine with access to the archive storage system."
% (config.getfull('home_archive')))
sem = dbsem.DBSemaphore(
'verify_files_10', None,
config.getfull('submit_des_services'),
config.getfull('submit_des_db_section'))
print(
"\n\nVerifying archive file sizes on disk (0 is success)")
verify_status = cu.compare(
dbh=dbh,
archive=config.getfull('home_archive'),
pfwid=attid,
filesize=True,
md5sum=False,
quick=True,
debug=False,
script=False,
verbose=False,
silent=True)
if sem is not None:
del sem
print(" Verification of files returned status %i" %
(verify_status))
if verify_status != 0:
print(
" This indicates that one or more files do not have the correct file size (based on DB entries). Run"
)
print(
"\n compare_db.py --des_services %s --section %s --archive %s --pfwid %i --filesize --verbose"
% (config.getfull('submit_des_services'),
config.getfull('submit_des_db_section'),
config.getfull('home_archive'), int(attid)))
print("\n to see the details.")
if miscutils.convertBool(config.getfull(pfwdefs.PF_USE_QCF)):
import qcframework.qcfdb as qcfdb
qdbh = qcfdb.QCFDB(config.getfull('submit_des_services'),
config.getfull('submit_des_db_section'))
print("\n\nChecking non-job block task status from task table in DB (%s is success)" % \
pfwdefs.PF_EXIT_SUCCESS)
num_bltasks_failed = 0
bltasks = {}
blktid = None
if ('block' in config['task_id']
and str(blknum) in config['task_id']['block']):
blktid = int(config['task_id']['block'][str(blknum)])
miscutils.fwdebug_print("Getting block task info from DB")
start_time = time.time()
bltasks = dbh.get_block_task_info(blktid)
end_time = time.time()
miscutils.fwdebug_print(
"Done getting block task info from DB (%s secs)" %
(end_time - start_time))
for bltdict in list(bltasks.values()):
print("Block status = ", bltdict['status'])
if bltdict['status'] == pfwdefs.PF_EXIT_DRYRUN:
print("setting return value to dryrun")
retval = bltdict['status']
elif bltdict['status'] != pfwdefs.PF_EXIT_SUCCESS:
num_bltasks_failed += 1
msg2 += "\t%s" % (bltdict['name'])
if bltdict['label'] is not None:
msg2 += " - %s" % (bltdict['label'])
msg2 += " failed\n"
if bltdict['name'] == 'begblock':
# try to read the begblock.out and begblock.err files
print(
"Trying to get begblock.out and begblock.err")
msg2 += get_subblock_output("begblock")
# try to get QCF messages (especially from query codes)
begblock_tid = int(config['task_id']['begblock'])
sql = "select id from task where parent_task_id=%i and status!=0" % (
begblock_tid)
curs = dbh.cursor()
curs.execute(sql)
res = curs.fetchall()
msg2 += "\n===== QCF Messages =====\n"
msg2 += "\n begblock\n"
wrapids = [blktid, begblock_tid]
for r in res:
wrapids.append(r[0])
wrapmsg = {}
if qdbh is not None:
miscutils.fwdebug_print(
"Querying QCF messages")
start_time = time.time()
wrapmsg = qdbh.get_qcf_messages_for_wrappers(
wrapids)
end_time = time.time()
miscutils.fwdebug_print(
"Done querying QCF messages (%s secs)" %
(end_time - start_time))
miscutils.fwdebug_print("wrapmsg = %s" %
wrapmsg)
if len(wrapmsg) == 0:
msg2 += " No QCF messages\n"
else:
for msgs in list(wrapmsg.values()):
for m in msgs:
msg2 += " " + m['message'] + "\n"
retval = pfwdefs.PF_EXIT_FAILURE
if retval != pfwdefs.PF_EXIT_DRYRUN:
print("\n\nChecking job status from pfw_job table in DB (%s is success)" % \
pfwdefs.PF_EXIT_SUCCESS)
miscutils.fwdebug_print("Getting job info from DB")
start_time = time.time()
jobinfo = dbh.get_job_info({'pfw_block_task_id': blktid})
end_time = time.time()
miscutils.fwdebug_print(
"Done getting job info from DB (%s secs)" %
(end_time - start_time))
miscutils.fwdebug_print("Getting wrapper info from DB")
start_time = time.time()
wrapinfo = dbh.get_wrapper_info(pfw_attempt_id=attid,
pfw_block_task_id=blktid)
end_time = time.time()
miscutils.fwdebug_print(
"Done getting wrapper info from DB (%s secs)" %
(end_time - start_time))
else:
msg = "Could not find task id for block %s in config.des" % blockname
print("Error:", msg)
if 'attempt' in config['task_id']:
miscutils.fwdebug_print("Saving pfw message")
start_time = time.time()
Messaging.pfw_message(dbh, attid,
config['task_id']['attempt'], msg,
pfw_utils.PFW_DB_INFO,
'blockpost.out', 0)
end_time = time.time()
miscutils.fwdebug_print(
"Done saving pfw message (%s secs)" %
(end_time - start_time))
print("all the task ids:", config['task_id'])
archive = None
if pfwdefs.HOME_ARCHIVE in config:
archive = config.getfull(pfwdefs.HOME_ARCHIVE)
logfullnames = dbh.get_fail_log_fullnames(attid, archive)
dbh.close()
print("len(jobinfo) = ", len(jobinfo))
print("len(wrapinfo) = ", len(wrapinfo))
job_byblk = pfwutils.index_job_info(jobinfo)
print("blktid: ", blktid)
print("job_byblk:", job_byblk)
if blktid not in job_byblk:
print("Warn: could not find jobs for block %s" % blknum)
print(" This is ok if attempt died before jobs ran")
print(" block task_ids in job_byblk:" %
list(job_byblk.keys()))
else:
wrap_byjob, wrap_bymod = pfwutils.index_wrapper_info(wrapinfo)
#print "wrap_byjob:", wrap_byjob
#print "wrap_bymod:", wrap_bymod
for jobtid, jobdict in sorted(job_byblk[blktid].items()):
failedwraps[jobtid] = []
whyfailwraps[jobtid] = []
jobkeys = ""
# don't print out successful wrappers
if jobtid in wrap_byjob and jobdict[
'status'] == pfwdefs.PF_EXIT_SUCCESS:
continue
if jobdict['jobkeys'] is not None:
jobkeys = jobdict['jobkeys']
#print "jobkeys = ", jobkeys, type(jobkeys)
submit_job_path = "%s/B%02d-%s/%04d" % (
config.getfull('work_dir'),
int(config.getfull('blknum')),
config.getfull('blockname'), int(jobdict['jobnum']))
msg2 += "\n\t%s (%s) " % (pfwutils.pad_jobnum(
jobdict['jobnum']), jobkeys)
if jobtid not in wrap_byjob:
msg2 += "\tNo wrapper instances"
else:
#print "wrapnum in job =", wrap_byjob[jobtid].keys()
maxwrap = max(wrap_byjob[jobtid].keys())
#print "maxwrap =", maxwrap
modname = wrap_byjob[jobtid][maxwrap]['modname']
#print "modname =", modname
msg2 += "%d/%s %s" % (len(
wrap_byjob[jobtid]), jobdict['expect_num_wrap'],
modname)
# determine wrappers for this job without success exit
for wrapnum, wdict in list(wrap_byjob[jobtid].items()):
if wdict['status'] is None or wdict[
'status'] != pfwdefs.PF_EXIT_SUCCESS:
if wdict['modname'] == modname:
failedwraps[jobtid].append(wrapnum)
else:
whyfailwraps[jobtid].append(wrapnum)
if jobdict['status'] == pfwdefs.PF_EXIT_EUPS_FAILURE:
msg2 += " - FAIL - EUPS setup failure"
retval = jobdict['status']
elif jobdict['status'] == pfwdefs.PF_EXIT_CONDOR:
msg2 += " - FAIL - Condor/Globus failure"
retval = jobdict['status']
elif jobdict['status'] is None:
msg2 += " - FAIL - NULL status"
retval = pfwdefs.PF_EXIT_FAILURE
elif jobdict['status'] != pfwdefs.PF_EXIT_SUCCESS:
msg2 += " - FAIL - Non-zero status"
retval = jobdict['status']
if jobdict['status'] != pfwdefs.PF_EXIT_SUCCESS:
msg2 += "\n\t\t%s/runjob.out " % (submit_job_path)
msg2 += '\n'
# print pfw_messages
if 'message' in jobdict:
print(jobdict['message'])
for msgdict in sorted(jobdict['message'],
key=lambda k: k['message_time']):
level = int(msgdict['message_lvl'])
levelstr = 'info'
if level == pfwdefs.PFWDB_MSG_WARN:
levelstr = 'WARN'
elif level == pfwdefs.PFWDB_MSG_ERROR:
levelstr = 'ERROR'
msg2 += "\t\t%s - %s\n" % (
levelstr, msgdict['message'].replace(
'\n', '\n\t\t\t'))
if jobtid in wrap_byjob:
# print log file name for failed/unfinished wrappers
for wrapnum in failedwraps[jobtid]:
wrapdict = wrap_byjob[jobtid][wrapnum]
if wrapdict['log'] in logfullnames:
msg2 += "\t\t%s - %s\n" % (
wrapnum, logfullnames[wrapdict['log']])
else:
msg2 += "\t\t%s - Could not find log in archive (%s)\n" % (
wrapnum, wrapdict['log'])
wrapmsg = get_qcf_messages(qdbh, config,
[wrapdict['task_id']])
msg2 = print_qcf_messages(config, wrapdict,
wrapmsg, msg2)
msg2 += '\n'
# If weirdness happened in run, print a message
if len(whyfailwraps[jobtid]) > 0:
msg2 += "\n*** Contact framework developers. Wrappers ran after at least 1 wrapper from a previous module that doesn't have success status.\n"
msg2 += "\t%s\n" % ','.join(whyfailwraps[jobtid])
except Exception as exc:
if sem is not None:
del sem
msg2 += "\n\nEncountered error trying to gather status information for email."
msg2 += "\nCheck output for blockpost for further details."
print(
"\n\nEncountered error trying to gather status information for email"
)
print("%s: %s" % (exc.__class__.__name__, str(exc)))
(extype, exvalue, trback) = sys.exc_info()
traceback.print_exception(extype, exvalue, trback, file=sys.stdout)
retval = pfwdefs.PF_EXIT_FAILURE
retval = int(retval) + verify_status
print("before email retval =", retval)
when_to_email = 'run'
if 'when_to_email' in config:
when_to_email = config.getfull('when_to_email').lower()
if miscutils.convertBool(dryrun):
if when_to_email != 'never':
print("dryrun = ", dryrun)
print("Sending dryrun email")
if retval == pfwdefs.PF_EXIT_DRYRUN:
msg1 = "%s: In dryrun mode, block %s has finished successfully." % (
run, blockname)
else:
msg1 = "%s: In dryrun mode, block %s has failed." % (
run, blockname)
send_email(config, blockname, retval, "", msg1, msg2)
else:
print("Not sending dryrun email")
print("retval = ", retval)
retval = pfwdefs.PF_EXIT_DRYRUN
elif retval:
if when_to_email != 'never':
print("Sending block failed email\n")
msg1 = "%s: block %s has failed." % (run, blockname)
send_email(config, blockname, retval, "", msg1, msg2)
else:
print("Not sending failed email")
print("retval = ", retval)
elif retval == pfwdefs.PF_EXIT_SUCCESS:
if when_to_email == 'block':
msg1 = "%s: block %s has finished successfully." % (run,
blockname)
msg2 = ""
print("Sending success email\n")
send_email(config, blockname, retval, "", msg1, msg2)
elif when_to_email == 'run':
numblocks = len(
miscutils.fwsplit(config[pfwdefs.SW_BLOCKLIST], ','))
if int(config[pfwdefs.PF_BLKNUM]) == numblocks:
msg1 = "%s: run has finished successfully." % (run)
msg2 = ""
print("Sending success email\n")
send_email(config, blockname, retval, "", msg1, msg2)
else:
print("Not sending run email because not last block")
print("retval = ", retval)
else:
print("Not sending success email")
print("retval = ", retval)
else:
print("Not sending email")
print("retval = ", retval)
# Store values in DB and hist file
dbh = None
if miscutils.convertBool(config[pfwdefs.PF_USE_DB_OUT]):
dbh = pfwdb.PFWDB(config.getfull('submit_des_services'),
config.getfull('submit_des_db_section'))
if blktid is not None:
print("Updating end of block task", blktid)
dbh.end_task(blktid, retval, True)
else:
print("Could not update end of block task without block task id")
if retval != pfwdefs.PF_EXIT_SUCCESS:
print("Updating end of attempt", config['task_id']['attempt'])
dbh.end_task(config['task_id']['attempt'], retval, True)
dbh.commit()
dbh.close()
print("before next block retval = ", retval)
if retval == pfwdefs.PF_EXIT_SUCCESS:
# Get ready for next block
config.inc_blknum()
with open(configfile, 'w') as cfgfh:
config.write(cfgfh)
print("new blknum = ", config[pfwdefs.PF_BLKNUM])
print("number of blocks = ",
len(miscutils.fwsplit(config[pfwdefs.SW_BLOCKLIST], ',')))
miscutils.fwdebug_print("Returning retval = %s (%s)" %
(retval, type(retval)))
miscutils.fwdebug_print("END")
debugfh.close()
return int(retval)
