def _get_update_values_explicit(self, update_info): """ include values explicitly set by operator/framework """ upinfo2 = collections.OrderedDict() # for each set of header updates for updset in update_info.values(): headers = ['0'] # default to primary header if 'headers' in updset: headers = miscutils.fwsplit(update_info[updset], ',') hdu_updset = collections.OrderedDict() for key, val in updset.items(): if key != 'headers': uval = ucomment = udatatype = None header_info = miscutils.fwsplit(val, '/') uval = header_info[0] if len(header_info) == 3: ucomment = header_info[1] udatatype = header_info[2] hdu_updset[key] = (uval, ucomment, udatatype) for hdname in headers: if hdname not in update_info: upinfo2[hdname] = collections.OrderedDict() upinfo2[hdname].update(hdu_updset) return upinfo2
def read_fullnames_from_listfile(listfile, linefmt, colstr): """ Read a list file returning fullnames from the list """ if miscutils.fwdebug_check(3, 'INTGMISC_DEBUG'): miscutils.fwdebug_print('colstr=%s' % colstr) columns = convert_col_string_to_list(colstr, False) if miscutils.fwdebug_check(3, 'INTGMISC_DEBUG'): miscutils.fwdebug_print('columns=%s' % columns) fullnames = {} pos2fsect = {} for pos in range(0, len(columns)): lcol = columns[pos].lower() if lcol.endswith('.fullname'): filesect = lcol[:-9] pos2fsect[pos] = filesect fullnames[filesect] = [] # else a data column instead of a filename if miscutils.fwdebug_check(3, 'INTGMISC_DEBUG'): miscutils.fwdebug_print('pos2fsect=%s' % pos2fsect) if linefmt == 'config' or linefmt == 'wcl': miscutils.fwdie( 'Error: wcl list format not currently supported (%s)' % listfile, 1) else: with open(listfile, 'r') as listfh: for line in listfh: line = line.strip() # convert line into python list lineinfo = [] if linefmt == 'textcsv': lineinfo = miscutils.fwsplit(line, ',') elif linefmt == 'texttab': lineinfo = miscutils.fwsplit(line, '\t') elif linefmt == 'textsp': lineinfo = miscutils.fwsplit(line, ' ') else: miscutils.fwdie('Error: unknown linefmt (%s)' % linefmt, 1) # save each fullname in line for pos in pos2fsect: # use common routine to parse actual fullname (e.g., remove [0]) parsemask = miscutils.CU_PARSE_PATH | miscutils.CU_PARSE_FILENAME | \ miscutils.CU_PARSE_COMPRESSION (path, filename, compression) = miscutils.parse_fullname( lineinfo[pos], parsemask) fname = "%s/%s" % (path, filename) if compression is not None: fname += compression fullnames[pos2fsect[pos]].append(fname) if miscutils.fwdebug_check(6, 'INTGMISC_DEBUG'): miscutils.fwdebug_print('fullnames = %s' % fullnames) return fullnames
def process_csv(fncsv, delim, version, fnskymap, ocurs): datacnt = 0 with open(fncsv) as infh: # first line has headers line = infh.readline() headers = miscutils.fwsplit(line, delim) print headers # read data line = infh.readline() while line: line = line.split('#')[0] line = line.strip() data = miscutils.fwsplit(line, delim) svals = dict(zip(headers, data)) newpatch = re.sub(oldpatchsep, newpatchsep, svals['patch']) ovals = { 'tract': svals['tract'], 'patch': newpatch, 'visit': svals['visit'], 'ccd': svals['ccd'], 'version': version, 'skymap': fnskymap } datacnt += 1 insert_row_oracle(ocurs, ovals) line = infh.readline() print datacnt, "lines from csv"
def check_block(config, indent=''): """ check blocks level defs """ cnts = [0] * NUMCNTS blocklist = miscutils.fwsplit(config[pfwdefs.SW_BLOCKLIST].lower(), ',') for blockname in blocklist: print "%sChecking block %s..." % (indent, blockname) config.set_block_info() # check modules block = config[pfwdefs.SW_BLOCKSECT][blockname] if pfwdefs.SW_MODULELIST in block: modulelist = miscutils.fwsplit(block[pfwdefs.SW_MODULELIST].lower(), ',') for modname in modulelist: if modname not in config[pfwdefs.SW_MODULESECT]: error(indent+' ', "block %s - invalid %s" % (blockname, pfwdefs.SW_MODULELIST)) print "%s (bad module name: %s, list: %s)" % (indent, modname, modulelist) cnts[ERRCNT_POS] += 1 else: cnts2 = check_module(config, blockname, modname, indent+' ') cnts = [x + y for x, y in zip(cnts, cnts2)] # increment counts else: error(indent+' ', "block %s - missing %s value" % (blockname, pfwdefs.SW_MODULESECT)) cnts[ERRCNT_POS] += 1 config.inc_blknum() config.reset_blknum() return cnts
def check_block(config, indent=''): """Check blocks level defs. """ cnts = [0] * NUMCNTS blocklist = miscutils.fwsplit(config[pfwdefs.SW_BLOCKLIST].lower(), ',') for blockname in blocklist: print("%sChecking block %s..." % (indent, blockname)) config.set_block_info() for key in [pfwdefs.PF_USE_DB_IN, pfwdefs.PF_USE_DB_OUT]: if key in config and miscutils.convertBool(config.getfull(key)): (found, val) = config.search('target_des_db_section') if not found: error( indent + ' ', "using DB (%s), but missing target_des_db_section" % (key)) cnts[ERRCNT_POS] += 1 (found, val) = config.search('target_des_services') if not found: error( indent + ' ', "using DB (%s), but missing target_des_services" % (key)) cnts[ERRCNT_POS] += 1 # check modules block = config[pfwdefs.SW_BLOCKSECT][blockname] if pfwdefs.SW_MODULELIST in block: modulelist = miscutils.fwsplit( block[pfwdefs.SW_MODULELIST].lower(), ',') for modname in modulelist: if modname not in config[pfwdefs.SW_MODULESECT]: error( indent + ' ', "block %s - invalid %s" % (blockname, pfwdefs.SW_MODULELIST)) print("%s (bad module name: %s, list: %s)" % (indent, modname, modulelist)) cnts[ERRCNT_POS] += 1 else: cnts2 = check_module(config, blockname, modname, indent + ' ') cnts = [x + y for x, y in zip(cnts, cnts2)] # increment counts else: error( indent + ' ', "block %s - missing %s value" % (blockname, pfwdefs.SW_MODULESECT)) cnts[ERRCNT_POS] += 1 config.inc_blknum() config.reset_blknum() return cnts
def check_exec_parentchild(modname, dataobjs, xsectname, xsectdict, indent=''): #pylint: disable=unbalanced-tuple-unpacking """ Check that parent and children appear in inputs and outputs """ # assumes check_exec_input and check_exec_output have already been executed so there are entries in dataobjs cnts = [0] * NUMCNTS if pfwdefs.SW_PARENTCHILD in xsectdict: print "%sChecking %s %s..." % (indent, xsectname, pfwdefs.SW_PARENTCHILD) indent += ' ' #print "%sxsectdict[pfwdefs.SW_PARENTCHILD] = %s" % (indent, xsectdict[pfwdefs.SW_PARENTCHILD]) #print "%sdataobjs[pfwdefs.SW_INPUTS] = %s" % (indent, dataobjs[pfwdefs.SW_INPUTS]) #print "%sdataobjs[pfwdefs.SW_OUTPUTS] = %s" % (indent, dataobjs[pfwdefs.SW_OUTPUTS]) #print "%sfsplit = %s" % (indent, miscutils.fwsplit(xsectdict[pfwdefs.SW_PARENTCHILD], ',') ) msginfo = "module %s, %s, %s" % \ (modname, xsectname, pfwdefs.SW_PARENTCHILD) for pair in miscutils.fwsplit(xsectdict[pfwdefs.SW_PARENTCHILD], ','): pair = pair.lower() if ':' in pair: (parent, child) = miscutils.fwsplit(pair, ':') if '.' in parent: if parent not in dataobjs[pfwdefs.SW_INPUTS]: error(indent, "%s - parent %s not listed in %s" % \ (msginfo, parent, pfwdefs.SW_INPUTS)) cnts[ERRCNT_POS] += 1 else: error(indent, "%s - parent %s missing section label" % \ (msginfo, parent)) cnts[ERRCNT_POS] += 1 if '.' in child: if child not in dataobjs[pfwdefs.SW_OUTPUTS]: error(indent, "%s - child %s not listed in %s" % \ (msginfo, child, pfwdefs.SW_OUTPUTS)) cnts[ERRCNT_POS] += 1 else: error(indent, "%s - child %s missing section label" % \ (msginfo, child)) cnts[ERRCNT_POS] += 1 else: error(indent, "%s - Invalid parent/child pair (%s). Missing colon." % \ (msginfo, pair)) cnts[ERRCNT_POS] += 1 elif pfwdefs.SW_INPUTS in xsectdict and pfwdefs.SW_OUTPUTS in xsectdict: msginfo = "module %s, %s" % \ (modname, xsectname) warning(indent, "%s - has %s and %s, but not %s" % \ (msginfo, pfwdefs.SW_INPUTS, pfwdefs.SW_OUTPUTS, pfwdefs.SW_PARENTCHILD)) cnts[WARNCNT_POS] += 1 return cnts
def check_input_files(sect, filewcl): """ Check that the files for a single input file section exist """ sectkeys = sect.split('.') fnames = miscutils.fwsplit(filewcl[sectkeys[1]]['fullname'], ',') (exists1, missing1) = check_files(fnames) return (exists1, missing1)
def main(args): """ run the splitter """ if args.verbose: print args # # Make sure that required arguments are present and appear OK. # if args.infile is None: print "Input filename (-i) is required" print "Aborting!" return False if not os.path.isfile(args.infile): print "Missing input file: {:s}".format(args.infile) print "Aborting!" return False if args.outfile is None: print "Output filename (-o) is required" print "Aborting!" return False if args.ccdlist == "All": ccd_list = range(1, 63) else: ccd_list = fwsplit(args.ccdlist) ccd_list = [int(x) for x in ccd_list] status = scamputil.split_ahead_by_ccd(args.infile, args.outfile, ccd_list) return status
def parse_provided_list(listname): """ create dictionary of files from list in file """ #cwd = os.getcwd() cwd = os.getenv('PWD') # don't use getcwd as it canonicallizes path # which is not what we want for links internal to archive uniqfiles = {} filelist = {} try: with open(listname, "r") as listfh: for line in listfh: (fullname, filetype) = miscutils.fwsplit(line, ',') if fullname[0] != '/': fullname = cwd + '/' + fullname if not os.path.exists(fullname): miscutils.fwdie( f"Error: could not find file on disk: {fullname}", 1) (_, fname) = os.path.split(fullname) if fname in uniqfiles: miscutils.fwdie( f"Error: Found duplicate filenames in list: {fname}", 1) uniqfiles[fname] = True if filetype not in filelist: filelist[filetype] = [] filelist[filetype].append(fullname) except IOError as err: miscutils.fwdie(f"Error: Problems reading file '{listname}': {err}", 1) return filelist
def check_exec_outputs(config, modname, dataobjs, xsectname, xsectdict, indent=''): """ Check if exec output definition is valid """ # initialize cnts = [0] * NUMCNTS moddict = config[pfwdefs.SW_MODULESECT][modname] if pfwdefs.SW_OUTPUTS in xsectdict: # for each entry in inputs print "%sChecking %s %s..." % (indent, xsectname, pfwdefs.SW_OUTPUTS) indent += ' ' #print "%sxsectdict[pfwdefs.SW_OUTPUTS] = %s" % (indent, xsectdict[pfwdefs.SW_OUTPUTS]) for objname in miscutils.fwsplit(xsectdict[pfwdefs.SW_OUTPUTS], ','): objname = objname.lower() #print '%sobjname=%s' % (indent, objname) (sect, name, _) = parse_wcl_objname(objname) #print '%s(sect, name, subname) = (%s, %s, %s)' % (indent, sect, name, subname) if sect is None: error(indent+' ', "module %s, %s, %s - Invalid entry (%s). Missing section label" % (modname, xsectname, pfwdefs.SW_OUTPUTS, objname)) cnts[ERRCNT_POS] += 1 else: # check that appears in [file/list]sect : err if sect not in moddict or name not in moddict[sect]: error(indent+' ', "module %s, %s, %s - Invalid entry (%s). Cannot find definition." % (modname, xsectname, pfwdefs.SW_OUTPUTS, objname)) cnts[ERRCNT_POS] += 1 else: dataobjs[pfwdefs.SW_OUTPUTS][objname] = True return cnts
def parse_args(argv): """Parse command line arguments. """ parser = argparse.ArgumentParser( description='Print task information for a processing attempt') parser.add_argument('--des_services', action='store', help='') parser.add_argument( '--section', '-s', action='store', help='Must be specified if DES_DB_SECTION is not set in environment') parser.add_argument('attempt_str', nargs='?', action='store') parser.add_argument('-r', '--reqnum', action='store') parser.add_argument('-u', '--unitname', action='store') parser.add_argument('-a', '--attnum', action='store') parser.add_argument('--verbose', action='store_true') args = vars(parser.parse_args(argv)) # convert to dict if args['attempt_str'] is None: if args['reqnum'] is None: print("Error: Must specify attempt_str or r,u,a") sys.exit(1) else: args['attempt_str'] = '%s_r%sp%02d' % ( args['unitname'], args['reqnum'], int(args['attnum'])) args['runs'] = miscutils.fwsplit(args['attempt_str'], ',') return args
def check_proxy(config): """Check for proxy. Check if any block will submit to remote machine needing proxy, if so check for proxy. """ if miscutils.fwdebug_check(3, 'PFWSUBMIT_DEBUG'): miscutils.fwdebug_print("Beg") config.reset_blknum() blocklist = miscutils.fwsplit(config[pfwdefs.SW_BLOCKLIST].lower(), ',') for blockname in blocklist: if miscutils.fwdebug_check(3, 'PFWSUBMIT_DEBUG'): miscutils.fwdebug_print("Checking block %s..." % (blockname)) config.set_block_info() (exists, check_proxy) = config.search(pfwdefs.SW_CHECK_PROXY, {intgdefs.REPLACE_VARS: True}) if exists and miscutils.convertBool(check_proxy): timeleft = pfwcondor.get_grid_proxy_timeleft() assert timeleft > 0 if timeleft < 21600: # 5 * 60 * 60 print("Warning: Proxy expires in less than 5 hours") break config.inc_blknum() config.reset_blknum() if miscutils.fwdebug_check(3, 'PFWSUBMIT_DEBUG'): miscutils.fwdebug_print("End")
def min_wcl_checks(config): """ execute minimal submit wcl checks """ max_label_length = 30 # todo: figure out how to get length from DB msg = "ERROR\nError: Missing %s in submit wcl. Make sure submitting correct file. " msg += "Aborting submission." # check that reqnum and unitname exist (exists, _) = config.search(pfwdefs.REQNUM, {intgdefs.REPLACE_VARS: True}) if not exists: miscutils.fwdie(msg % pfwdefs.REQNUM, pfwdefs.PF_EXIT_FAILURE) (exists, _) = config.search(pfwdefs.UNITNAME, {intgdefs.REPLACE_VARS: True}) if not exists: miscutils.fwdie(msg % pfwdefs.UNITNAME, pfwdefs.PF_EXIT_FAILURE) # check that any given labels are short enough (exists, labelstr) = config.search(pfwdefs.SW_LABEL, {intgdefs.REPLACE_VARS: True}) if exists: labels = miscutils.fwsplit(labelstr, ',') for lab in labels: if len(lab) > max_label_length: miscutils.fwdie("ERROR\nError: label %s is longer (%s) than allowed (%s). " \ "Aborting submission." % \ (lab, len(lab), max_label_length), pfwdefs.PF_EXIT_FAILURE)
def main(argv=None): """Program entry point. """ if argv is None: argv = sys.argv if len(argv) != 3: miscutils.fwdie("Usage: runqueries.pl configfile condorjobid\n", pfwdefs.PF_EXIT_FAILURE) configfile = argv[1] condorid = argv[2] config = pfwconfig.PfwConfig({'wclfile': configfile}) # log condor jobid log_pfw_event(config, config['curr_block'], 'runqueries', 'j', ['cid', condorid]) if pfwdefs.SW_MODULELIST not in config: miscutils.fwdie("Error: No modules to run.", pfwdefs.PF_EXIT_FAILURE) ### Get master lists and files calling external codes when needed modulelist = miscutils.fwsplit(config[pfwdefs.SW_MODULELIST].lower()) modules_prev_in_list = {} for modname in modulelist: if modname not in config[pfwdefs.SW_MODULESECT]: miscutils.fwdie("Error: Could not find module description for module %s\n" % (modname), pfwdefs.PF_EXIT_FAILURE) runqueries(config, configfile, modname, modules_prev_in_list) modules_prev_in_list[modname] = True return 0
def insert_attempt_val(self, config): """Insert key/val pairs into the pfw_attempt_val table. Insert key/val pairs of information about an attempt into the pfw_attempt_val table. """ if miscutils.fwdebug_check(3, 'PFWDB_DEBUG'): miscutils.fwdebug_print("Inserting into pfw_attempt_val table\n") row = {} row['pfw_attempt_id'] = config['pfw_attempt_id'] if pfwdefs.SW_SAVE_RUN_VALS in config: keys2save = config.getfull(pfwdefs.SW_SAVE_RUN_VALS) keys = miscutils.fwsplit(keys2save, ',') for key in keys: row['key'] = key val = config.getfull(key) if isinstance(val, list): for v in val: row['val'] = v self.insert_PFW_row('PFW_ATTEMPT_VAL', row) else: row['val'] = val self.insert_PFW_row('PFW_ATTEMPT_VAL', row)
def get_file_fullnames(sect, filewcl, fullwcl): """ Get the full name of the files in the specified section. Parameters ---------- sect : str The WCL section to use filewcl : WCL The WCl to use fullwcl : WCL The full WCL, used to generate the full names Returns ------- set The full file names """ sectkeys = sect.split('.') sectname = sectkeys[1] if miscutils.fwdebug_check(3, 'INTGMISC_DEBUG'): miscutils.fwdebug_print("INFO: Beg sectname=%s" % sectname) fnames = [] if sectname in filewcl: filesect = filewcl[sectname] if 'fullname' in filesect: fnames = replfuncs.replace_vars(filesect['fullname'], fullwcl)[0] fnames = miscutils.fwsplit(fnames, ',') if miscutils.fwdebug_check(3, 'INTGMISC_DEBUG'): miscutils.fwdebug_print("INFO: fullname = %s" % fnames) return set(fnames)
def runqueries(config, configfile, modname, modules_prev_in_list): """Run any queries for a particular module. """ moddict = config[pfwdefs.SW_MODULESECT][modname] # process each "list" in each module if pfwdefs.SW_LISTSECT in moddict: uber_list_dict = moddict[pfwdefs.SW_LISTSECT] if 'list_order' in moddict: listorder = miscutils.fwsplit(moddict['list_order'].lower()) else: listorder = list(uber_list_dict.keys()) for listname in listorder: list_dict = uber_list_dict[listname] if 'depends' not in list_dict or \ list_dict['depends'] not in modules_prev_in_list: print("\t%s-%s: creating master list\n" % \ (modname, listname)) create_master_list(config, configfile, modname, moddict, listname, list_dict, pfwdefs.SW_LISTSECT) # process each "file" in each module if pfwdefs.SW_FILESECT in moddict: for filename, file_dict in list(moddict[pfwdefs.SW_FILESECT].items()): if 'depends' not in file_dict or \ not file_dict['depends'] not in modules_prev_in_list: print("\t%s-%s: creating master list\n" % \ (modname, filename)) create_master_list(config, configfile, modname, moddict, filename, file_dict, pfwdefs.SW_FILESECT)
def write_main_dag(config, maindag): """ Writes main manager dag input file """ pfwdir = config.getfull('processingfw_dir') print(f"maindag = '{maindag}', type={type(maindag)}") dagfh = open(maindag, 'w') dagfh.write(f""" JOB begrun {pfwdir}/share/condor/runtask.condor VARS begrun exec="$(pfwdir)/libexec/begrun.py" VARS begrun arguments="../uberctrl/config.des" """) varstr = create_common_vars(config, 'begrun') dagfh.write(f"{varstr}\n") blocklist = miscutils.fwsplit(config[pfwdefs.SW_BLOCKLIST].lower(), ',') for i, blockname in enumerate(blocklist): blockdir = f"../B{i + 1:02d}-{blockname}" cjobname = f"B{i + 1:02d}-{blockname}" blockdag = write_block_dag(config, blockdir, blockname) dagfh.write(f""" JOB {cjobname} {blockdir}/{blockdag}.condor.sub SCRIPT pre {cjobname} {pfwdir}/libexec/blockpre.py ../uberctrl/config.des SCRIPT post {cjobname} {blockdir}/blockpost.sh $RETURN """) varstr = create_common_vars(config, cjobname) dagfh.write(f"{varstr}\n") #SCRIPT post %(cjob)s %(pdir)s/libexec/blockpost.py ../uberctrl/config.des $RETURN with open(f"{blockdir}/blockpost.sh", 'w') as bpostfh: bpostfh.write("#!/usr/bin/env sh\n") bpostfh.write("sem --record-env\n") bpostfh.write( f"sem --fg --id blockpost -j 20 {pfwdir}/libexec/blockpost.py ../uberctrl/config.des $1\n" ) os.chmod(f"{blockdir}/blockpost.sh", stat.S_IRWXU | stat.S_IRWXG) dagfh.write(f""" JOB endrun {pfwdir}/share/condor/runtask.condor VARS endrun exec="$(pfwdir)/libexec/endrun.py" VARS endrun arguments="../uberctrl/config.des" """) varstr = create_common_vars(config, 'endrun') dagfh.write(f"{varstr}\n") child = f"B{1:02d}-{blocklist[0]}" dagfh.write(f"PARENT begrun CHILD {child}\n") for i in range(1, len(blocklist)): parent = child child = f"B{i + 1:02d}-{blocklist[i]}" dagfh.write(f"PARENT {parent} CHILD {child}\n") dagfh.write(f"PARENT {child} CHILD endrun\n") dagfh.close() pfwcondor.add2dag(maindag, config.get_dag_cmd_opts(), config.get_condor_attributes('uberctrl', 'mainmngr'), None, sys.stdout)
def get_block_name(self, blknum): """ Return block name based upon given block num """ blknum = int(blknum) # read in from file as string blockname = '' blockarray = miscutils.fwsplit(self[pfwdefs.SW_BLOCKLIST], ',') if 1 <= blknum <= len(blockarray): blockname = blockarray[blknum - 1] return blockname
def write_main_dag(config, maindag): """Writes main manager dag input file. """ pfwdir = config.getfull('processingfw_dir') print("maindag = '%s', type=%s" % (maindag, type(maindag))) dagfh = open(maindag, 'w') dagfh.write(""" JOB begrun %s/share/condor/runtask.condor VARS begrun exec="$(pfwdir)/libexec/begrun.py" VARS begrun arguments="../uberctrl/config.des" """ % (pfwdir)) varstr = create_common_vars(config, 'begrun') dagfh.write('%s\n' % varstr) blocklist = miscutils.fwsplit(config[pfwdefs.SW_BLOCKLIST].lower(), ',') for i in range(len(blocklist)): blockname = blocklist[i] blockdir = "../B%02d-%s" % (i + 1, blockname) cjobname = "B%02d-%s" % (i + 1, blockname) blockdag = write_block_dag(config, blockdir, blockname) dagfh.write(""" JOB %(cjob)s %(bdir)s/%(bdag)s.condor.sub SCRIPT pre %(cjob)s %(pdir)s/libexec/blockpre.py ../uberctrl/config.des SCRIPT post %(cjob)s %(pdir)s/libexec/blockpost.py ../uberctrl/config.des $RETURN """ % { 'cjob': cjobname, 'bdir': blockdir, 'bdag': blockdag, 'pdir': pfwdir }) varstr = create_common_vars(config, cjobname) dagfh.write('%s\n' % varstr) dagfh.write(""" JOB endrun %s/share/condor/runtask.condor VARS endrun exec="$(pfwdir)/libexec/endrun.py" VARS endrun arguments="../uberctrl/config.des" """ % (pfwdir)) varstr = create_common_vars(config, 'endrun') dagfh.write('%s\n' % varstr) child = "B%02d-%s" % (1, blocklist[0]) dagfh.write("PARENT begrun CHILD %s\n" % child) for i in range(1, len(blocklist)): parent = child child = "B%02d-%s" % (i + 1, blocklist[i]) dagfh.write("PARENT %s CHILD %s\n" % (parent, child)) dagfh.write("PARENT %s CHILD endrun\n" % child) dagfh.close() pfwcondor.add2dag(maindag, config.get_dag_cmd_opts(), config.get_condor_attributes('uberctrl', 'mainmngr'), None, sys.stdout)
def check_target_archive(config, indent=''): """ check info related to target archive """ cnts = [0] * NUMCNTS print "%sChecking target archive..." % (indent) blocklist = miscutils.fwsplit(config[pfwdefs.SW_BLOCKLIST].lower(), ',') for blockname in blocklist: config.set_block_info() (found_input, use_target_archive_input) = config.search(pfwdefs.USE_TARGET_ARCHIVE_INPUT, {pfwdefs.PF_CURRVALS: {'curr_block': blockname}}) (found_output, use_target_archive_output) = config.search(pfwdefs.USE_TARGET_ARCHIVE_OUTPUT, {pfwdefs.PF_CURRVALS: {'curr_block': blockname}}) (found_archive, _) = config.search(pfwdefs.TARGET_ARCHIVE, {pfwdefs.PF_CURRVALS: {'curr_block': blockname}}) if not found_input: error(indent+' ', "Could not determine %s" % (pfwdefs.USE_TARGET_ARCHIVE_INPUT)) cnts[ERRCNT_POS] += 1 elif use_target_archive_input.lower() not in pfwdefs.VALID_TARGET_ARCHIVE_INPUT: error(indent+' ', "Invalid %s value" % (pfwdefs.USE_TARGET_ARCHIVE_INPUT)) cnts[ERRCNT_POS] += 1 if not found_output: error(indent+' ', "Could not determine %s" % (pfwdefs.USE_TARGET_ARCHIVE_OUTPUT)) cnts[ERRCNT_POS] += 1 elif use_target_archive_output.lower() not in pfwdefs.VALID_TARGET_ARCHIVE_OUTPUT: error(indent+' ', "Invalid %s value" % (pfwdefs.USE_TARGET_ARCHIVE_OUTPUT)) cnts[ERRCNT_POS] += 1 # if need to use a target_archive for this block if ((found_input and use_target_archive_input.lower() != 'never') or (found_output and use_target_archive_output.lower() != 'never')): if not found_archive: error(indent+' ', "Missing %s value" % (pfwdefs.TARGET_ARCHIVE)) cnts[ERRCNT_POS] += 1 elif pfwdefs.SW_ARCHIVESECT not in config: error(indent+' ', "Needs archive section which doesn't exist") cnts[ERRCNT_POS] += 1 elif pfwdefs.TARGET_ARCHIVE not in config[pfwdefs.SW_ARCHIVESECT]: error(indent+' ', "Invalid %s value" % (pfwdefs.TARGET_ARCHIVE)) cnts[ERRCNT_POS] += 1 else: # check that we have all archive req values exist pass config.inc_blknum() config.reset_blknum() return cnts
def insert_attempt_label(self, config): """ Insert label for an attempt into pfw_attempt_label table """ if miscutils.fwdebug_check(3, 'PFWDB_DEBUG'): miscutils.fwdebug_print("Inserting into pfw_attempt_label table\n") row = {} row['pfw_attempt_id'] = config['pfw_attempt_id'] if pfwdefs.SW_LABEL in config: labels = config.getfull(pfwdefs.SW_LABEL) labels = miscutils.fwsplit(labels, ',') for label in labels: row['label'] = label self.insert_PFW_row('PFW_ATTEMPT_LABEL', row)
def get_file_fullnames(sect, filewcl, fullwcl): sectkeys = sect.split('.') sectname = sectkeys[1] if miscutils.fwdebug_check(3, 'INTGMISC_DEBUG'): miscutils.fwdebug_print("INFO: Beg sectname=%s" % sectname) fnames = [] if sectname in filewcl: filesect = filewcl[sectname] if 'fullname' in filesect: fnames = replfuncs.replace_vars(filesect['fullname'], fullwcl)[0] fnames = miscutils.fwsplit(fnames, ',') if miscutils.fwdebug_check(3, 'INTGMISC_DEBUG'): miscutils.fwdebug_print("INFO: fullname = %s" % fnames) return set(fnames)
def parse_wcl_objname(objname): #pylint: disable=unbalanced-tuple-unpacking """ Parse WCL object name into parts """ sect = name = subname = None parts = miscutils.fwsplit(objname, '.') #print 'parts=', parts if len(parts) == 3: # lists have 3 parts (sect, name, subname) = parts elif len(parts) == 2: # files have 2 parts (sect, name) = parts elif len(parts) == 1: name = parts[0] else: error('', "cannot parse objname %s (too many sections/periods)" % (objname)) return sect, name, subname
def check_globals(config, indent=''): """ Check global settings """ print "%sChecking globals..." % (indent) # initialize counters cnts = [0] * NUMCNTS # always required for key in [ 'pipeline', 'pipeprod', 'pipever', 'project', 'target_site', pfwdefs.SW_SITESECT, 'filename_pattern', 'directory_pattern', 'job_file_mvmt', pfwdefs.ATTEMPT_ARCHIVE_PATH, pfwdefs.SW_MODULESECT, 'create_junk_tarball', 'campaign' ]: try: if key not in config: error(indent + ' ', "missing %s global key or section" % (key)) cnts[ERRCNT_POS] += 1 except: error(indent + ' ', "missing %s global key or section" % (key)) cnts[ERRCNT_POS] += 1 print '%s Checking %s...' % (indent, pfwdefs.SW_SAVE_RUN_VALS) if pfwdefs.SW_SAVE_RUN_VALS in config: keys2save = config.getfull(pfwdefs.SW_SAVE_RUN_VALS) keys = miscutils.fwsplit(keys2save, ',') for key in keys: exists = False try: (exists, _) = config.search(key, { intgdefs.REPLACE_VARS: True, 'expand': True }) except SystemExit: pass if not exists: error(indent + ' ', 'Cannot determine %s value (%s).' % \ (pfwdefs.SW_SAVE_RUN_VALS, key)) cnts[ERRCNT_POS] += 1 return cnts
def __init__(self, dbh, config): self.config = config self.dbh = dbh self.verbose = 0 keyfile = self.config['keywords_file'] self.keywords = {'pri':{}, 'ext':{}} with open(keyfile, 'r') as fh: for line in fh: line = line.upper() (keyname, pri, ext) = miscutils.fwsplit(line, ',') if pri != 'Y' and pri != 'N' and pri != 'R': raise ValueError('Error: Invalid primary entry in keyword file (%s)' % line) if ext != 'Y' and ext != 'N' and ext != 'R': raise ValueError('Error: Invalid extenstion entry in keyword file (%s)' % line) self.keywords['pri'][keyname] = pri self.keywords['ext'][keyname] = ext
def check_input_files(sect, filewcl): """ Check that the files for a single input file section exist Parameters ---------- sect : str The WCL section to look in. filewcl : WCL The WCl to use Returns ------- tuple Two element tuple of lists containing the existing and missing files. """ sectkeys = sect.split('.') fnames = miscutils.fwsplit(filewcl[sectkeys[1]]['fullname'], ',') (exists1, missing1) = check_files(fnames) return (exists1, missing1)
def check_valid(self, listfullnames): """ Check whether the given files are valid raw files """ assert isinstance(listfullnames, list) results = {} for fname in listfullnames: results[fname] = False keyfile = None if 'raw_keywords_file' in self.config: keyfile = self.config['raw_keywords_file'] elif 'FILEMGMT_DIR' in os.environ: keyfile = '%s/etc/decam_src_keywords.txt' % os.environ[ 'FILEMGMT_DIR'] miscutils.fwdebug_print("keyfile = %s" % keyfile) if keyfile is not None and os.path.exists(keyfile): keywords = {'pri': {}, 'ext': {}} with open(keyfile, 'r') as keyfh: for line in keyfh: line = line.upper() [keyname, pri, ext] = miscutils.fwsplit(line, ',')[0:3] if pri != 'Y' and pri != 'N' and pri != 'R': raise ValueError( 'Invalid primary entry in keyword file (%s)' % line) if ext != 'Y' and ext != 'N' and ext != 'R': raise ValueError( 'Invalid extenstion entry in keyword file (%s)' % line) keywords['pri'][keyname] = pri keywords['ext'][keyname] = ext for fname in listfullnames: results[fname] = check_single_valid(keywords, fname, 0) else: raise OSError('Error: Could not find keywords file') return results
print args # # Make sure that required arguments are present and appear OK. # if (args.infile is None): print("Input filename (-i) is required") print("Aborting!") exit(1) if (not(os.path.isfile(args.infile))): print("Missing input file: {:s}".format(args.infile)) print("Aborting!") exit(1) if (args.outfile is None): print("Output filename (-o) is required") print("Aborting!") exit(1) if (args.ccdlist == "All"): ccd_list=range(1,63) else: ccd_list=fwsplit(args.ccdlist) ccd_list=[int(x) for x in ccd_list] status = scamputil.split_ahead_by_ccd(args.infile,args.outfile,ccd_list) if (not(status)): exit(1) else: exit(0)
def begblock(argv): """Program entry point. """ if argv == None: argv = sys.argv configfile = argv[0] config = pfwconfig.PfwConfig({'wclfile': configfile}) config.set_block_info() blknum = config[pfwdefs.PF_BLKNUM] blkdir = config.getfull('block_dir') os.chdir(blkdir) (exists, submit_des_services) = config.search('submit_des_services') if exists and submit_des_services is not None: os.environ['DES_SERVICES'] = submit_des_services (exists, submit_des_db_section) = config.search('submit_des_db_section') if exists and submit_des_db_section is not None: os.environ['DES_DB_SECTION'] = submit_des_db_section dbh = None blktid = -1 if miscutils.fwdebug_check(3, 'PFWBLOCK_DEBUG'): miscutils.fwdebug_print("blknum = %s" % (config[pfwdefs.PF_BLKNUM])) if miscutils.convertBool(config.getfull(pfwdefs.PF_USE_DB_OUT)): dbh = pfwdb.PFWDB(submit_des_services, submit_des_db_section) dbh.insert_block(config) blktid = config['task_id']['block'][str(blknum)] config['task_id']['begblock'] = dbh.create_task( name='begblock', info_table=None, parent_task_id=blktid, root_task_id=int(config['task_id']['attempt']), label=None, do_begin=True, do_commit=True) try: modulelist = miscutils.fwsplit( config.getfull(pfwdefs.SW_MODULELIST).lower()) modules_prev_in_list = {} joblist = {} parlist = OrderedDict() masterdata = OrderedDict() filelist = {'infiles': {}, 'outfiles': {}} for num, modname in enumerate(modulelist): print("XXXXXXXXXXXXXXXXXXXX %s XXXXXXXXXXXXXXXXXXXX" % modname) if modname not in config[pfwdefs.SW_MODULESECT]: miscutils.fwdie( "Error: Could not find module description for module %s\n" % (modname), pfwdefs.PF_EXIT_FAILURE) moddict = config[pfwdefs.SW_MODULESECT][modname] runqueries(config, configfile, modname, modules_prev_in_list) pfwblock.read_master_lists(config, modname, masterdata, modules_prev_in_list) (infsect, outfsect) = pfwblock.get_datasect_types(config, modname) pfwblock.fix_master_lists(config, modname, masterdata, outfsect) if pfwdefs.PF_NOOP not in moddict or not miscutils.convertBool( moddict[pfwdefs.PF_NOOP]): pfwblock.create_fullnames(config, modname, masterdata) if miscutils.fwdebug_check( 9, 'PFWBLOCK_DEBUG') and modname in masterdata: with open('%s-masterdata.txt' % modname, 'w') as fh: miscutils.pretty_print_dict(masterdata[modname], fh) pfwblock.add_file_metadata(config, modname) sublists = pfwblock.create_sublists(config, modname, masterdata) if sublists is not None: if miscutils.fwdebug_check(3, 'PFWBLOCK_DEBUG'): miscutils.fwdebug_print("sublists.keys() = %s" % (list(sublists.keys()))) loopvals = pfwblock.get_wrapper_loopvals(config, modname) wrapinst = pfwblock.create_wrapper_inst( config, modname, loopvals) wcnt = 1 for winst in list(wrapinst.values()): if miscutils.fwdebug_check(6, 'PFWBLOCK_DEBUG'): miscutils.fwdebug_print("winst %d - BEG" % wcnt) pfwblock.assign_data_wrapper_inst(config, modname, winst, masterdata, sublists, infsect, outfsect) pfwblock.finish_wrapper_inst(config, modname, winst, outfsect) tempfiles = pfwblock.create_module_wrapper_wcl( config, modname, winst) for fl in tempfiles['infiles']: if fl not in list(filelist['infiles'].keys()): filelist['infiles'][fl] = num for fl in tempfiles['outfiles']: filelist['outfiles'][fl] = num #filelist['infiles'] += tempfiles['infiles'] #filelist['outfiles'] += tempfiles['outfiles'] pfwblock.divide_into_jobs(config, modname, winst, joblist, parlist) if miscutils.fwdebug_check(6, 'PFWBLOCK_DEBUG'): miscutils.fwdebug_print("winst %d - %s - END" % (wcnt, etime - stime)) wcnt += 1 modules_prev_in_list[modname] = True if miscutils.fwdebug_check( 9, 'PFWBLOCK_DEBUG') and modname in masterdata: with open('%s-masterdata.txt' % modname, 'w') as fh: miscutils.pretty_print_dict(masterdata[modname], fh) scriptfile = pfwblock.write_runjob_script(config) intersect = list( set(filelist['infiles'].keys()) & set(filelist['outfiles'].keys())) finallist = [] for fl in list(filelist['infiles'].keys()): if fl not in intersect: finallist.append(fl) else: if filelist['infiles'][fl] <= filelist['outfiles'][fl]: raise Exception( 'Input file %s requested before it is generated.' % (fl)) if miscutils.convertBool(config.getfull(pfwdefs.PF_USE_DB_OUT)): missingfiles = dbh.check_files(config, finallist) if len(missingfiles) > 0: raise Exception( "The following input files cannot be found in the archive:" + ",".join(missingfiles)) miscutils.fwdebug_print("Creating job files - BEG") for jobkey, jobdict in sorted(joblist.items()): jobdict['jobnum'] = pfwutils.pad_jobnum(config.inc_jobnum()) jobdict['jobkeys'] = jobkey jobdict['numexpwrap'] = len(jobdict['tasks']) if miscutils.fwdebug_check(6, 'PFWBLOCK_DEBUG'): miscutils.fwdebug_print("jobnum = %s, jobkey = %s:" % (jobkey, jobdict['jobnum'])) jobdict['tasksfile'] = write_workflow_taskfile( config, jobdict['jobnum'], jobdict['tasks']) if (len(jobdict['inlist']) > 0 and config.getfull(pfwdefs.USE_HOME_ARCHIVE_OUTPUT) != 'never' and 'submit_files_mvmt' in config and (pfwdefs.PF_DRYRUN not in config or not miscutils.convertBool( config.getfull(pfwdefs.PF_DRYRUN)))): # get home archive info home_archive = config.getfull('home_archive') archive_info = config[pfwdefs.SW_ARCHIVESECT][home_archive] # load filemgmt class attempt_tid = config['task_id']['attempt'] filemgmt = pfwutils.pfw_dynam_load_class( dbh, config, attempt_tid, attempt_tid, "filemgmt", archive_info['filemgmt'], archive_info) # save file information filemgmt.register_file_data('list', jobdict['inlist'], config['pfw_attempt_id'], attempt_tid, False, None, None) pfwblock.copy_input_lists_home_archive(config, filemgmt, archive_info, jobdict['inlist']) filemgmt.commit() jobdict['inputwcltar'] = pfwblock.tar_inputfiles( config, jobdict['jobnum'], jobdict['inwcl'] + jobdict['inlist']) if miscutils.convertBool(config.getfull(pfwdefs.PF_USE_DB_OUT)): dbh.insert_job(config, jobdict) pfwblock.write_jobwcl(config, jobkey, jobdict) if ('glidein_use_wall' in config and miscutils.convertBool( config.getfull('glidein_use_wall')) and 'jobwalltime' in config): jobdict['wall'] = config['jobwalltime'] miscutils.fwdebug_print("Creating job files - END") numjobs = len(joblist) if miscutils.convertBool(config.getfull(pfwdefs.PF_USE_DB_OUT)): dbh.update_block_numexpjobs(config, numjobs) #if miscutils.fwdebug_check(6, 'PFWBLOCK_DEBUG'): # miscutils.fwdebug_print("inputfiles: %s, %s" % (type(inputfiles), inputfiles)) # miscutils.fwdebug_print("outputfiles: %s, %s" % (type(outputfiles), outputfiles)) #files2stage = set(inputfiles) - set(outputfiles) #pfwblock.stage_inputs(config, files2stage) #if pfwdefs.USE_HOME_ARCHIVE_OUTPUT in config and \ # config.getfull(pfwdefs.USE_HOME_ARCHIVE_OUTPUT).lower() == 'block': # config['block_outputlist'] = 'potential_outputfiles.list' # pfwblock.write_output_list(config, outputfiles) dagfile = config.get_filename('jobdag') pfwblock.create_jobmngr_dag(config, dagfile, scriptfile, joblist) except: retval = pfwdefs.PF_EXIT_FAILURE with open(configfile, 'w') as cfgfh: config.write( cfgfh) # save config, have updated jobnum, wrapnum, etc if miscutils.convertBool(config.getfull(pfwdefs.PF_USE_DB_OUT)): dbh.end_task(config['task_id']['begblock'], retval, True) dbh.end_task(blktid, retval, True) raise # save config, have updated jobnum, wrapnum, etc with open(configfile, 'w') as cfgfh: config.write(cfgfh) (exists, dryrun) = config.search(pfwdefs.PF_DRYRUN) if exists and miscutils.convertBool(dryrun): retval = pfwdefs.PF_EXIT_DRYRUN else: retval = pfwdefs.PF_EXIT_SUCCESS if miscutils.convertBool(config.getfull(pfwdefs.PF_USE_DB_OUT)): dbh.end_task(config['task_id']['begblock'], retval, True) miscutils.fwdebug_print("END - exiting with code %s" % retval) return retval
def blockpost(argv=None): """Program entry point. """ if argv is None: argv = sys.argv # open file to catch error messages about command line debugfh = open('blockpost.out', 'w') sys.stdout = debugfh sys.stderr = debugfh print(' '.join(argv)) # print command line for debugging print("running on %s" % (socket.gethostname())) if len(argv) != 3: print('Usage: blockpost.py configfile retval') debugfh.close() return pfwdefs.PF_EXIT_FAILURE configfile = argv[1] retval = int(argv[2]) if miscutils.fwdebug_check(3, 'PFWPOST_DEBUG'): miscutils.fwdebug_print("configfile = %s" % configfile) miscutils.fwdebug_print("retval = %s" % retval) # read sysinfo file config = pfwconfig.PfwConfig({'wclfile': configfile}) if miscutils.fwdebug_check(3, 'PFWPOST_DEBUG'): miscutils.fwdebug_print("done reading config file") blockname = config.getfull('blockname') blkdir = config.getfull('block_dir') # now that have more information, can rename output file miscutils.fwdebug_print("getting new_log_name") new_log_name = config.get_filename( 'block', {pfwdefs.PF_CURRVALS: { 'flabel': 'blockpost', 'fsuffix': 'out' }}) new_log_name = "%s/%s" % (blkdir, new_log_name) miscutils.fwdebug_print("new_log_name = %s" % new_log_name) debugfh.close() os.chmod('blockpost.out', 0o666) os.rename('blockpost.out', new_log_name) debugfh = open(new_log_name, 'a+') sys.stdout = debugfh sys.stderr = debugfh os.chdir(blkdir) log_pfw_event(config, blockname, 'blockpost', 'j', ['posttask', retval]) dryrun = config.getfull(pfwdefs.PF_DRYRUN) run = config.getfull('run') attid = config['pfw_attempt_id'] reqnum = config.getfull(pfwdefs.REQNUM) unitname = config.getfull(pfwdefs.UNITNAME) attnum = config.getfull(pfwdefs.ATTNUM) blknum = int(config.getfull(pfwdefs.PF_BLKNUM)) blktid = None msg2 = "" dbh = None job_byblk = {} wrap_byjob = {} wrap_bymod = {} wrapinfo = {} jobinfo = {} failedwraps = {} whyfailwraps = {} # mod failures for other modname, shouldn't happen usedb = miscutils.convertBool(config.getfull(pfwdefs.PF_USE_DB_OUT)) verify_files = miscutils.convertBool(config.getfull('verify_files')) verify_status = 0 if verify_files and not usedb: print('Skipping file verification due to lack of database connection') if miscutils.convertBool(config.getfull(pfwdefs.PF_USE_DB_OUT)): sem = None try: miscutils.fwdebug_print("Connecting to DB") dbh = pfwdb.PFWDB(config.getfull('submit_des_services'), config.getfull('submit_des_db_section')) if verify_files: curs = dbh.cursor() curs.execute("select root from ops_archive where name='%s'" % (config.getfull('home_archive'))) rows = curs.fetchall() if rows is None or len(rows) != 1: raise Exception( "Invalid archive name (%s). Found %s rows in ops_archive" % (config.getfull('home_archive'), len(rows))) root = rows[0][0] if not os.path.isdir(root): print( "Cannot read archive root directory:%s This program must be run on an NCSA machine with access to the archive storage system." % (config.getfull('home_archive'))) sem = dbsem.DBSemaphore( 'verify_files_10', None, config.getfull('submit_des_services'), config.getfull('submit_des_db_section')) print( "\n\nVerifying archive file sizes on disk (0 is success)") verify_status = cu.compare( dbh=dbh, archive=config.getfull('home_archive'), pfwid=attid, filesize=True, md5sum=False, quick=True, debug=False, script=False, verbose=False, silent=True) if sem is not None: del sem print(" Verification of files returned status %i" % (verify_status)) if verify_status != 0: print( " This indicates that one or more files do not have the correct file size (based on DB entries). Run" ) print( "\n compare_db.py --des_services %s --section %s --archive %s --pfwid %i --filesize --verbose" % (config.getfull('submit_des_services'), config.getfull('submit_des_db_section'), config.getfull('home_archive'), int(attid))) print("\n to see the details.") if miscutils.convertBool(config.getfull(pfwdefs.PF_USE_QCF)): import qcframework.qcfdb as qcfdb qdbh = qcfdb.QCFDB(config.getfull('submit_des_services'), config.getfull('submit_des_db_section')) print("\n\nChecking non-job block task status from task table in DB (%s is success)" % \ pfwdefs.PF_EXIT_SUCCESS) num_bltasks_failed = 0 bltasks = {} blktid = None if ('block' in config['task_id'] and str(blknum) in config['task_id']['block']): blktid = int(config['task_id']['block'][str(blknum)]) miscutils.fwdebug_print("Getting block task info from DB") start_time = time.time() bltasks = dbh.get_block_task_info(blktid) end_time = time.time() miscutils.fwdebug_print( "Done getting block task info from DB (%s secs)" % (end_time - start_time)) for bltdict in list(bltasks.values()): print("Block status = ", bltdict['status']) if bltdict['status'] == pfwdefs.PF_EXIT_DRYRUN: print("setting return value to dryrun") retval = bltdict['status'] elif bltdict['status'] != pfwdefs.PF_EXIT_SUCCESS: num_bltasks_failed += 1 msg2 += "\t%s" % (bltdict['name']) if bltdict['label'] is not None: msg2 += " - %s" % (bltdict['label']) msg2 += " failed\n" if bltdict['name'] == 'begblock': # try to read the begblock.out and begblock.err files print( "Trying to get begblock.out and begblock.err") msg2 += get_subblock_output("begblock") # try to get QCF messages (especially from query codes) begblock_tid = int(config['task_id']['begblock']) sql = "select id from task where parent_task_id=%i and status!=0" % ( begblock_tid) curs = dbh.cursor() curs.execute(sql) res = curs.fetchall() msg2 += "\n===== QCF Messages =====\n" msg2 += "\n begblock\n" wrapids = [blktid, begblock_tid] for r in res: wrapids.append(r[0]) wrapmsg = {} if qdbh is not None: miscutils.fwdebug_print( "Querying QCF messages") start_time = time.time() wrapmsg = qdbh.get_qcf_messages_for_wrappers( wrapids) end_time = time.time() miscutils.fwdebug_print( "Done querying QCF messages (%s secs)" % (end_time - start_time)) miscutils.fwdebug_print("wrapmsg = %s" % wrapmsg) if len(wrapmsg) == 0: msg2 += " No QCF messages\n" else: for msgs in list(wrapmsg.values()): for m in msgs: msg2 += " " + m['message'] + "\n" retval = pfwdefs.PF_EXIT_FAILURE if retval != pfwdefs.PF_EXIT_DRYRUN: print("\n\nChecking job status from pfw_job table in DB (%s is success)" % \ pfwdefs.PF_EXIT_SUCCESS) miscutils.fwdebug_print("Getting job info from DB") start_time = time.time() jobinfo = dbh.get_job_info({'pfw_block_task_id': blktid}) end_time = time.time() miscutils.fwdebug_print( "Done getting job info from DB (%s secs)" % (end_time - start_time)) miscutils.fwdebug_print("Getting wrapper info from DB") start_time = time.time() wrapinfo = dbh.get_wrapper_info(pfw_attempt_id=attid, pfw_block_task_id=blktid) end_time = time.time() miscutils.fwdebug_print( "Done getting wrapper info from DB (%s secs)" % (end_time - start_time)) else: msg = "Could not find task id for block %s in config.des" % blockname print("Error:", msg) if 'attempt' in config['task_id']: miscutils.fwdebug_print("Saving pfw message") start_time = time.time() Messaging.pfw_message(dbh, attid, config['task_id']['attempt'], msg, pfw_utils.PFW_DB_INFO, 'blockpost.out', 0) end_time = time.time() miscutils.fwdebug_print( "Done saving pfw message (%s secs)" % (end_time - start_time)) print("all the task ids:", config['task_id']) archive = None if pfwdefs.HOME_ARCHIVE in config: archive = config.getfull(pfwdefs.HOME_ARCHIVE) logfullnames = dbh.get_fail_log_fullnames(attid, archive) dbh.close() print("len(jobinfo) = ", len(jobinfo)) print("len(wrapinfo) = ", len(wrapinfo)) job_byblk = pfwutils.index_job_info(jobinfo) print("blktid: ", blktid) print("job_byblk:", job_byblk) if blktid not in job_byblk: print("Warn: could not find jobs for block %s" % blknum) print(" This is ok if attempt died before jobs ran") print(" block task_ids in job_byblk:" % list(job_byblk.keys())) else: wrap_byjob, wrap_bymod = pfwutils.index_wrapper_info(wrapinfo) #print "wrap_byjob:", wrap_byjob #print "wrap_bymod:", wrap_bymod for jobtid, jobdict in sorted(job_byblk[blktid].items()): failedwraps[jobtid] = [] whyfailwraps[jobtid] = [] jobkeys = "" # don't print out successful wrappers if jobtid in wrap_byjob and jobdict[ 'status'] == pfwdefs.PF_EXIT_SUCCESS: continue if jobdict['jobkeys'] is not None: jobkeys = jobdict['jobkeys'] #print "jobkeys = ", jobkeys, type(jobkeys) submit_job_path = "%s/B%02d-%s/%04d" % ( config.getfull('work_dir'), int(config.getfull('blknum')), config.getfull('blockname'), int(jobdict['jobnum'])) msg2 += "\n\t%s (%s) " % (pfwutils.pad_jobnum( jobdict['jobnum']), jobkeys) if jobtid not in wrap_byjob: msg2 += "\tNo wrapper instances" else: #print "wrapnum in job =", wrap_byjob[jobtid].keys() maxwrap = max(wrap_byjob[jobtid].keys()) #print "maxwrap =", maxwrap modname = wrap_byjob[jobtid][maxwrap]['modname'] #print "modname =", modname msg2 += "%d/%s %s" % (len( wrap_byjob[jobtid]), jobdict['expect_num_wrap'], modname) # determine wrappers for this job without success exit for wrapnum, wdict in list(wrap_byjob[jobtid].items()): if wdict['status'] is None or wdict[ 'status'] != pfwdefs.PF_EXIT_SUCCESS: if wdict['modname'] == modname: failedwraps[jobtid].append(wrapnum) else: whyfailwraps[jobtid].append(wrapnum) if jobdict['status'] == pfwdefs.PF_EXIT_EUPS_FAILURE: msg2 += " - FAIL - EUPS setup failure" retval = jobdict['status'] elif jobdict['status'] == pfwdefs.PF_EXIT_CONDOR: msg2 += " - FAIL - Condor/Globus failure" retval = jobdict['status'] elif jobdict['status'] is None: msg2 += " - FAIL - NULL status" retval = pfwdefs.PF_EXIT_FAILURE elif jobdict['status'] != pfwdefs.PF_EXIT_SUCCESS: msg2 += " - FAIL - Non-zero status" retval = jobdict['status'] if jobdict['status'] != pfwdefs.PF_EXIT_SUCCESS: msg2 += "\n\t\t%s/runjob.out " % (submit_job_path) msg2 += '\n' # print pfw_messages if 'message' in jobdict: print(jobdict['message']) for msgdict in sorted(jobdict['message'], key=lambda k: k['message_time']): level = int(msgdict['message_lvl']) levelstr = 'info' if level == pfwdefs.PFWDB_MSG_WARN: levelstr = 'WARN' elif level == pfwdefs.PFWDB_MSG_ERROR: levelstr = 'ERROR' msg2 += "\t\t%s - %s\n" % ( levelstr, msgdict['message'].replace( '\n', '\n\t\t\t')) if jobtid in wrap_byjob: # print log file name for failed/unfinished wrappers for wrapnum in failedwraps[jobtid]: wrapdict = wrap_byjob[jobtid][wrapnum] if wrapdict['log'] in logfullnames: msg2 += "\t\t%s - %s\n" % ( wrapnum, logfullnames[wrapdict['log']]) else: msg2 += "\t\t%s - Could not find log in archive (%s)\n" % ( wrapnum, wrapdict['log']) wrapmsg = get_qcf_messages(qdbh, config, [wrapdict['task_id']]) msg2 = print_qcf_messages(config, wrapdict, wrapmsg, msg2) msg2 += '\n' # If weirdness happened in run, print a message if len(whyfailwraps[jobtid]) > 0: msg2 += "\n*** Contact framework developers. Wrappers ran after at least 1 wrapper from a previous module that doesn't have success status.\n" msg2 += "\t%s\n" % ','.join(whyfailwraps[jobtid]) except Exception as exc: if sem is not None: del sem msg2 += "\n\nEncountered error trying to gather status information for email." msg2 += "\nCheck output for blockpost for further details." print( "\n\nEncountered error trying to gather status information for email" ) print("%s: %s" % (exc.__class__.__name__, str(exc))) (extype, exvalue, trback) = sys.exc_info() traceback.print_exception(extype, exvalue, trback, file=sys.stdout) retval = pfwdefs.PF_EXIT_FAILURE retval = int(retval) + verify_status print("before email retval =", retval) when_to_email = 'run' if 'when_to_email' in config: when_to_email = config.getfull('when_to_email').lower() if miscutils.convertBool(dryrun): if when_to_email != 'never': print("dryrun = ", dryrun) print("Sending dryrun email") if retval == pfwdefs.PF_EXIT_DRYRUN: msg1 = "%s: In dryrun mode, block %s has finished successfully." % ( run, blockname) else: msg1 = "%s: In dryrun mode, block %s has failed." % ( run, blockname) send_email(config, blockname, retval, "", msg1, msg2) else: print("Not sending dryrun email") print("retval = ", retval) retval = pfwdefs.PF_EXIT_DRYRUN elif retval: if when_to_email != 'never': print("Sending block failed email\n") msg1 = "%s: block %s has failed." % (run, blockname) send_email(config, blockname, retval, "", msg1, msg2) else: print("Not sending failed email") print("retval = ", retval) elif retval == pfwdefs.PF_EXIT_SUCCESS: if when_to_email == 'block': msg1 = "%s: block %s has finished successfully." % (run, blockname) msg2 = "" print("Sending success email\n") send_email(config, blockname, retval, "", msg1, msg2) elif when_to_email == 'run': numblocks = len( miscutils.fwsplit(config[pfwdefs.SW_BLOCKLIST], ',')) if int(config[pfwdefs.PF_BLKNUM]) == numblocks: msg1 = "%s: run has finished successfully." % (run) msg2 = "" print("Sending success email\n") send_email(config, blockname, retval, "", msg1, msg2) else: print("Not sending run email because not last block") print("retval = ", retval) else: print("Not sending success email") print("retval = ", retval) else: print("Not sending email") print("retval = ", retval) # Store values in DB and hist file dbh = None if miscutils.convertBool(config[pfwdefs.PF_USE_DB_OUT]): dbh = pfwdb.PFWDB(config.getfull('submit_des_services'), config.getfull('submit_des_db_section')) if blktid is not None: print("Updating end of block task", blktid) dbh.end_task(blktid, retval, True) else: print("Could not update end of block task without block task id") if retval != pfwdefs.PF_EXIT_SUCCESS: print("Updating end of attempt", config['task_id']['attempt']) dbh.end_task(config['task_id']['attempt'], retval, True) dbh.commit() dbh.close() print("before next block retval = ", retval) if retval == pfwdefs.PF_EXIT_SUCCESS: # Get ready for next block config.inc_blknum() with open(configfile, 'w') as cfgfh: config.write(cfgfh) print("new blknum = ", config[pfwdefs.PF_BLKNUM]) print("number of blocks = ", len(miscutils.fwsplit(config[pfwdefs.SW_BLOCKLIST], ','))) miscutils.fwdebug_print("Returning retval = %s (%s)" % (retval, type(retval))) miscutils.fwdebug_print("END") debugfh.close() return int(retval)