def getfull(self, key, opts=None, default=None): """ Return with variables replaced and expanded if string(s) """ if miscutils.fwdebug_check(9, "WCL_DEBUG"): miscutils.fwdebug_print(f"BEG - key={key}") miscutils.fwdebug_print(f"default - {default}") miscutils.fwdebug_print(f"opts - {opts}") (found, value) = self.search(key, opts) if not found: value = default elif isinstance(value, str): if opts is None: newopts = {'expand': True, intgdefs.REPLACE_VARS: True} else: newopts = copy.deepcopy(opts) if intgdefs.REPLACE_VARS not in newopts or \ miscutils.convertBool(newopts[intgdefs.REPLACE_VARS]): newopts['expand'] = True if miscutils.fwdebug_check(9, "WCL_DEBUG"): miscutils.fwdebug_print( f"calling replace_vars value={value}, opts={newopts}") (value, _) = replfuncs.replace_vars(value, self, newopts) if len(value) == 1: value = value[0] return value
def get_file_fullnames(sect, filewcl, fullwcl): """ Get the full name of the files in the specified section. Parameters ---------- sect : str The WCL section to use filewcl : WCL The WCl to use fullwcl : WCL The full WCL, used to generate the full names Returns ------- set The full file names """ sectkeys = sect.split('.') sectname = sectkeys[1] if miscutils.fwdebug_check(3, 'INTGMISC_DEBUG'): miscutils.fwdebug_print("INFO: Beg sectname=%s" % sectname) fnames = [] if sectname in filewcl: filesect = filewcl[sectname] if 'fullname' in filesect: fnames = replfuncs.replace_vars(filesect['fullname'], fullwcl)[0] fnames = miscutils.fwsplit(fnames, ',') if miscutils.fwdebug_check(3, 'INTGMISC_DEBUG'): miscutils.fwdebug_print("INFO: fullname = %s" % fnames) return set(fnames)
def get_file_fullnames(sect, filewcl, fullwcl): sectkeys = sect.split('.') sectname = sectkeys[1] if miscutils.fwdebug_check(3, 'INTGMISC_DEBUG'): miscutils.fwdebug_print("INFO: Beg sectname=%s" % sectname) fnames = [] if sectname in filewcl: filesect = filewcl[sectname] if 'fullname' in filesect: fnames = replfuncs.replace_vars(filesect['fullname'], fullwcl)[0] fnames = miscutils.fwsplit(fnames, ',') if miscutils.fwdebug_check(3, 'INTGMISC_DEBUG'): miscutils.fwdebug_print("INFO: fullname = %s" % fnames) return set(fnames)
def getfull(self, key, opts=None, default=None): """ Return with variables replaced and expanded if string(s) Parameters ---------- key : str The key whose value is to be returned. opts : dict, optional Options for processing the value. Default is ``None``. default : various, optional The default value to return. Default is ``None``. """ if miscutils.fwdebug_check(9, "WCL_DEBUG"): miscutils.fwdebug_print("BEG - key=%s" % key) miscutils.fwdebug_print("default - %s" % default) miscutils.fwdebug_print("opts - %s" % opts) (found, value) = self.search(key, opts) if not found: value = default elif isinstance(value, (str, unicode)): if opts is None: newopts = {'expand': True, intgdefs.REPLACE_VARS: True} else: newopts = copy.deepcopy(opts) if intgdefs.REPLACE_VARS not in newopts or \ miscutils.convertBool(newopts[intgdefs.REPLACE_VARS]): newopts['expand'] = True if miscutils.fwdebug_check(9, "WCL_DEBUG"): miscutils.fwdebug_print("calling replace_vars value=%s, opts=%s" % \ (value, newopts)) (value, _) = replfuncs.replace_vars(value, self, newopts) if len(value) == 1: value = value[0] return value
def create_command_line(self, execnum, exwcl): """ Create command line string handling hyphens appropriately""" if miscutils.fwdebug_check(3, 'BASICWRAP_DEBUG'): miscutils.fwdebug_print( f"execnum = '{execnum}', exwcl = '{exwcl}'", WRAPPER_OUTPUT_PREFIX) self.start_exec_task('create_command_line') #print 'WCL -----' #print exwcl #print 'END -----\n' cmdstr = "" if 'execname' in exwcl: cmdlist = [exwcl['execname']] if 'cmdline' in exwcl: posargs = {} # save positional args to insert later hyphen_type = 'allsingle' if 'cmd_hyphen' in exwcl: hyphen_type = exwcl['cmd_hyphen'] # loop through command line args for key, val in exwcl['cmdline'].items(): if miscutils.fwdebug_check(3, 'BASICWRAP_DEBUG'): miscutils.fwdebug_print( f"key = '{key}', val = '{val}'", WRAPPER_OUTPUT_PREFIX) # replace any variables expandval = replfuncs.replace_vars(val, self.inputwcl)[0] if miscutils.fwdebug_check(3, 'BASICWRAP_DEBUG'): miscutils.fwdebug_print(f"expandval = '{expandval}'", WRAPPER_OUTPUT_PREFIX) if key.startswith('_'): patmatch = re.match(r'_(\d+)', key) if patmatch: posargs[patmatch.group( 1)] = expandval # save for later else: raise ValueError( f'Invalid positional argument name: {key}') else: hyphen = intgmisc.get_cmd_hyphen(hyphen_type, key) if expandval == '_flag': cmdlist.append(f" {hyphen}{key}") else: cmdlist.append(f" {hyphen}{key} {expandval}") # insert position sensitive arguments into specified location in argument list for k in sorted(posargs.keys()): cmdlist.insert(int(k), f"{posargs[k]}") # convert list of args into string if miscutils.fwdebug_check(6, 'BASICWRAP_DEBUG'): miscutils.fwdebug_print(f"cmdlist = '{cmdlist}'", WRAPPER_OUTPUT_PREFIX) cmdstr = ' '.join(cmdlist) else: print(f"Error: missing execname in wcl for exec {execnum}") print(f"exec wcl = {exwcl}") raise KeyError(f'Missing execname in wcl for exec {execnum}') self.curr_exec['cmdline'] = cmdstr self.end_exec_task(0)
def get_filename(self, filepat=None, searchopts=None): """ Return filename based upon given file pattern name """ if miscutils.fwdebug_check(6, 'PFWCONFIG_DEBUG'): miscutils.fwdebug_print("given filepat = %s, type = %s" % (filepat, type(filepat))) miscutils.fwdebug_print("given searchopts = %s" % (searchopts)) origreq = False if searchopts is not None and 'required' in searchopts: origreq = searchopts['required'] searchopts['required'] = False if filepat is None: # first check for filename pattern override if miscutils.fwdebug_check(6, 'PFWCONFIG_DEBUG'): miscutils.fwdebug_print("first check for filename pattern override") (found, filenamepat) = self.search('filename', searchopts) if not found: # get filename pattern from global settings: if miscutils.fwdebug_check(6, 'PFWCONFIG_DEBUG'): miscutils.fwdebug_print("get filename pattern from global settings") (found, filepat) = self.search(pfwdefs.SW_FILEPAT, searchopts) if not found: islist = 'searchobj' in searchopts and 'fsuffix' in searchopts['searchobj'] and searchopts['searchobj']['fsuffix'] == pfwdefs.SW_LISTSECT msg = "Error: Could not find file pattern (%s) in " % pfwdefs.SW_FILEPAT if islist: msg += "list def section" else: msg += "file def section" if pfwdefs.PF_CURRVALS in searchopts and 'curr_module' in searchopts[pfwdefs.PF_CURRVALS]: msg += " of %s" % searchopts[pfwdefs.PF_CURRVALS]['curr_module'] if 'searchobj' in searchopts and 'flabel' in searchopts['searchobj']: if islist: msg += ", list" else: msg += ", file" msg += " %s" % searchopts['searchobj']['flabel'] miscutils.fwdie(msg, pfwdefs.PF_EXIT_FAILURE, 2) elif miscutils.fwdebug_check(6, 'PFWCONFIG_DEBUG'): miscutils.fwdebug_print("working with given filepat = %s" % (filepat)) if miscutils.fwdebug_check(6, 'PFWCONFIG_DEBUG'): miscutils.fwdebug_print("filepat = %s" % (filepat)) if pfwdefs.SW_FILEPATSECT not in self: self.write() miscutils.fwdie("Error: Could not find filename pattern section (%s) in config" % \ pfwdefs.SW_FILEPATSECT, pfwdefs.PF_EXIT_FAILURE) elif filepat in self[pfwdefs.SW_FILEPATSECT]: filenamepat = self[pfwdefs.SW_FILEPATSECT][filepat] else: miscutils.fwdebug_print("%s keys: %s" % (pfwdefs.SW_FILEPATSECT, self[pfwdefs.SW_FILEPATSECT].keys())) print "searchopts =", searchopts miscutils.fwdie("Error: Could not find value for filename pattern '%s' in file pattern section" % filepat, pfwdefs.PF_EXIT_FAILURE, 2) if searchopts is not None: searchopts['required'] = origreq retval = filenamepat if (searchopts is None or intgdefs.REPLACE_VARS not in searchopts or miscutils.convertBool(searchopts[intgdefs.REPLACE_VARS])): sopt2 = {} if searchopts is not None: sopt2 = copy.deepcopy(searchopts) sopt2[intgdefs.REPLACE_VARS] = True if 'expand' not in sopt2: sopt2['expand'] = True if 'keepvars' not in sopt2: sopt2['keepvars'] = False retval = replfuncs.replace_vars(filenamepat, self, sopt2) if not miscutils.convertBool(sopt2['keepvars']): retval = retval[0] return retval
def create_command_line(self, execnum, exwcl): """ Create command line string handling hyphens appropriately""" if miscutils.fwdebug_check(3, 'BASICWRAP_DEBUG'): miscutils.fwdebug_print( "execnum = '%s', exwcl = '%s'" % (execnum, exwcl), WRAPPER_OUTPUT_PREFIX) self.start_exec_task('create_command_line') cmdstr = "" if 'execname' in exwcl: cmdlist = [exwcl['execname']] if 'cmdline' in exwcl: posargs = {} # save positional args to insert later hyphen_type = 'allsingle' if 'cmd_hyphen' in exwcl: hyphen_type = exwcl['cmd_hyphen'] # loop through command line args for key, val in exwcl['cmdline'].items(): if miscutils.fwdebug_check(3, 'BASICWRAP_DEBUG'): miscutils.fwdebug_print( "key = '%s', val = '%s'" % (key, val), WRAPPER_OUTPUT_PREFIX) # replace any variables expandval = replfuncs.replace_vars(val, self.inputwcl)[0] if miscutils.fwdebug_check(3, 'BASICWRAP_DEBUG'): miscutils.fwdebug_print( "expandval = '%s'" % (expandval), WRAPPER_OUTPUT_PREFIX) if key.startswith('_'): patmatch = re.match(r'_(\d+)', key) if patmatch: posargs[patmatch.group( 1)] = expandval # save for later else: raise ValueError( 'Invalid positional argument name: %s' % key) else: hyphen = intgmisc.get_cmd_hyphen(hyphen_type, key) if expandval == '_flag': cmdlist.append(" %s%s" % (hyphen, key)) else: cmdlist.append(" %s%s %s" % (hyphen, key, expandval)) # insert position sensitive arguments into specified location in argument list for k in sorted(posargs.iterkeys()): cmdlist.insert(int(k), "%s" % posargs[k]) # convert list of args into string if miscutils.fwdebug_check(6, 'BASICWRAP_DEBUG'): miscutils.fwdebug_print("cmdlist = '%s'" % (cmdlist), WRAPPER_OUTPUT_PREFIX) cmdstr = ' '.join(cmdlist) else: print "Error: missing execname in wcl for exec #%d" % execnum print "exec wcl = %s" % exwcl raise KeyError('Missing execname in wcl for exec #%d' % execnum) self.curr_exec['cmdline'] = cmdstr self.end_exec_task(0)
def main(argv): """ Program entry point """ parser = argparse.ArgumentParser(description='genquery.py') parser.add_argument('--qoutfile', action='store') parser.add_argument('--qouttype', action='store') parser.add_argument('--config', action='store', dest='configfile') parser.add_argument('--module', action='store', dest='modulename') parser.add_argument('--search', action='store', dest='searchname') args = parser.parse_args(argv) if args.modulename is None: raise Exception("Error: Must specify module\n") print(args.configfile) config = pfwconfig.PfwConfig({'wclfile':args.configfile}) if args.modulename not in config[pfwdefs.SW_MODULESECT]: raise Exception(f"Error: module '{args.modulename}' does not exist.\n") module_dict = config[pfwdefs.SW_MODULESECT][args.modulename] if args.searchname is not None: if pfwdefs.SW_LISTSECT in module_dict and \ args.searchname in module_dict[pfwdefs.SW_LISTSECT]: search_dict = module_dict[pfwdefs.SW_LISTSECT][args.searchname] elif pfwdefs.SW_FILESECT in module_dict and \ args.searchname in module_dict[pfwdefs.SW_FILESECT]: search_dict = module_dict[pfwdefs.SW_FILESECT][args.searchname] else: raise Exception(f"Error: Could not find either list or file by name {args.searchname} in module {args.modulename}\n") else: raise Exception("Error: need to define either list or file or search\n") archive_names = [] if config.getfull(pfwdefs.USE_HOME_ARCHIVE_INPUT) != 'never': archive_names.append(config.getfull(pfwdefs.HOME_ARCHIVE)) if config.getfull(pfwdefs.USE_TARGET_ARCHIVE_INPUT) != 'never': archive_names.append(config.getfull(pfwdefs.TARGET_ARCHIVE)) fields = miscutils.fwsplit(search_dict[pfwdefs.SW_QUERYFIELDS].lower()) if ('query_run' in config and 'fileclass' in search_dict and 'fileclass' in config and search_dict['fileclass'] == config['fileclass']): query_run = config['query_run'].lower() if query_run == 'current': fields.append('run') elif query_run == 'allbutfirstcurrent': if 'current' not in config: raise Exception("Internal Error: Current object doesn't exist\n") if 'curr_blocknum' not in config['current']: raise Exception("Internal Error: current->curr_blocknum doesn't exist\n") block_num = config['current']['curr_blocknum'] if block_num > 0: fields.append('run') query = {} qtable = search_dict['query_table'] for fld in fields: table = qtable if '.' in fld: table, fld = fld.split('.') if fld in search_dict: value = search_dict[fld] elif fld in module_dict: value = module_dict[fld] elif fld in config: value = config.getfull(fld) else: raise Exception(f"Error: genquery could not find value for query field {fld}\n") value = replfuncs.replace_vars(value, config, {pfwdefs.PF_CURRVALS: {'modulename': args.modulename}, 'searchobj': search_dict, intgdefs.REPLACE_VARS: True, 'expand': True})[0] if value is None: raise Exception(f"Value=None for query field {fld}\n") if ',' in value: value = miscutils.fwsplit(value) if ':' in value: value = miscutils.fwsplit(value) if table not in query: query[table] = {} if 'key_vals' not in query[table]: query[table]['key_vals'] = {} query[table]['key_vals'][fld] = value # if specified, insert join into query hash if 'join' in search_dict: joins = miscutils.fwsplit(search_dict['join'].lower()) for j in joins: jmatch = re.search(r"(\S+)\.(\S+)\s*=\s*(\S+)", j) if jmatch: table = jmatch.group(1) if table not in query: query[table] = {} if 'join' not in query[table]: query[table]['join'] = j else: query[jmatch.group(1)]['join'] += "," + j #query[table]['join']=search_dict['join'] query[qtable]['select_fields'] = ['filename'] # check output fields for fields from other tables. if 'output_fields' in search_dict: output_fields = miscutils.fwsplit(search_dict['output_fields'].lower()) for ofield in output_fields: ofmatch = re.search(r"(\S+)\.(\S+)", ofield) if ofmatch: table = ofmatch.group(1) field = ofmatch.group(2) else: table = qtable field = ofield if table not in query: query[table] = {} if 'select_fields' not in query[table]: query[table]['select_fields'] = [] if field not in query[table]['select_fields']: query[table]['select_fields'].append(field) for tbl in query: if 'select_fields' in query[tbl]: query[tbl]['select_fields'] = ','.join(query[tbl]['select_fields']) if archive_names: #query[qtable]['join'] = "%s.filename=file_archive_info.filename" % qtable query['file_archive_info'] = {'select_fields': 'compression'} query['file_archive_info']['join'] = f"file_archive_info.filename={qtable}.filename" query['file_archive_info']['key_vals'] = {'archive_name': ','.join(archive_names)} print("Calling gen_file_list with the following query:\n") miscutils.pretty_print_dict(query, out_file=None, sortit=False, indent=4) print("\n\n") dbh = pfwdb.PFWDB(config.getfull('submit_des_services'), config.getfull('submit_des_db_section')) files = queryutils.gen_file_list(dbh, query) if not files: raise Exception(f"genquery: query returned zero results for {args.searchname}\nAborting\n") ## output list lines = queryutils.convert_single_files_to_lines(files) queryutils.output_lines(args.qoutfile, lines, args.qouttype) return 0