def getfull(self, key, opts=None, default=None):
""" Return with variables replaced and expanded if string(s) """
if miscutils.fwdebug_check(9, "WCL_DEBUG"):
miscutils.fwdebug_print(f"BEG - key={key}")
miscutils.fwdebug_print(f"default - {default}")
miscutils.fwdebug_print(f"opts - {opts}")
(found, value) = self.search(key, opts)
if not found:
value = default
elif isinstance(value, str):
if opts is None:
newopts = {'expand': True, intgdefs.REPLACE_VARS: True}
else:
newopts = copy.deepcopy(opts)
if intgdefs.REPLACE_VARS not in newopts or \
miscutils.convertBool(newopts[intgdefs.REPLACE_VARS]):
newopts['expand'] = True
if miscutils.fwdebug_check(9, "WCL_DEBUG"):
miscutils.fwdebug_print(
f"calling replace_vars value={value}, opts={newopts}")
(value, _) = replfuncs.replace_vars(value, self, newopts)
if len(value) == 1:
value = value[0]
return value
def get_file_fullnames(sect, filewcl, fullwcl):
""" Get the full name of the files in the specified section.
Parameters
----------
sect : str
The WCL section to use
filewcl : WCL
The WCl to use
fullwcl : WCL
The full WCL, used to generate the full names
Returns
-------
set
The full file names
"""
sectkeys = sect.split('.')
sectname = sectkeys[1]
if miscutils.fwdebug_check(3, 'INTGMISC_DEBUG'):
miscutils.fwdebug_print("INFO: Beg sectname=%s" % sectname)
fnames = []
if sectname in filewcl:
filesect = filewcl[sectname]
if 'fullname' in filesect:
fnames = replfuncs.replace_vars(filesect['fullname'], fullwcl)[0]
fnames = miscutils.fwsplit(fnames, ',')
if miscutils.fwdebug_check(3, 'INTGMISC_DEBUG'):
miscutils.fwdebug_print("INFO: fullname = %s" % fnames)
return set(fnames)
def get_file_fullnames(sect, filewcl, fullwcl):
sectkeys = sect.split('.')
sectname = sectkeys[1]
if miscutils.fwdebug_check(3, 'INTGMISC_DEBUG'):
miscutils.fwdebug_print("INFO: Beg sectname=%s" % sectname)
fnames = []
if sectname in filewcl:
filesect = filewcl[sectname]
if 'fullname' in filesect:
fnames = replfuncs.replace_vars(filesect['fullname'], fullwcl)[0]
fnames = miscutils.fwsplit(fnames, ',')
if miscutils.fwdebug_check(3, 'INTGMISC_DEBUG'):
miscutils.fwdebug_print("INFO: fullname = %s" % fnames)
return set(fnames)
def getfull(self, key, opts=None, default=None):
""" Return with variables replaced and expanded if string(s)
Parameters
----------
key : str
The key whose value is to be returned.
opts : dict, optional
Options for processing the value. Default is ``None``.
default : various, optional
The default value to return. Default is ``None``.
"""
if miscutils.fwdebug_check(9, "WCL_DEBUG"):
miscutils.fwdebug_print("BEG - key=%s" % key)
miscutils.fwdebug_print("default - %s" % default)
miscutils.fwdebug_print("opts - %s" % opts)
(found, value) = self.search(key, opts)
if not found:
value = default
elif isinstance(value, (str, unicode)):
if opts is None:
newopts = {'expand': True, intgdefs.REPLACE_VARS: True}
else:
newopts = copy.deepcopy(opts)
if intgdefs.REPLACE_VARS not in newopts or \
miscutils.convertBool(newopts[intgdefs.REPLACE_VARS]):
newopts['expand'] = True
if miscutils.fwdebug_check(9, "WCL_DEBUG"):
miscutils.fwdebug_print("calling replace_vars value=%s, opts=%s" % \
(value, newopts))
(value, _) = replfuncs.replace_vars(value, self, newopts)
if len(value) == 1:
value = value[0]
return value
def create_command_line(self, execnum, exwcl):
""" Create command line string handling hyphens appropriately"""
if miscutils.fwdebug_check(3, 'BASICWRAP_DEBUG'):
miscutils.fwdebug_print(
f"execnum = '{execnum}', exwcl = '{exwcl}'",
WRAPPER_OUTPUT_PREFIX)
self.start_exec_task('create_command_line')
#print 'WCL -----'
#print exwcl
#print 'END -----\n'
cmdstr = ""
if 'execname' in exwcl:
cmdlist = [exwcl['execname']]
if 'cmdline' in exwcl:
posargs = {} # save positional args to insert later
hyphen_type = 'allsingle'
if 'cmd_hyphen' in exwcl:
hyphen_type = exwcl['cmd_hyphen']
# loop through command line args
for key, val in exwcl['cmdline'].items():
if miscutils.fwdebug_check(3, 'BASICWRAP_DEBUG'):
miscutils.fwdebug_print(
f"key = '{key}', val = '{val}'",
WRAPPER_OUTPUT_PREFIX)
# replace any variables
expandval = replfuncs.replace_vars(val, self.inputwcl)[0]
if miscutils.fwdebug_check(3, 'BASICWRAP_DEBUG'):
miscutils.fwdebug_print(f"expandval = '{expandval}'",
WRAPPER_OUTPUT_PREFIX)
if key.startswith('_'):
patmatch = re.match(r'_(\d+)', key)
if patmatch:
posargs[patmatch.group(
1)] = expandval # save for later
else:
raise ValueError(
f'Invalid positional argument name: {key}')
else:
hyphen = intgmisc.get_cmd_hyphen(hyphen_type, key)
if expandval == '_flag':
cmdlist.append(f" {hyphen}{key}")
else:
cmdlist.append(f" {hyphen}{key} {expandval}")
# insert position sensitive arguments into specified location in argument list
for k in sorted(posargs.keys()):
cmdlist.insert(int(k), f"{posargs[k]}")
# convert list of args into string
if miscutils.fwdebug_check(6, 'BASICWRAP_DEBUG'):
miscutils.fwdebug_print(f"cmdlist = '{cmdlist}'",
WRAPPER_OUTPUT_PREFIX)
cmdstr = ' '.join(cmdlist)
else:
print(f"Error: missing execname in wcl for exec {execnum}")
print(f"exec wcl = {exwcl}")
raise KeyError(f'Missing execname in wcl for exec {execnum}')
self.curr_exec['cmdline'] = cmdstr
self.end_exec_task(0)
def get_filename(self, filepat=None, searchopts=None):
""" Return filename based upon given file pattern name """
if miscutils.fwdebug_check(6, 'PFWCONFIG_DEBUG'):
miscutils.fwdebug_print("given filepat = %s, type = %s" % (filepat, type(filepat)))
miscutils.fwdebug_print("given searchopts = %s" % (searchopts))
origreq = False
if searchopts is not None and 'required' in searchopts:
origreq = searchopts['required']
searchopts['required'] = False
if filepat is None:
# first check for filename pattern override
if miscutils.fwdebug_check(6, 'PFWCONFIG_DEBUG'):
miscutils.fwdebug_print("first check for filename pattern override")
(found, filenamepat) = self.search('filename', searchopts)
if not found:
# get filename pattern from global settings:
if miscutils.fwdebug_check(6, 'PFWCONFIG_DEBUG'):
miscutils.fwdebug_print("get filename pattern from global settings")
(found, filepat) = self.search(pfwdefs.SW_FILEPAT, searchopts)
if not found:
islist = 'searchobj' in searchopts and 'fsuffix' in searchopts['searchobj'] and searchopts['searchobj']['fsuffix'] == pfwdefs.SW_LISTSECT
msg = "Error: Could not find file pattern (%s) in " % pfwdefs.SW_FILEPAT
if islist:
msg += "list def section"
else:
msg += "file def section"
if pfwdefs.PF_CURRVALS in searchopts and 'curr_module' in searchopts[pfwdefs.PF_CURRVALS]:
msg += " of %s" % searchopts[pfwdefs.PF_CURRVALS]['curr_module']
if 'searchobj' in searchopts and 'flabel' in searchopts['searchobj']:
if islist:
msg += ", list"
else:
msg += ", file"
msg += " %s" % searchopts['searchobj']['flabel']
miscutils.fwdie(msg, pfwdefs.PF_EXIT_FAILURE, 2)
elif miscutils.fwdebug_check(6, 'PFWCONFIG_DEBUG'):
miscutils.fwdebug_print("working with given filepat = %s" % (filepat))
if miscutils.fwdebug_check(6, 'PFWCONFIG_DEBUG'):
miscutils.fwdebug_print("filepat = %s" % (filepat))
if pfwdefs.SW_FILEPATSECT not in self:
self.write()
miscutils.fwdie("Error: Could not find filename pattern section (%s) in config" % \
pfwdefs.SW_FILEPATSECT, pfwdefs.PF_EXIT_FAILURE)
elif filepat in self[pfwdefs.SW_FILEPATSECT]:
filenamepat = self[pfwdefs.SW_FILEPATSECT][filepat]
else:
miscutils.fwdebug_print("%s keys: %s" % (pfwdefs.SW_FILEPATSECT,
self[pfwdefs.SW_FILEPATSECT].keys()))
print "searchopts =", searchopts
miscutils.fwdie("Error: Could not find value for filename pattern '%s' in file pattern section" % filepat, pfwdefs.PF_EXIT_FAILURE, 2)
if searchopts is not None:
searchopts['required'] = origreq
retval = filenamepat
if (searchopts is None or intgdefs.REPLACE_VARS not in searchopts or
miscutils.convertBool(searchopts[intgdefs.REPLACE_VARS])):
sopt2 = {}
if searchopts is not None:
sopt2 = copy.deepcopy(searchopts)
sopt2[intgdefs.REPLACE_VARS] = True
if 'expand' not in sopt2:
sopt2['expand'] = True
if 'keepvars' not in sopt2:
sopt2['keepvars'] = False
retval = replfuncs.replace_vars(filenamepat, self, sopt2)
if not miscutils.convertBool(sopt2['keepvars']):
retval = retval[0]
return retval
def create_command_line(self, execnum, exwcl):
""" Create command line string handling hyphens appropriately"""
if miscutils.fwdebug_check(3, 'BASICWRAP_DEBUG'):
miscutils.fwdebug_print(
"execnum = '%s', exwcl = '%s'" % (execnum, exwcl),
WRAPPER_OUTPUT_PREFIX)
self.start_exec_task('create_command_line')
cmdstr = ""
if 'execname' in exwcl:
cmdlist = [exwcl['execname']]
if 'cmdline' in exwcl:
posargs = {} # save positional args to insert later
hyphen_type = 'allsingle'
if 'cmd_hyphen' in exwcl:
hyphen_type = exwcl['cmd_hyphen']
# loop through command line args
for key, val in exwcl['cmdline'].items():
if miscutils.fwdebug_check(3, 'BASICWRAP_DEBUG'):
miscutils.fwdebug_print(
"key = '%s', val = '%s'" % (key, val),
WRAPPER_OUTPUT_PREFIX)
# replace any variables
expandval = replfuncs.replace_vars(val, self.inputwcl)[0]
if miscutils.fwdebug_check(3, 'BASICWRAP_DEBUG'):
miscutils.fwdebug_print(
"expandval = '%s'" % (expandval),
WRAPPER_OUTPUT_PREFIX)
if key.startswith('_'):
patmatch = re.match(r'_(\d+)', key)
if patmatch:
posargs[patmatch.group(
1)] = expandval # save for later
else:
raise ValueError(
'Invalid positional argument name: %s' % key)
else:
hyphen = intgmisc.get_cmd_hyphen(hyphen_type, key)
if expandval == '_flag':
cmdlist.append(" %s%s" % (hyphen, key))
else:
cmdlist.append(" %s%s %s" %
(hyphen, key, expandval))
# insert position sensitive arguments into specified location in argument list
for k in sorted(posargs.iterkeys()):
cmdlist.insert(int(k), "%s" % posargs[k])
# convert list of args into string
if miscutils.fwdebug_check(6, 'BASICWRAP_DEBUG'):
miscutils.fwdebug_print("cmdlist = '%s'" % (cmdlist),
WRAPPER_OUTPUT_PREFIX)
cmdstr = ' '.join(cmdlist)
else:
print "Error: missing execname in wcl for exec #%d" % execnum
print "exec wcl = %s" % exwcl
raise KeyError('Missing execname in wcl for exec #%d' % execnum)
self.curr_exec['cmdline'] = cmdstr
self.end_exec_task(0)
def main(argv):
""" Program entry point """
parser = argparse.ArgumentParser(description='genquery.py')
parser.add_argument('--qoutfile', action='store')
parser.add_argument('--qouttype', action='store')
parser.add_argument('--config', action='store', dest='configfile')
parser.add_argument('--module', action='store', dest='modulename')
parser.add_argument('--search', action='store', dest='searchname')
args = parser.parse_args(argv)
if args.modulename is None:
raise Exception("Error: Must specify module\n")
print(args.configfile)
config = pfwconfig.PfwConfig({'wclfile':args.configfile})
if args.modulename not in config[pfwdefs.SW_MODULESECT]:
raise Exception(f"Error: module '{args.modulename}' does not exist.\n")
module_dict = config[pfwdefs.SW_MODULESECT][args.modulename]
if args.searchname is not None:
if pfwdefs.SW_LISTSECT in module_dict and \
args.searchname in module_dict[pfwdefs.SW_LISTSECT]:
search_dict = module_dict[pfwdefs.SW_LISTSECT][args.searchname]
elif pfwdefs.SW_FILESECT in module_dict and \
args.searchname in module_dict[pfwdefs.SW_FILESECT]:
search_dict = module_dict[pfwdefs.SW_FILESECT][args.searchname]
else:
raise Exception(f"Error: Could not find either list or file by name {args.searchname} in module {args.modulename}\n")
else:
raise Exception("Error: need to define either list or file or search\n")
archive_names = []
if config.getfull(pfwdefs.USE_HOME_ARCHIVE_INPUT) != 'never':
archive_names.append(config.getfull(pfwdefs.HOME_ARCHIVE))
if config.getfull(pfwdefs.USE_TARGET_ARCHIVE_INPUT) != 'never':
archive_names.append(config.getfull(pfwdefs.TARGET_ARCHIVE))
fields = miscutils.fwsplit(search_dict[pfwdefs.SW_QUERYFIELDS].lower())
if ('query_run' in config and 'fileclass' in search_dict and
'fileclass' in config and search_dict['fileclass'] == config['fileclass']):
query_run = config['query_run'].lower()
if query_run == 'current':
fields.append('run')
elif query_run == 'allbutfirstcurrent':
if 'current' not in config:
raise Exception("Internal Error: Current object doesn't exist\n")
if 'curr_blocknum' not in config['current']:
raise Exception("Internal Error: current->curr_blocknum doesn't exist\n")
block_num = config['current']['curr_blocknum']
if block_num > 0:
fields.append('run')
query = {}
qtable = search_dict['query_table']
for fld in fields:
table = qtable
if '.' in fld:
table, fld = fld.split('.')
if fld in search_dict:
value = search_dict[fld]
elif fld in module_dict:
value = module_dict[fld]
elif fld in config:
value = config.getfull(fld)
else:
raise Exception(f"Error: genquery could not find value for query field {fld}\n")
value = replfuncs.replace_vars(value, config,
{pfwdefs.PF_CURRVALS: {'modulename': args.modulename},
'searchobj': search_dict,
intgdefs.REPLACE_VARS: True,
'expand': True})[0]
if value is None:
raise Exception(f"Value=None for query field {fld}\n")
if ',' in value:
value = miscutils.fwsplit(value)
if ':' in value:
value = miscutils.fwsplit(value)
if table not in query:
query[table] = {}
if 'key_vals' not in query[table]:
query[table]['key_vals'] = {}
query[table]['key_vals'][fld] = value
# if specified, insert join into query hash
if 'join' in search_dict:
joins = miscutils.fwsplit(search_dict['join'].lower())
for j in joins:
jmatch = re.search(r"(\S+)\.(\S+)\s*=\s*(\S+)", j)
if jmatch:
table = jmatch.group(1)
if table not in query:
query[table] = {}
if 'join' not in query[table]:
query[table]['join'] = j
else:
query[jmatch.group(1)]['join'] += "," + j
#query[table]['join']=search_dict['join']
query[qtable]['select_fields'] = ['filename']
# check output fields for fields from other tables.
if 'output_fields' in search_dict:
output_fields = miscutils.fwsplit(search_dict['output_fields'].lower())
for ofield in output_fields:
ofmatch = re.search(r"(\S+)\.(\S+)", ofield)
if ofmatch:
table = ofmatch.group(1)
field = ofmatch.group(2)
else:
table = qtable
field = ofield
if table not in query:
query[table] = {}
if 'select_fields' not in query[table]:
query[table]['select_fields'] = []
if field not in query[table]['select_fields']:
query[table]['select_fields'].append(field)
for tbl in query:
if 'select_fields' in query[tbl]:
query[tbl]['select_fields'] = ','.join(query[tbl]['select_fields'])
if archive_names:
#query[qtable]['join'] = "%s.filename=file_archive_info.filename" % qtable
query['file_archive_info'] = {'select_fields': 'compression'}
query['file_archive_info']['join'] = f"file_archive_info.filename={qtable}.filename"
query['file_archive_info']['key_vals'] = {'archive_name': ','.join(archive_names)}
print("Calling gen_file_list with the following query:\n")
miscutils.pretty_print_dict(query, out_file=None, sortit=False, indent=4)
print("\n\n")
dbh = pfwdb.PFWDB(config.getfull('submit_des_services'),
config.getfull('submit_des_db_section'))
files = queryutils.gen_file_list(dbh, query)
if not files:
raise Exception(f"genquery: query returned zero results for {args.searchname}\nAborting\n")
## output list
lines = queryutils.convert_single_files_to_lines(files)
queryutils.output_lines(args.qoutfile, lines, args.qouttype)
return 0