def write_outputwcl(self, outfilename=None):
""" Write output wcl to file
Parameters
----------
outfilename : str, optional
The anem of the output wcl file to write. Default is ``None``
which indicates that the file name is stored in the inputwcl.
"""
if outfilename is None:
outfilename = self.inputwcl['wrapper']['outputwcl']
if miscutils.fwdebug_check(3, 'BASICWRAP_DEBUG'):
miscutils.fwdebug_print("outfilename = %s" % outfilename,
WRAPPER_OUTPUT_PREFIX)
# create output wcl directory if needed
outwcldir = miscutils.parse_fullname(outfilename,
miscutils.CU_PARSE_PATH)
if miscutils.fwdebug_check(3, 'BASICWRAP_DEBUG'):
miscutils.fwdebug_print("outwcldir = %s" % outwcldir,
WRAPPER_OUTPUT_PREFIX)
miscutils.coremakedirs(outwcldir)
with open(outfilename, 'w') as wclfh:
self.outputwcl.write(wclfh, True)
def save_outputs_by_section(self, ekey, outexist):
""" save fullnames from outexist to outputs by section """
if miscutils.fwdebug_check(3, 'BASICWRAP_DEBUG'):
miscutils.fwdebug_print(
f"INFO: before adding outputs_by_sect={self.outputwcl[intgdefs.OW_OUTPUTS_BY_SECT]}",
WRAPPER_OUTPUT_PREFIX)
for exlabel, exlist in outexist.items():
if exlist:
if exlabel not in self.outputwcl[intgdefs.OW_OUTPUTS_BY_SECT]:
self.outputwcl[intgdefs.OW_OUTPUTS_BY_SECT][exlabel] = {}
if ekey not in self.outputwcl[
intgdefs.OW_OUTPUTS_BY_SECT][exlabel]:
self.outputwcl[
intgdefs.OW_OUTPUTS_BY_SECT][exlabel][ekey] = []
if miscutils.fwdebug_check(3, 'BASICWRAP_DEBUG'):
miscutils.fwdebug_print(
f"INFO: adding to sect={exlabel}: {exlist}",
WRAPPER_OUTPUT_PREFIX)
self.outputwcl[
intgdefs.OW_OUTPUTS_BY_SECT][exlabel][ekey].extend(exlist)
else:
miscutils.fwdebug_print(
f"WARN: 0 output files in exlist for {exlabel}",
WRAPPER_OUTPUT_PREFIX)
if miscutils.fwdebug_check(3, 'BASICWRAP_DEBUG'):
miscutils.fwdebug_print(
f"INFO: after adding outputs_by_sect={self.outputwcl[intgdefs.OW_OUTPUTS_BY_SECT]}",
WRAPPER_OUTPUT_PREFIX)
def perform_metadata_tasks(self, fullname, do_update, update_info):
""" Read metadata from file, updating file values """
if miscutils.fwdebug_check(3, 'FTMGMT_DEBUG'):
miscutils.fwdebug_print("INFO: beg")
# open file
#hdulist = pyfits.open(fullname, 'update')
primary_hdr = fits.getheader(fullname, 0)
prihdu = fits.PrimaryHDU(header=primary_hdr)
hdulist = fits.HDUList([prihdu])
# read metadata and call any special calc functions
metadata, _ = self._gather_metadata_file(fullname, hdulist=hdulist)
if miscutils.fwdebug_check(6, 'FTMGMT_DEBUG'):
miscutils.fwdebug_print(f"INFO: file={fullname}")
# call function to update headers
if do_update:
miscutils.fwdebug_print(
"WARN: cannot update a raw file's metadata")
# close file
hdulist.close()
if miscutils.fwdebug_check(3, 'FTMGMT_DEBUG'):
miscutils.fwdebug_print("INFO: end")
return metadata
def search_wcl_for_variables(wcl):
"""Find variables in given wcl.
"""
if miscutils.fwdebug_check(9, "PFWUTILS_DEBUG"):
miscutils.fwdebug_print("BEG")
usedvars = {}
for key, val in list(wcl.items()):
if isinstance(val, dict):
uvars = search_wcl_for_variables(val)
if uvars is not None:
usedvars.update(uvars)
elif isinstance(val, str):
viter = [
m.group(1) for m in re.finditer(r'(?i)\$\{([^}]+)\}', val)
]
for vstr in viter:
if ':' in vstr:
vstr = vstr.split(':')[0]
usedvars[vstr] = True
else:
if miscutils.fwdebug_check(9, "PFWUTILS_DEBUG"):
miscutils.fwdebug_print("Note: wcl is not string.")
miscutils.fwdebug_print(
"key = %s, type(val) = %s, val = '%s'" %
(key, type(val), val))
if miscutils.fwdebug_check(9, "PFWUTILS_DEBUG"):
miscutils.fwdebug_print("END")
return usedvars
def compress_files(listfullnames, compresssuffix, execname, argsorig, max_try_cnt=3, cleanup=True):
""" Compress given files """
if miscutils.fwdebug_check(3, 'PFWCOMPRESS_DEBUG'):
miscutils.fwdebug_print("BEG num files to compress = %s" % (len(listfullnames)))
results = {}
tot_bytes_before = 0
tot_bytes_after = 0
for fname in listfullnames:
errstr = None
cmd = None
fname_compressed = None
returncode = 1
try:
if not os.path.exists(fname):
errstr = "Error: Uncompressed file does not exist (%s)" % fname
returncode = 1
else:
tot_bytes_before += os.path.getsize(fname)
fname_compressed = fname + compresssuffix
# create command
args = copy.deepcopy(argsorig)
args = replfuncs.replace_vars_single(args,
{'__UCFILE__': fname,
'__CFILE__': fname_compressed},
None)
cmd = '%s %s' % (execname, args)
if miscutils.fwdebug_check(3, 'PFWCOMPRESS_DEBUG'):
miscutils.fwdebug_print("compression command: %s" % cmd)
returncode = run_compression_command(cmd, fname_compressed, max_try_cnt)
except IOError as exc:
errstr = "I/O error({0}): {1}".format(exc.errno, exc.strerror)
returncode = 1
if returncode != 0:
errstr = "Compression failed with exit code %i" % returncode
# check for partial compressed output and remove
if os.path.exists(fname_compressed):
miscutils.fwdebug_print("Compression failed. Removing compressed file.")
os.unlink(fname_compressed)
elif miscutils.convertBool(cleanup): # if successful, remove uncompressed if requested
os.unlink(fname)
if returncode == 0:
tot_bytes_after += os.path.getsize(fname_compressed)
else:
tot_bytes_after += os.path.getsize(fname)
# save exit code, cmd and new name
results[fname] = {'status': returncode,
'outname': fname_compressed,
'err': errstr,
'cmd': cmd}
if miscutils.fwdebug_check(3, 'PFWCOMPRESS_DEBUG'):
miscutils.fwdebug_print("END bytes %s => %s" % (tot_bytes_before, tot_bytes_after))
return (results, tot_bytes_before, tot_bytes_after)
def job2home(self, filelist, verify=False):
""" Transfer files from the job to the home archive
Parameters
----------
filelist : dict
Dictionary containing the file names and path information
Returns
-------
dict
The results
"""
if miscutils.fwdebug_check(3, "JOBFILEMVMT_DEBUG"):
miscutils.fwdebug_print("len(filelist)=%s" % len(filelist))
if miscutils.fwdebug_check(6, "JOBFILEMVMT_DEBUG"):
miscutils.fwdebug_print("filelist=%s" % filelist)
# if staging outside job, this function shouldn't be called
if self.home is None:
raise Exception(
"Home archive info is None. Should not be calling this function"
)
absfilelist = copy.deepcopy(filelist)
for finfo in absfilelist.values():
finfo['dst'] = self.home['root'] + '/' + finfo['dst']
if self.tstats is not None:
self.tstats.stat_beg_batch(
'job2home', 'job_scratch', self.home['name'],
self.__module__ + '.' + self.__class__.__name__)
(status, results) = disk_utils_local.copyfiles(absfilelist,
self.tstats, verify)
if self.tstats is not None:
self.tstats.stat_end_batch(status)
return results
def read_fullnames_from_listfile(listfile, linefmt, colstr):
""" Read a list file returning fullnames from the list """
if miscutils.fwdebug_check(3, 'INTGMISC_DEBUG'):
miscutils.fwdebug_print('colstr=%s' % colstr)
columns = convert_col_string_to_list(colstr, False)
if miscutils.fwdebug_check(3, 'INTGMISC_DEBUG'):
miscutils.fwdebug_print('columns=%s' % columns)
fullnames = {}
pos2fsect = {}
for pos in range(0, len(columns)):
lcol = columns[pos].lower()
if lcol.endswith('.fullname'):
filesect = lcol[:-9]
pos2fsect[pos] = filesect
fullnames[filesect] = []
# else a data column instead of a filename
if miscutils.fwdebug_check(3, 'INTGMISC_DEBUG'):
miscutils.fwdebug_print('pos2fsect=%s' % pos2fsect)
if linefmt == 'config' or linefmt == 'wcl':
miscutils.fwdie(
'Error: wcl list format not currently supported (%s)' % listfile,
1)
else:
with open(listfile, 'r') as listfh:
for line in listfh:
line = line.strip()
# convert line into python list
lineinfo = []
if linefmt == 'textcsv':
lineinfo = miscutils.fwsplit(line, ',')
elif linefmt == 'texttab':
lineinfo = miscutils.fwsplit(line, '\t')
elif linefmt == 'textsp':
lineinfo = miscutils.fwsplit(line, ' ')
else:
miscutils.fwdie('Error: unknown linefmt (%s)' % linefmt,
1)
# save each fullname in line
for pos in pos2fsect:
# use common routine to parse actual fullname (e.g., remove [0])
parsemask = miscutils.CU_PARSE_PATH | miscutils.CU_PARSE_FILENAME | \
miscutils.CU_PARSE_COMPRESSION
(path, filename, compression) = miscutils.parse_fullname(
lineinfo[pos], parsemask)
fname = "%s/%s" % (path, filename)
if compression is not None:
fname += compression
fullnames[pos2fsect[pos]].append(fname)
if miscutils.fwdebug_check(6, 'INTGMISC_DEBUG'):
miscutils.fwdebug_print('fullnames = %s' % fullnames)
return fullnames
def ingest_contents(self, listfullnames, **kwargs):
""" Ingest certain content into a non-metadata table """
assert isinstance(listfullnames, list)
for fname in listfullnames:
miscutils.fwdebug_print("********************* %s" % fname)
numrows = dfiutils.datafile_ingest_main(self.dbh, self.filetype,
fname, self.tablename,
self.didatadefs)
if numrows in [None, 0]:
miscutils.fwdebug_print(
f"WARN: 0 rows ingested from {fname} for table {self.tablename}"
)
elif miscutils.fwdebug_check(1, 'FTMGMT_DEBUG'):
miscutils.fwdebug_print(
f"INFO: {numrows} rows ingested from {fname} for table {self.tablename}"
)
numrows = dfiutils.datafile_ingest_main(self.dbh, self.filetype2,
fname, self.tablename2,
self.didatadefs2)
if numrows in [None, 0]:
miscutils.fwdebug_print(
f"WARN: 0 rows ingested from {fname} for table {self.tablename2}"
)
elif miscutils.fwdebug_check(1, 'FTMGMT_DEBUG'):
miscutils.fwdebug_print(
f"INFO: {numrows} rows ingested from {fname} for table {self.tablename2}"
)
def check_proxy(config):
"""Check for proxy.
Check if any block will submit to remote machine needing proxy,
if so check for proxy.
"""
if miscutils.fwdebug_check(3, 'PFWSUBMIT_DEBUG'):
miscutils.fwdebug_print("Beg")
config.reset_blknum()
blocklist = miscutils.fwsplit(config[pfwdefs.SW_BLOCKLIST].lower(), ',')
for blockname in blocklist:
if miscutils.fwdebug_check(3, 'PFWSUBMIT_DEBUG'):
miscutils.fwdebug_print("Checking block %s..." % (blockname))
config.set_block_info()
(exists, check_proxy) = config.search(pfwdefs.SW_CHECK_PROXY,
{intgdefs.REPLACE_VARS: True})
if exists and miscutils.convertBool(check_proxy):
timeleft = pfwcondor.get_grid_proxy_timeleft()
assert timeleft > 0
if timeleft < 21600: # 5 * 60 * 60
print("Warning: Proxy expires in less than 5 hours")
break
config.inc_blknum()
config.reset_blknum()
if miscutils.fwdebug_check(3, 'PFWSUBMIT_DEBUG'):
miscutils.fwdebug_print("End")
def insert_exec(self, wcl, sect):
"""Insert row into pfw_exec.
"""
if miscutils.fwdebug_check(3, 'PFWDB_DEBUG'):
miscutils.fwdebug_print(sect)
miscutils.fwdebug_print(wcl[sect])
row = {}
row['pfw_attempt_id'] = wcl['pfw_attempt_id']
row['pfw_block_task_id'] = wcl['task_id']['block']
row['pfw_job_task_id'] = wcl['task_id']['job']
row['pfw_wrapper_task_id'] = wcl['task_id']['wrapper']
row['execnum'] = wcl[sect]['execnum']
row['name'] = wcl[sect]['execname']
row['task_id'] = self.create_task(
name=sect,
info_table='pfw_exec',
parent_task_id=wcl['task_id']['wrapper'],
root_task_id=int(wcl['task_id']['attempt']),
label=wcl[sect]['execname'],
do_commit=True)
if 'version' in wcl[sect] and wcl[sect]['version'] is not None:
row['version'] = wcl[sect]['version']
self.insert_PFW_row('PFW_EXEC', row)
if miscutils.fwdebug_check(3, 'PFWDB_DEBUG'):
miscutils.fwdebug_print("end")
return row['task_id']
def insert_data_query(self, wcl, modname, datatype, dataname, execname,
cmdargs, version):
"""Insert row into pfw_data_query table.
"""
if miscutils.fwdebug_check(3, 'PFWDB_DEBUG'):
miscutils.fwdebug_print("BEG")
parent_tid = wcl['task_id']['begblock']
row = {}
row['pfw_attempt_id'] = wcl['pfw_attempt_id']
row['pfw_block_task_id'] = wcl['task_id']['block'][wcl['blknum']]
row['modname'] = modname
row['datatype'] = datatype # file, list
row['dataname'] = dataname
row['task_id'] = self.create_task(name='dataquery',
info_table='PFW_DATA_QUERY',
parent_task_id=parent_tid,
root_task_id=int(
wcl['task_id']['attempt']),
label=None,
do_begin=True,
do_commit=True)
row['execname'] = os.path.basename(execname)
row['cmdargs'] = cmdargs
row['version'] = version
self.insert_PFW_row('PFW_DATA_QUERY', row)
if miscutils.fwdebug_check(3, 'PFWDB_DEBUG'):
miscutils.fwdebug_print("END")
return row['task_id']
def update_job_info(self, wcl, jobnum, jobinfo):
"""Update job information.
Updates row in pfw_job with information gathered post job from condor
log.
"""
if miscutils.fwdebug_check(1, 'PFWDB_DEBUG'):
miscutils.fwdebug_print("Updating job information post job (%s)" %
jobnum)
if miscutils.fwdebug_check(3, 'PFWDB_DEBUG'):
miscutils.fwdebug_print("jobinfo=%s" % jobinfo)
wherevals = {}
wherevals['task_id'] = wcl['task_id']['job'][jobnum]
if miscutils.fwdebug_check(3, 'PFWDB_DEBUG'):
miscutils.fwdebug_print("wherevals = %s" % (wherevals))
if len(jobinfo) > 0:
self.update_PFW_row('PFW_JOB', jobinfo, wherevals)
else:
if miscutils.fwdebug_check(3, 'PFWDB_DEBUG'):
miscutils.fwdebug_print("Found 0 values to update (%s)" %
(wherevals))
if miscutils.fwdebug_check(6, 'PFWDB_DEBUG'):
miscutils.fwdebug_print("\tjobnum = %s, jobinfo = %s" %
(jobnum, jobinfo))
def stat_beg_batch(self, transfer_name, src, dst, transclass=None):
""" Starting a batch transfer between src and dst (archive or job scratch) """
if miscutils.fwdebug_check(3, 'TRANSFERSTATS_DEBUG'):
miscutils.fwdebug_print(f"beg {transfer_name} {src} {dst} {transclass}")
self.currvals['transfer_name'] = transfer_name
self.currvals['src'] = src
self.currvals['dst'] = dst
self.currvals['batch_task_id'] = self.create_task(name=transfer_name,
info_table='transfer_batch',
parent_task_id=self.parent_task_id,
root_task_id=self.root_task_id,
label=None,
do_begin=True,
do_commit=False)
row = {'src': src, 'dst': dst,
'transfer_class': transclass,
'parent_task_id': self.parent_task_id,
'task_id': self.currvals['batch_task_id']}
self.basic_insert_row('transfer_batch', row)
self.commit()
if miscutils.fwdebug_check(3, 'TRANSFERSTATS_DEBUG'):
miscutils.fwdebug_print("end")
return self.currvals['batch_task_id']
def stat_beg_file(self, filename):
""" Insert a row into a file transfer stats table (and task table) and commit """
self.currvals['numfiles'] += 1
self.currvals['file_task_id'] = -1
if self.transfer_stats_per_file:
if miscutils.fwdebug_check(3, 'TRANSFERSTATS_DEBUG'):
miscutils.fwdebug_print(f"beg - {filename}")
if self.currvals['batch_task_id'] is None:
raise Exception('Cannot call this function without prior calling stat_beg_batch')
row = {'filename': filename}
row['task_id'] = self.create_task(name='transfer_file',
info_table='transfer_file',
parent_task_id=self.currvals['batch_task_id'],
root_task_id=self.root_task_id,
label=None,
do_begin=True,
do_commit=False)
row['batch_task_id'] = self.currvals['batch_task_id']
self.basic_insert_row('transfer_file', row)
self.commit()
self.currvals['file_task_id'] = row['task_id']
if miscutils.fwdebug_check(3, 'TRANSFERSTATS_DEBUG'):
miscutils.fwdebug_print(f"end - file_task_id = {self.currvals['file_task_id']}")
return self.currvals['file_task_id']
def search_wcl_for_variables(cls, wcl):
""" Search the wcl for variables """
if miscutils.fwdebug_check(9, "WCL_DEBUG"):
miscutils.fwdebug_print("BEG")
usedvars = {}
for key, val in wcl.items():
if isinstance(val, (dict, collections.OrderedDict)):
uvars = cls.search_wcl_for_variables(val)
if uvars:
usedvars.update(uvars)
elif isinstance(val, str):
viter = [
m.group(1) for m in re.finditer(r'(?i)\$\{([^}]+)\}', val)
]
for vstr in viter:
if ':' in vstr:
vstr = vstr.split(':')[0]
usedvars[vstr] = True
else:
if miscutils.fwdebug_check(9, "WCL_DEBUG"):
miscutils.fwdebug_print("Note: wcl is not string.")
miscutils.fwdebug_print(
f"\tkey = {key}, type(val) = {type(val)}, val = '{val}'"
)
if miscutils.fwdebug_check(9, "WCL_DEBUG"):
miscutils.fwdebug_print("END")
return usedvars
def get_file_fullnames(sect, filewcl, fullwcl):
""" Get the full name of the files in the specified section.
Parameters
----------
sect : str
The WCL section to use
filewcl : WCL
The WCl to use
fullwcl : WCL
The full WCL, used to generate the full names
Returns
-------
set
The full file names
"""
sectkeys = sect.split('.')
sectname = sectkeys[1]
if miscutils.fwdebug_check(3, 'INTGMISC_DEBUG'):
miscutils.fwdebug_print("INFO: Beg sectname=%s" % sectname)
fnames = []
if sectname in filewcl:
filesect = filewcl[sectname]
if 'fullname' in filesect:
fnames = replfuncs.replace_vars(filesect['fullname'], fullwcl)[0]
fnames = miscutils.fwsplit(fnames, ',')
if miscutils.fwdebug_check(3, 'INTGMISC_DEBUG'):
miscutils.fwdebug_print("INFO: fullname = %s" % fnames)
return set(fnames)
def getfull(self, key, opts=None, default=None):
""" Return with variables replaced and expanded if string(s) """
if miscutils.fwdebug_check(9, "WCL_DEBUG"):
miscutils.fwdebug_print(f"BEG - key={key}")
miscutils.fwdebug_print(f"default - {default}")
miscutils.fwdebug_print(f"opts - {opts}")
(found, value) = self.search(key, opts)
if not found:
value = default
elif isinstance(value, str):
if opts is None:
newopts = {'expand': True, intgdefs.REPLACE_VARS: True}
else:
newopts = copy.deepcopy(opts)
if intgdefs.REPLACE_VARS not in newopts or \
miscutils.convertBool(newopts[intgdefs.REPLACE_VARS]):
newopts['expand'] = True
if miscutils.fwdebug_check(9, "WCL_DEBUG"):
miscutils.fwdebug_print(
f"calling replace_vars value={value}, opts={newopts}")
(value, _) = replfuncs.replace_vars(value, self, newopts)
if len(value) == 1:
value = value[0]
return value
def get_exec_sections(wcl, prefix):
""" Returns exec sections appearing in given wcl
Parameters
----------
wcl : WCL
The WCL object to look in.
prefix : str
The exec prefix to look for.
Returns
-------
dict
Dictionary of the found exec section names and their contents.
"""
execs = {}
for key, val in wcl.items():
if miscutils.fwdebug_check(3, "DEBUG"):
miscutils.fwdebug_print("\tsearching for exec prefix in %s" % key)
if re.search(r"^%s\d+$" % prefix, key):
if miscutils.fwdebug_check(4, "DEBUG"):
miscutils.fwdebug_print("\tFound exec prefex %s" % key)
execs[key] = val
return execs
def target2job(self, filelist):
""" Transfer files from the target archive
Parameters
----------
filelist : dict
Dictionary containing the file names and path information
Returns
-------
dict
The results
"""
if miscutils.fwdebug_check(3, "JOBFILEMVMT_DEBUG"):
miscutils.fwdebug_print("len(filelist)=%s" % len(filelist))
if miscutils.fwdebug_check(6, "JOBFILEMVMT_DEBUG"):
miscutils.fwdebug_print("filelist=%s" % filelist)
if self.target is None:
raise Exception(
"Target archive info is None. Should not be calling this function"
)
absfilelist = copy.deepcopy(filelist)
for finfo in absfilelist.values():
finfo['src'] = self.target['root'] + '/' + finfo['src']
if self.tstats is not None:
self.tstats.stat_beg_batch(
'target2job', self.target['name'], 'job_scratch',
self.__module__ + '.' + self.__class__.__name__)
(status, results) = disk_utils_local.copyfiles(absfilelist,
self.tstats)
if self.tstats is not None:
self.tstats.stat_end_batch(status)
return results
def get_single_file_disk_info(fname, save_md5sum=False, archive_root=None):
""" Method to get disk info for a single file
"""
if miscutils.fwdebug_check(3, "DISK_UTILS_LOCAL_DEBUG"):
miscutils.fwdebug_print(f"fname={fname}, save_md5sum={save_md5sum}, archive_root={archive_root}")
parsemask = miscutils.CU_PARSE_PATH | miscutils.CU_PARSE_FILENAME | miscutils.CU_PARSE_COMPRESSION
(path, filename, compress) = miscutils.parse_fullname(fname, parsemask)
if miscutils.fwdebug_check(3, "DISK_UTILS_LOCAL_DEBUG"):
miscutils.fwdebug_print(f"path={path}, filename={filename}, compress={compress}")
fdict = {'filename' : filename,
'compression': compress,
'path': path,
'filesize': os.path.getsize(fname)
}
if save_md5sum:
fdict['md5sum'] = get_md5sum_file(fname)
if archive_root and path.startswith('/'):
fdict['relpath'] = path[len(archive_root)+1:]
if compress is None:
compext = ""
else:
compext = compress
fdict['rel_filename'] = f"{fdict['relpath']}/{filename}{compext}"
return fdict
def combine_lists_files(self, modulename):
""" Return python list of file and file list objects """
if miscutils.fwdebug_check(3, 'PFWCONFIG_DEBUG'):
miscutils.fwdebug_print("BEG")
moduledict = self[pfwdefs.SW_MODULESECT][modulename]
# create python list of files and lists for this module
dataset = []
if pfwdefs.SW_LISTSECT in moduledict and moduledict[pfwdefs.SW_LISTSECT]:
if 'list_order' in moduledict:
listorder = moduledict['list_order'].replace(' ', '').split(',')
else:
listorder = moduledict[pfwdefs.SW_LISTSECT].keys()
for key in listorder:
dataset.append(('list-%s' % key, moduledict[pfwdefs.SW_LISTSECT][key]))
elif miscutils.fwdebug_check(3, 'PFWCONFIG_DEBUG'):
miscutils.fwdebug_print("no lists")
if pfwdefs.SW_FILESECT in moduledict and moduledict[pfwdefs.SW_FILESECT]:
for key, val in moduledict[pfwdefs.SW_FILESECT].items():
dataset.append(('file-%s' % key, val))
elif miscutils.fwdebug_check(3, 'PFWCONFIG_DEBUG'):
miscutils.fwdebug_print("no files")
if miscutils.fwdebug_check(3, 'PFWCONFIG_DEBUG'):
miscutils.fwdebug_print("END")
return dataset
def search_wcl_for_variables(cls, wcl):
""" Search the wcl for variables """
if miscutils.fwdebug_check(9, "WCL_DEBUG"):
miscutils.fwdebug_print("BEG")
usedvars = {}
for key, val in wcl.items():
if type(val) is dict or type(val) is OrderedDict:
uvars = cls.search_wcl_for_variables(val)
if uvars is not None:
usedvars.update(uvars)
elif type(val) is str:
viter = [
m.group(1) for m in re.finditer(r'(?i)\$\{([^}]+)\}', val)
]
for vstr in viter:
if ':' in vstr:
vstr = vstr.split(':')[0]
usedvars[vstr] = True
else:
if miscutils.fwdebug_check(9, "WCL_DEBUG"):
miscutils.fwdebug_print("Note: wcl is not string.")
miscutils.fwdebug_print("\tkey = %s, type(val) = %s, val = '%s'" % \
(key, type(val), val))
if miscutils.fwdebug_check(9, "WCL_DEBUG"):
miscutils.fwdebug_print("END")
return usedvars
def set(self, key, val):
""" Sets value of key in wcl, follows section notation
Parameters
----------
key : str
The key to set the value for.
val : str
The value to set.
"""
if miscutils.fwdebug_check(9, "WCL_DEBUG"):
miscutils.fwdebug_print("BEG key=%s, val=%s" % (key, val))
subkeys = key.split('.')
valkey = subkeys.pop()
wcldict = self
for k in subkeys:
wcldict = OrderedDict.__getitem__(wcldict, k)
OrderedDict.__setitem__(wcldict, valkey, val)
if miscutils.fwdebug_check(9, "WCL_DEBUG"):
miscutils.fwdebug_print("END")
def list_missing_contents(filemgmt, ftype, filelist):
""" Return list of files from given set which still need contents ingested """
# filelist = list of file dicts
if miscutils.fwdebug_check(6, "REGISTER_FILES_DEBUG"):
miscutils.fwdebug_print(f"filelist={filelist}")
print("\tChecking which files still need contents ingested", flush=True)
starttime = time.time()
results = filemgmt.has_contents_ingested(ftype, filelist)
endtime = time.time()
print(f"({endtime - starttime:0.2f} secs)", flush=True)
# no metadata if results[name] == False
misslist = [fname for fname in results if not results[fname]]
print(
f"\t\t{len(filelist) - len(misslist):0d} file(s) already have content ingested",
flush=True)
print(f"\t\t{len(misslist):0d} file(s) still to have content ingested",
flush=True)
if miscutils.fwdebug_check(6, "REGISTER_FILES_DEBUG"):
miscutils.fwdebug_print(f"misslist={misslist}")
return misslist
def determine_status(self):
""" Check all task status to determine wrapper status """
status = 0
execs = intgmisc.get_exec_sections(self.inputwcl,
intgdefs.IW_EXEC_PREFIX)
if miscutils.fwdebug_check(6, 'BASICWRAP_DEBUG'):
miscutils.fwdebug_print(f"INFO: exec sections = {execs}",
WRAPPER_OUTPUT_PREFIX)
for ekey in sorted(execs.keys()):
if ekey in self.outputwcl:
if 'task_info' in self.outputwcl[ekey]:
for taskd in self.outputwcl[ekey]['task_info'].values():
if 'status' in taskd:
if taskd['status'] != 0:
status = taskd['status']
else:
if miscutils.fwdebug_check(3, "BASICWRAP_DEBUG"):
miscutils.fwdebug_print(
f"WARN: Missing status in outputwcl task_info for {ekey}",
WRAPPER_OUTPUT_PREFIX)
status = 1
else:
if miscutils.fwdebug_check(3, "BASICWRAP_DEBUG"):
miscutils.fwdebug_print(
f"WARN: Missing task_info in outputwcl for {ekey}",
WRAPPER_OUTPUT_PREFIX)
status = 1
else:
status = 1
return status
def home2job(self, filelist):
""" From inside job, pull files from home archive to job scratch directory
Parameters
----------
filelist : dict
Dictionary containing the file names and path information
Returns
-------
dict of the results
"""
if miscutils.fwdebug_check(3, "JOBFILEMVMT_DEBUG"):
miscutils.fwdebug_print("len(filelist)=%s" % len(filelist))
if miscutils.fwdebug_check(6, "JOBFILEMVMT_DEBUG"):
miscutils.fwdebug_print("filelist=%s" % filelist)
# if staging outside job, this function shouldn't be called
if self.home is None:
raise Exception("Home archive info is None. Should not be calling this function")
absfilelist = copy.deepcopy(filelist)
for finfo in absfilelist.values():
finfo['src'] = self.home['root_http'] + '/' + finfo['src']
if self.tstats is not None:
self.tstats.stat_beg_batch('home2job', self.home['name'], 'job_scratch',
self.__module__ + '.' + self.__class__.__name__)
(status, results) = self.HU.copyfiles(absfilelist, self.tstats)
if self.tstats is not None:
self.tstats.stat_end_batch(status)
return results
def mass_register_files(self, ftype, filedata):
self.dynam_load_ftmgmt(ftype)
badfiles = set()
for fname, data in filedata.items():
try:
has_metadata = self.has_metadata_ingested(ftype, fname)
if not has_metadata:
self.save_file_info(data['diskinfo'], data['metadata'])
elif miscutils.fwdebug_check(3, 'FILEMGMT_DEBUG'):
miscutils.fwdebug_print(f"INFO: {fname} already has metadata ingested")
except:
miscutils.fwdebug_print(f"\n\nError: Problem gathering metadata for file {fname}")
traceback.print_exc(1, sys.stdout)
badfiles.add(fname)
try:
has_contents = self.ftmgmt.has_contents_ingested([fname])
if not has_contents[fname]:
self.ftmgmt.ingest_contents([fname])
elif miscutils.fwdebug_check(3, 'FILEMGMT_DEBUG'):
miscutils.fwdebug_print(f"INFO: {fname} already has contents ingested")
except:
miscutils.fwdebug_print(f"\n\nError: Problem gathering metadata for file {fname}")
traceback.print_exc(1, sys.stdout)
badfiles.add(fname)
return list(badfiles)
def process_file(self, outfile, filedef, metadef):
""" Steps """
if miscutils.fwdebug_check(3, 'FM_METAUTILS_DEBUG'):
miscutils.fwdebug_print("INFO: begin")
miscutils.fwdebug_print(f"INFO: outfile = {outfile}")
if miscutils.fwdebug_check(6, 'FM_METAUTILS_DEBUG'):
miscutils.fwdebug_print(f"INFO: filedef = {filedef}")
# open file
hdulist = fits.open(outfile, 'update')
# update headers
updatedef = None
if 'update' in metadef:
updatedef = metadef['update']
# call even if no update in case special wrapper has overloaded func
self.update_headers_file(hdulist, filedef, updatedef)
# read metadata
metadata = self.gather_metadata_file(hdulist, metadef)
# close file
hdulist.close()
if miscutils.fwdebug_check(3, 'FM_METAUTILS_DEBUG'):
miscutils.fwdebug_print("INFO: end")
return metadata
def _gather_metadata_file(self, fullname, **kwargs):
""" Gather metadata for a single file """
if miscutils.fwdebug_check(3, 'FTMGMT_DEBUG'):
miscutils.fwdebug_print(f"INFO: beg file={fullname}")
metadata = FtMgmtGeneric._gather_metadata_file(self, fullname,
**kwargs)
# need nite for the archive path
with open(fullname, 'r') as jsonfh:
line = jsonfh.readline()
linedata = json.loads(line)
expcnt = 0
while expcnt < len(linedata['exposures']) and \
'date' not in linedata['exposures'][expcnt]:
expcnt += 1
if expcnt >= len(linedata['exposures']):
raise KeyError(
'Could not find date value for any exposure in manifest')
datestr = linedata['exposures'][expcnt]['date']
metadata['nite'] = misctime.convert_utc_str_to_nite(datestr)
if miscutils.fwdebug_check(3, 'FTMGMT_DEBUG'):
miscutils.fwdebug_print("INFO: end")
return metadata
def get_job_info(self, wherevals):
""" Get job information """
whclause = []
for c in wherevals.keys():
whclause.append(f"{c}={self.get_named_bind_string(c)}")
sql = f"select j.jobkeys as jobkeys,j.jobnum as jobnum, j.expect_num_wrap as expect_num_wrap, j.task_id as task_id, j.pfw_block_task_id as pfw_block_task_id, t.status as status, t.start_time as start_time, t.end_time as end_time from pfw_job j, task t where t.id=j.task_id and {' and '.join(whclause)}"
#sql = "select j.*,t.* from pfw_job j, task t where t.id=j.task_id and %s" % (' and '.join(whclause))
if miscutils.fwdebug_check(3, 'PFWDB_DEBUG'):
miscutils.fwdebug_print(f"sql> {sql}")
if miscutils.fwdebug_check(3, 'PFWDB_DEBUG'):
miscutils.fwdebug_print(f"params> {wherevals}")
curs = self.cursor()
curs.execute(sql, wherevals)
desc = [d[0].lower() for d in curs.description]
jobinfo = {}
get_messages = []
for line in curs:
d = dict(zip(desc, line))
d['message'] = []
if d['status'] != pfwdefs.PF_EXIT_SUCCESS:
get_messages.append(d['task_id'])
jobinfo[d['task_id']] = d
if not get_messages:
qdbh = qcfdb.QCFDB(connection=self)
qcmsg = qdbh.get_all_qcf_messages_by_task_id(get_messages, level=3)
for tid, val in qcmsg.items():
jobinfo[tid]['message'] = val
return jobinfo