def compress_files(listfullnames, compresssuffix, execname, argsorig, max_try_cnt=3, cleanup=True):
""" Compress given files """
if miscutils.fwdebug_check(3, 'PFWCOMPRESS_DEBUG'):
miscutils.fwdebug_print("BEG num files to compress = %s" % (len(listfullnames)))
results = {}
tot_bytes_before = 0
tot_bytes_after = 0
for fname in listfullnames:
errstr = None
cmd = None
fname_compressed = None
returncode = 1
try:
if not os.path.exists(fname):
errstr = "Error: Uncompressed file does not exist (%s)" % fname
returncode = 1
else:
tot_bytes_before += os.path.getsize(fname)
fname_compressed = fname + compresssuffix
# create command
args = copy.deepcopy(argsorig)
args = replfuncs.replace_vars_single(args,
{'__UCFILE__': fname,
'__CFILE__': fname_compressed},
None)
cmd = '%s %s' % (execname, args)
if miscutils.fwdebug_check(3, 'PFWCOMPRESS_DEBUG'):
miscutils.fwdebug_print("compression command: %s" % cmd)
returncode = run_compression_command(cmd, fname_compressed, max_try_cnt)
except IOError as exc:
errstr = "I/O error({0}): {1}".format(exc.errno, exc.strerror)
returncode = 1
if returncode != 0:
errstr = "Compression failed with exit code %i" % returncode
# check for partial compressed output and remove
if os.path.exists(fname_compressed):
miscutils.fwdebug_print("Compression failed. Removing compressed file.")
os.unlink(fname_compressed)
elif miscutils.convertBool(cleanup): # if successful, remove uncompressed if requested
os.unlink(fname)
if returncode == 0:
tot_bytes_after += os.path.getsize(fname_compressed)
else:
tot_bytes_after += os.path.getsize(fname)
# save exit code, cmd and new name
results[fname] = {'status': returncode,
'outname': fname_compressed,
'err': errstr,
'cmd': cmd}
if miscutils.fwdebug_check(3, 'PFWCOMPRESS_DEBUG'):
miscutils.fwdebug_print("END bytes %s => %s" % (tot_bytes_before, tot_bytes_after))
return (results, tot_bytes_before, tot_bytes_after)
def stagefiles(self, opts=None):
"""Return whether to save stage files to target archive.
"""
retval = True
notarget_exists, notarget = self.search(pfwdefs.PF_DRYRUN, opts)
if notarget_exists and miscutils.convertBool(notarget):
print("Do not stage file due to dry run\n")
retval = False
else:
stagefiles_exists, stagefiles = self.search(
pfwdefs.STAGE_FILES, opts)
if stagefiles_exists:
#print "checking stagefiles (%s)" % stagefiles
results = replfuncs.replace_vars_single(stagefiles, self, opts)
retval = miscutils.convertBool(results)
#print "after interpolation stagefiles (%s)" % retval
else:
envkey = 'DESDM_%s' % pfwdefs.STAGE_FILES.upper()
if envkey in os.environ and not miscutils.convertBool(
os.environ[envkey]):
retval = False
#print "stagefiles retval = %s" % retval
return retval
def get_filepath(self, pathtype, dirpat=None, searchopts=None):
""" Return filepath based upon given pathtype and directory pattern name """
# get filename pattern from global settings:
if not dirpat:
(found, dirpat) = self.search(pfwdefs.DIRPAT, searchopts)
if not found:
miscutils.fwdie("Error: Could not find dirpat", pfwdefs.PF_EXIT_FAILURE)
if dirpat in self[pfwdefs.DIRPATSECT]:
filepathpat = self[pfwdefs.DIRPATSECT][dirpat][pathtype]
else:
miscutils.fwdie("Error: Could not find pattern %s in directory patterns" % \
dirpat, pfwdefs.PF_EXIT_FAILURE)
results = replfuncs.replace_vars_single(filepathpat, self, searchopts)
return results
def savefiles(self, opts=None):
""" Return whether to save files from job """
retval = True
savefiles_exists, savefiles = self.search(pfwdefs.SAVE_FILE_ARCHIVE, opts)
if savefiles_exists:
if miscutils.fwdebug_check(3, 'PFWCONFIG_DEBUG'):
miscutils.fwdebug_print("checking savefiles (%s)" % savefiles)
results = replfuncs.replace_vars_single(savefiles, self, opts)
retval = miscutils.convertBool(results)
if miscutils.fwdebug_check(3, 'PFWCONFIG_DEBUG'):
miscutils.fwdebug_print("after interpolation savefiles (%s)" % retval)
else:
envkey = 'DESDM_%s' % pfwdefs.SAVE_FILE_ARCHIVE.upper()
if envkey in os.environ and not miscutils.convertBool(os.environ[envkey]):
retval = False
if miscutils.fwdebug_check(3, 'PFWCONFIG_DEBUG'):
miscutils.fwdebug_print("savefiles retval = %s" % retval)
return retval
def create_master_list(config, configfile, modname, moddict,
search_name, search_dict, search_type):
"""Create master data list for a module's list or file def.
"""
miscutils.fwdebug_print("BEG")
if 'qouttype' in search_dict:
qouttype = search_dict['qouttype']
else:
qouttype = intgdefs.DEFAULT_QUERY_OUTPUT_FORMAT
qoutfile = config.get_filename('qoutput',
{pfwdefs.PF_CURRVALS: {'modulename': modname,
'searchname': search_name,
'suffix': qouttype}})
qlog = config.get_filename('qoutput',
{pfwdefs.PF_CURRVALS: {'modulename': modname,
'searchname': search_name,
'suffix': 'out'}})
prog = None
if 'exec' in search_dict:
prog = search_dict['exec']
if 'args' not in search_dict:
print("\t\tWarning: %s in module %s does not have args defined\n" % \
(search_name, modname))
args = ""
else:
args = search_dict['args']
elif 'query_fields' in search_dict:
if 'processingfw_dir' in config:
dirgenquery = config['processingfw_dir']
elif 'PROCESSINGFW_DIR' in os.environ:
dirgenquery = os.environ['PROCESSINGFW_DIR']
else:
miscutils.fwdie("Error: Could not determine base path for genquerydb.py",
pfwdefs.PF_EXIT_FAILURE)
prog = "%s/libexec/genquerydb.py" % (dirgenquery)
args = "--qoutfile %s --qouttype %s --config %s --module %s --search %s" % \
(qoutfile, qouttype, configfile, modname, search_name)
if not prog:
print("\tWarning: %s in module %s does not have exec or %s defined" % \
(search_name, modname, pfwdefs.SW_QUERYFIELDS))
return
search_dict['qoutfile'] = qoutfile
search_dict['qlog'] = qlog
prog = replfuncs.replace_vars_single(prog, config,
{pfwdefs.PF_CURRVALS: {pfwdefs.SW_MODULESECT: modname},
'searchobj': search_dict})
# handle both outputxml and outputfile args
args = replfuncs.replace_vars_single(args, config,
{pfwdefs.PF_CURRVALS: {pfwdefs.SW_MODULESECT: modname,
'outputxml': qoutfile,
'outputfile': qoutfile,
'qoutfile': qoutfile},
#intgdefs.REPLACE_VARS: True,
'searchobj': search_dict})
# get version for query code
query_version = None
if prog in config[pfwdefs.SW_EXEC_DEF]:
query_version = pfwutils.get_version(prog, config[pfwdefs.SW_EXEC_DEF])
if search_type == pfwdefs.SW_LISTSECT:
datatype = 'L'
elif search_type == pfwdefs.SW_FILESECT:
datatype = 'F'
else:
datatype = search_type[0].upper()
# call code
query_tid = None
if miscutils.convertBool(config.getfull(pfwdefs.PF_USE_DB_OUT)):
pfw_dbh = pfwdb.PFWDB()
query_tid = pfw_dbh.insert_data_query(config, modname, datatype, search_name,
prog, args, query_version)
#pfw_dbh.close()
else:
pfw_dbh = None
cwd = os.getcwd()
print("\t\tCalling code to create master list for obj %s in module %s" % \
(search_name, modname))
print("\t\t", prog, args)
print("\t\tSee output in %s/%s" % (cwd, qlog))
print("\t\tSee master list will be in %s/%s" % (cwd, qoutfile))
print("\t\tCreating master list - start ", time.time())
cmd = "%s %s" % (prog, args)
exitcode = None
try:
exitcode = pfwutils.run_cmd_qcf(cmd, qlog, query_tid, os.path.basename(prog),
config.getfull(pfwdefs.PF_USE_QCF), pfw_dbh,
config['pfw_attempt_id'])
#exitcode = pfwutils.run_cmd_qcf(cmd, qlog, query_tid, os.path.basename(prog),
# 5000, config.getfull(pfwdefs.PF_USE_QCF))
except:
print("******************************")
print("Error: ")
(extype, exvalue, trback) = sys.exc_info()
print("******************************")
traceback.print_exception(extype, exvalue, trback, file=sys.stdout)
exitcode = pfwdefs.PF_EXIT_FAILURE
print("\t\tCreating master list - end ", time.time())
sys.stdout.flush()
if miscutils.convertBool(config.getfull(pfwdefs.PF_USE_DB_OUT)):
pfw_dbh = pfwdb.PFWDB()
pfw_dbh.end_task(query_tid, exitcode, True)
pfw_dbh.close()
if exitcode != 0:
miscutils.fwdie("Error: problem creating master list (exitcode = %s)" %
(exitcode), exitcode)
miscutils.fwdebug_print("END")
def read(self, in_file=None, cmdline=False, filename='stdin'):
""" Reads WCL text from an open file object and converts it to the internal
content of this object.
Parameters
----------
in_file : file
The file handle to use for reading.
cmdline : bool, optional
Whether the read item is a command line, used internally. Default
is ``False``.
filename : str, optional
The name of the input file. Default is 'stdin'
Raises
------
SyntaxError
If there is a syntax error in the file.
"""
curr = self
stack = [] # to keep track of current sub-dictionary
stackkeys = ['__topwcl__'] # to keep track of current section key
stack.append(curr) #
line = in_file.readline()
linecnt = 1
while line:
line = line.strip()
while line.endswith('\\'):
linecnt += 1
line = line[:-1] + in_file.readline().strip()
# delete comments
line = line.split('#')[0]
# skip comment line or empty line
if re.search(r"\S", line):
# handle includes
patmatch = re.search(r"<<include (\S+)>>", line)
if patmatch is not None:
# replace wcl vars in filename
filename2 = replfuncs.replace_vars_single(
patmatch.group(1), self, None)
# expand ~ and env vars in filename
filename2 = os.path.expandvars(
os.path.expanduser(filename2))
wclobj2 = WCL()
with open(filename2, "r") as wclfh:
wclobj2.read(wclfh, cmdline, filename2)
self.update(wclobj2)
line = in_file.readline()
linecnt += 1
continue
# handle calls to external functions to get more information usually from db
patmatch = re.search(r"<<inclfunc ([^>]+)>>", line)
if patmatch is not None:
if miscutils.fwdebug_check(9, "WCL_DEBUG"):
miscutils.fwdebug_print("patmatch=%s" %
patmatch.group(0))
funcmatch = re.match(r'([^(]+)\(([^)]+)\)',
patmatch.group(1))
if funcmatch:
if miscutils.fwdebug_check(9, "WCL_DEBUG"):
miscutils.fwdebug_print("funcmatch keys=%s" %
funcmatch.group(2))
miscutils.fwdebug_print("funcmatch funcname=%s" %
funcmatch.group(1))
keys = miscutils.fwsplit(funcmatch.group(2), ',')
argd = {}
for k in keys:
argd[k] = self.getfull(k)
p, m = funcmatch.group(1).rsplit('.', 1)
mod = import_module(p)
get_info_func = getattr(mod, m)
newinfo = get_info_func(argd)
self.update(newinfo)
else:
raise SyntaxError(
'File %s Line %d - Error: Invalid inclfunc %s' %
(filename, linecnt, patmatch))
line = in_file.readline()
linecnt += 1
continue
# handle group closing line <\string>
pat_match = re.search(r"^\s*</\s*(\S+)\s*>\s*$", line)
if pat_match is not None:
key = pat_match.group(1).lower()
if key == 'cmdline' or key == 'replace':
cmdline = False
sublabel = '__sublabel__' in curr
if sublabel:
del curr['__sublabel__']
if key == stackkeys[len(stackkeys) - 1]:
stackkeys.pop()
stack.pop()
curr = stack[len(stack) - 1]
elif sublabel:
if key == stackkeys[len(stackkeys) - 2]:
stackkeys.pop()
stack.pop()
curr = stack[len(stack) - 1]
stackkeys.pop()
stack.pop()
curr = stack[len(stack) - 1]
else:
print "******************************"
print "Linecnt =", linecnt
print "Line =", line.strip()
print "Closing Key =", key
self._print_stack(stackkeys, stack)
raise SyntaxError('File %s Line %d - Error: Invalid or missing section'
'close. Got close for %s. Expecting close for %s.' % \
(filename, linecnt, key, stackkeys[len(stackkeys) - 2]))
else:
print "******************************"
print "Linecnt =", linecnt
print "Line =", line.strip()
print "Closing Key =", key
self._print_stack(stackkeys, stack)
raise SyntaxError('File %s Line %d - Error: Invalid or missing section'
'close. Got close for %s. Expecting close for %s.' % \
(filename, linecnt, key, stackkeys[len(stackkeys) - 1]))
line = in_file.readline()
linecnt += 1
continue
# handle group opening line <key sublabel> or <key>
pat_match = re.search(r"^\s*<(\S+)\s*(\S+)?>\s*$", line)
if pat_match is not None:
key = pat_match.group(1).lower()
# check for case where missing / when closing section
if key == stackkeys[-1]:
print "******************************"
print "Linecnt =", linecnt
print "Line =", line.strip()
print "Opening Key =", key
self._print_stack(stackkeys, stack)
raise SyntaxError('File %s Line %d - Error: found '
'child section with same name (%s)' % \
(filename, linecnt, key))
stackkeys.append(key)
if not key in curr:
curr[key] = OrderedDict()
stack.append(curr[key])
curr = curr[key]
if key == 'cmdline' or key == 'replace':
cmdline = True
if pat_match.group(2) is not None:
val = pat_match.group(2).lower()
stackkeys.append(val)
if not val in curr:
curr[val] = OrderedDict()
curr[val]['__sublabel__'] = True
stack.append(curr[val])
curr = curr[val]
line = in_file.readline()
linecnt += 1
continue
# handle key/val line: key = val
pat_key_val = r"^\s*(\S+)(\s*=\s*)(.+)\s*$"
pat_match = re.search(pat_key_val, line)
if pat_match is not None:
key = pat_match.group(1)
if not cmdline:
key = key.lower()
curr[key] = pat_match.group(3).strip()
line = in_file.readline()
linecnt += 1
continue
pat_key_val = r"^\s*(\S+)(\s+)([^=].*)\s*$"
pat_match = re.search(pat_key_val, line)
if pat_match is not None:
key = pat_match.group(1)
if not cmdline:
key = key.lower()
curr[key] = pat_match.group(3).strip()
line = in_file.readline()
linecnt += 1
continue
print "Warning: Ignoring line #%d (did not match patterns):" % linecnt
print line
# goto next line if no matches
line = in_file.readline()
linecnt += 1
# done parsing input, should only be main dict in stack
if len(stack) != 1 or len(stackkeys) != 1:
self._print_stack(stackkeys, stack)
print "File %s - Error parsing wcl_file." % filename
print "Check that all sections have closing line."
raise SyntaxError("File %s - missing section closing line." %
filename)
def set_submit_info(self):
"""Initialize submit time values.
"""
self['des_home'] = os.path.abspath(os.path.dirname(__file__)) + "/.."
self['submit_dir'] = os.getcwd()
self['submit_host'] = os.uname()[1]
if 'submit_time' in self: # operator providing submit_time
submit_time = self['submit_time']
submit_epoch = int(
time.mktime(time.strptime(submit_time, "%Y%m%d%H%M%S")))
else:
submit_epoch = time.time()
submit_time = time.strftime("%Y%m%d%H%M%S",
time.localtime(submit_epoch))
self['submit_time'] = submit_time
self['submit_epoch'] = submit_epoch
self[pfwdefs.PF_JOBNUM] = '0'
self[pfwdefs.PF_BLKNUM] = '1'
self[pfwdefs.PF_TASKNUM] = '0'
self[pfwdefs.PF_WRAPNUM] = '0'
self[pfwdefs.UNITNAME] = self.getfull(pfwdefs.UNITNAME)
self.reset_blknum()
self.set_block_info()
self['submit_run'] = replfuncs.replace_vars_single(
"${unitname}_r${reqnum}p${attnum:2}", self, None)
self['submit_%s' % pfwdefs.REQNUM] = self.getfull(pfwdefs.REQNUM)
self['submit_%s' % pfwdefs.UNITNAME] = self.getfull(pfwdefs.UNITNAME)
self['submit_%s' % pfwdefs.ATTNUM] = self.getfull(pfwdefs.ATTNUM)
self['run'] = self.getfull('submit_run')
work_dir = ''
if pfwdefs.SUBMIT_RUN_DIR in self:
work_dir = self.getfull(pfwdefs.SUBMIT_RUN_DIR)
if work_dir[0] != '/': # submit_run_dir was relative path
work_dir = self.getfull('submit_dir') + '/' + work_dir
else: # make a timestamp-based directory in cwd
work_dir = "%s/%s_%s" % (self.getfull('submit_dir'),
os.path.splitext(
self['submitwcl'])[0], submit_time)
self['work_dir'] = work_dir
self['uberctrl_dir'] = work_dir + "/uberctrl"
(exists, master_save_file) = self.search(pfwdefs.MASTER_SAVE_FILE,
{intgdefs.REPLACE_VARS: True})
if exists:
if master_save_file not in pfwdefs.VALID_MASTER_SAVE_FILE:
match = re.match(r'rand_(\d\d)', master_save_file.lower())
if match:
if random.randrange(100) <= int(match.group(1)):
if miscutils.fwdebug_check(3, 'PFWCONFIG_DEBUG'):
miscutils.fwdebug_print(
'Changing %s to %s' %
(pfwdefs.MASTER_SAVE_FILE, 'always'))
self[pfwdefs.MASTER_SAVE_FILE] = 'always'
else:
if miscutils.fwdebug_check(3, 'PFWCONFIG_DEBUG'):
miscutils.fwdebug_print(
'Changing %s to %s' %
(pfwdefs.MASTER_SAVE_FILE, 'file'))
self[pfwdefs.MASTER_SAVE_FILE] = 'file'
else:
miscutils.fwdie(
"Error: Invalid value for %s (%s)" %
(pfwdefs.MASTER_SAVE_FILE, master_save_file),
pfwdefs.PF_EXIT_FAILURE)
else:
self[pfwdefs.MASTER_SAVE_FILE] = pfwdefs.MASTER_SAVE_FILE_DEFAULT
def read(self, in_file=None, cmdline=False, filename='stdin'):
"""Reads WCL text from an open file object and returns a dictionary"""
curr = self
stack = [] # to keep track of current sub-dictionary
stackkeys = ['__topwcl__'] # to keep track of current section key
stack.append(curr) #
line = in_file.readline()
linecnt = 1
while line:
line = line.strip()
while line.endswith('\\'):
linecnt += 1
line = line[:-1] + in_file.readline().strip()
# delete comments
line = line.split('#')[0]
# skip comment line or empty line
if re.search(r"\S", line):
# handle includes
patmatch = re.search(r"<<include (\S+)>>", line)
if patmatch is not None:
# replace wcl vars in filename
filename2 = replfuncs.replace_vars_single(
patmatch.group(1), self, None)
# expand ~ and env vars in filename
filename2 = os.path.expandvars(
os.path.expanduser(filename2))
wclobj2 = WCL()
with open(filename2, "r") as wclfh:
wclobj2.read(wclfh, cmdline, filename2)
self.update(wclobj2)
line = in_file.readline()
linecnt += 1
continue
# handle calls to external functions to get more information usually from db
patmatch = re.search(r"<<inclfunc ([^>]+)>>", line)
if patmatch is not None:
if miscutils.fwdebug_check(9, "WCL_DEBUG"):
miscutils.fwdebug_print(
f"patmatch={patmatch.group(0)}")
funcmatch = re.match(r'([^(]+)\(([^)]+)\)',
patmatch.group(1))
if funcmatch:
if miscutils.fwdebug_check(9, "WCL_DEBUG"):
miscutils.fwdebug_print(
f"funcmatch keys={funcmatch.group(2)}")
miscutils.fwdebug_print(
f"funcmatch funcname={funcmatch.group(1)}")
keys = miscutils.fwsplit(funcmatch.group(2), ',')
argd = {}
for k in keys:
argd[k] = self.getfull(k)
p, m = funcmatch.group(1).rsplit('.', 1)
mod = import_module(p)
get_info_func = getattr(mod, m)
newinfo = get_info_func(argd)
self.update(newinfo)
else:
raise SyntaxError(
f'File {filename} Line {linecnt:d} - Error: Invalid inclfunc {patmatch}'
)
line = in_file.readline()
linecnt += 1
continue
# handle group closing line <\string>
pat_match = re.search(r"^\s*</\s*(\S+)\s*>\s*$", line)
if pat_match is not None:
key = pat_match.group(1).lower()
if key in ['cmdline', 'replace']:
cmdline = False
sublabel = '__sublabel__' in curr
if sublabel:
del curr['__sublabel__']
if key == stackkeys[len(stackkeys) - 1]:
stackkeys.pop()
stack.pop()
curr = stack[len(stack) - 1]
elif sublabel:
if key == stackkeys[len(stackkeys) - 2]:
stackkeys.pop()
stack.pop()
curr = stack[len(stack) - 1]
stackkeys.pop()
stack.pop()
curr = stack[len(stack) - 1]
else:
print("******************************")
print("Linecnt =", linecnt)
print("Line =", line.strip())
print("Closing Key =", key)
self._print_stack(stackkeys, stack)
raise SyntaxError(
f'File {filename} Line {linecnt:d} - Error: Invalid or missing section'
+
f'close. Got close for {key}. Expecting close for {stackkeys[len(stackkeys) - 2]}.'
)
else:
print("******************************")
print("Linecnt =", linecnt)
print("Line =", line.strip())
print("Closing Key =", key)
self._print_stack(stackkeys, stack)
raise SyntaxError(
f'File {filename} Line {linecnt:d} - Error: Invalid or missing section'
+
f'close. Got close for {key}. Expecting close for {stackkeys[len(stackkeys) - 1]}.'
)
line = in_file.readline()
linecnt += 1
continue
# handle group opening line <key sublabel> or <key>
pat_match = re.search(r"^\s*<(\S+)\s*(\S+)?>\s*$", line)
if pat_match is not None:
key = pat_match.group(1).lower()
# check for case where missing / when closing section
if key == stackkeys[-1]:
print("******************************")
print("Linecnt =", linecnt)
print("Line =", line.strip())
print("Opening Key =", key)
self._print_stack(stackkeys, stack)
raise SyntaxError(
f'File {filename} Line {linecnt:d} - Error: found '
+ f'child section with same name ({key})')
stackkeys.append(key)
if not key in curr:
curr[key] = collections.OrderedDict()
stack.append(curr[key])
curr = curr[key]
if key in ['cmdline', 'replace']:
cmdline = True
if pat_match.group(2) is not None:
val = pat_match.group(2).lower()
stackkeys.append(val)
if not val in curr:
curr[val] = collections.OrderedDict()
curr[val]['__sublabel__'] = True
stack.append(curr[val])
curr = curr[val]
line = in_file.readline()
linecnt += 1
continue
# handle key/val line: key = val
pat_key_val = r"^\s*(\S+)(\s*=\s*)(.+)\s*$"
pat_match = re.search(pat_key_val, line)
if pat_match is not None:
key = pat_match.group(1)
if not cmdline:
key = key.lower()
curr[key] = pat_match.group(3).strip()
line = in_file.readline()
linecnt += 1
continue
print("HERE")
pat_key_val = r"^\s*(\S+)(\s+)([^=].*)\s*$"
pat_match = re.search(pat_key_val, line)
if pat_match is not None:
key = pat_match.group(1)
if not cmdline:
key = key.lower()
curr[key] = pat_match.group(3).strip()
line = in_file.readline()
linecnt += 1
continue
print(
f"Warning: Ignoring line #{linecnt:d} (did not match patterns):"
)
print(line)
# goto next line if no matches
line = in_file.readline()
linecnt += 1
# done parsing input, should only be main dict in stack
if len(stack) != 1 or len(stackkeys) != 1:
self._print_stack(stackkeys, stack)
print(f"File {filename} - Error parsing wcl_file.")
print("Check that all sections have closing line.")
raise SyntaxError(
f"File {filename} - missing section closing line.")
def handle_file(notify_file, delivery_fullname, config, filemgmt, task_id):
""" Performs steps necessary for each file """
ftype = None
metadata = None
disk_info = None
new_fullname = None
# read values from notify file
notifydict = read_notify_file(notify_file)
# use dts_md5sum from notify_file
dts_md5sum = None
if 'md5sum' in notifydict:
dts_md5sum = notifydict['md5sum']
miscutils.fwdebug_print("%s: dts md5sum = %s" % (delivery_fullname, dts_md5sum))
#print config.keys()
try:
filename = miscutils.parse_fullname(delivery_fullname, miscutils.CU_PARSE_FILENAME)
miscutils.fwdebug_print("filename = %s" % filename)
if not os.path.exists(delivery_fullname):
miscutils.fwdebug_print("Warning: delivered file does not exist:")
miscutils.fwdebug_print("\tnotification file: %s" % notify_file)
miscutils.fwdebug_print("\tdelivered file: %s" % delivery_fullname)
miscutils.fwdebug_print("\tRemoving notification file and continuing")
os.unlink(notify_file)
return
ftype = dtsutils.determine_filetype(filename)
if miscutils.fwdebug_check(3, "DTSFILEHANDLER_DEBUG"):
miscutils.fwdebug_print("filetype = %s" % ftype)
if filemgmt.is_file_in_archive([delivery_fullname], config['archive_name']):
handle_bad_file(config, notify_file, delivery_fullname, None, filemgmt,
ftype, None, None, "Duplicate file")
elif filemgmt.check_valid(ftype, delivery_fullname):
starttime = datetime.now()
results = filemgmt.register_file_data(ftype, [delivery_fullname], None, task_id, False, None, None)
endtime = datetime.now()
miscutils.fwdebug_print("%s: gathering and registering file data (%0.2f secs)" % \
(delivery_fullname, (endtime - starttime).total_seconds()))
disk_info = results[delivery_fullname]['diskinfo']
metadata = results[delivery_fullname]['metadata']
md5sum_before_move = disk_info['md5sum']
# check that dts given md5sum matches md5sum in delivery directory
if dts_md5sum is not None:
miscutils.fwdebug_print("%s: md5sum before move %s" % (delivery_fullname,
md5sum_before_move))
if md5sum_before_move != dts_md5sum:
miscutils.fwdebug_print("%s: dts md5sum = %s" % (delivery_fullname, dts_md5sum))
miscutils.fwdebug_print("%s: py md5sum = %s" % (delivery_fullname,
md5sum_before_move))
raise IOError("md5sum in delivery dir not the same as DTS-provided md5sum")
# get path
patkey = 'dirpat_' + ftype
miscutils.fwdebug_print('patkey = %s' % patkey)
dirpat = config['directory_pattern'][config[patkey]]['ops']
miscutils.fwdebug_print('dirpat = %s' % dirpat)
archive_rel_path = replfuncs.replace_vars_single(dirpat, metadata)
if miscutils.fwdebug_check(3, "DTSFILEHANDLER_DEBUG"):
miscutils.fwdebug_print('archive_rel_path = %s' % archive_rel_path)
new_fullname = move_file_to_archive(config, delivery_fullname,
archive_rel_path, dts_md5sum)
miscutils.fwdebug_print("%s: fullname in archive %s" % (delivery_fullname,
new_fullname))
filemgmt.register_file_in_archive(new_fullname, config['archive_name'])
# if success
miscutils.fwdebug_print("%s: success. committing to db" % (delivery_fullname))
filemgmt.commit()
os.unlink(notify_file)
else:
handle_bad_file(config, notify_file, delivery_fullname, new_fullname, filemgmt,
ftype, metadata, disk_info, "Invalid file")
except Exception as err:
(extype, exvalue, trback) = sys.exc_info()
print "******************************"
print "Error: %s" % delivery_fullname
traceback.print_exception(extype, exvalue, trback, file=sys.stdout)
print "******************************"
handle_bad_file(config, notify_file, delivery_fullname, new_fullname, filemgmt,
ftype, metadata, disk_info, "Exception: %s" % err)
except SystemExit: # Wrappers code calls exit if cannot find header value
handle_bad_file(config, notify_file, delivery_fullname, new_fullname, filemgmt,
ftype, metadata, disk_info,
"SystemExit: Probably missing header value. Check log for error msg.")
filemgmt.commit()
