def copyfiles(filelist, tstats, verify=False):
""" Copies files in given src,dst in filelist """
status = 0
for filename, fdict in filelist.items():
fsize = 0
try:
src = fdict['src']
dst = fdict['dst']
if 'filesize' in fdict:
fsize = fdict['filesize']
elif os.path.exists(src):
fsize = os.path.getsize(src)
if not os.path.exists(dst):
if tstats is not None:
tstats.stat_beg_file(filename)
path = os.path.dirname(dst)
if path and not os.path.exists(path):
miscutils.coremakedirs(path)
shutil.copy(src, dst)
if tstats is not None:
tstats.stat_end_file(0, fsize)
if verify:
newfsize = os.path.getsize(dst)
if newfsize != fsize:
raise Exception(f"Incorrect files size for file {filename} ({newfsize:d} vs {fsize:d})")
except Exception:
status = 1
if tstats is not None:
tstats.stat_end_file(1, fsize)
(_, value, _) = sys.exc_info()
filelist[filename]['err'] = str(value)
return (status, filelist)
def write_outputwcl(self, outfilename=None):
""" Write output wcl to file
Parameters
----------
outfilename : str, optional
The anem of the output wcl file to write. Default is ``None``
which indicates that the file name is stored in the inputwcl.
"""
if outfilename is None:
outfilename = self.inputwcl['wrapper']['outputwcl']
if miscutils.fwdebug_check(3, 'BASICWRAP_DEBUG'):
miscutils.fwdebug_print("outfilename = %s" % outfilename,
WRAPPER_OUTPUT_PREFIX)
# create output wcl directory if needed
outwcldir = miscutils.parse_fullname(outfilename,
miscutils.CU_PARSE_PATH)
if miscutils.fwdebug_check(3, 'BASICWRAP_DEBUG'):
miscutils.fwdebug_print("outwcldir = %s" % outwcldir,
WRAPPER_OUTPUT_PREFIX)
miscutils.coremakedirs(outwcldir)
with open(outfilename, 'w') as wclfh:
self.outputwcl.write(wclfh, True)
def write_block_dag(config, blkdir, blockname, debugfh=None):
""" writes block dag file """
if not debugfh:
debugfh = sys.stderr
debugfh.write('write_block_dag pwd: %s\n' % os.getcwd())
pfwdir = config.getfull('processingfw_dir')
cwd = os.getcwd()
miscutils.coremakedirs(blkdir)
os.chdir(blkdir)
print "curr dir = ", os.getcwd()
configfile = "../uberctrl/config.des"
jobmngr = write_stub_jobmngr_dag(config, blockname, blkdir, debugfh)
dag = config.get_filename('blockdag', {intgdefs.REPLACE_VARS: True})
dagfh = open(dag, 'w')
dagfh.write('JOB begblock blocktask.condor\n')
dagfh.write('VARS begblock exec="$(pfwdir)/libexec/begblock.py"\n')
dagfh.write('VARS begblock args="%s"\n' % configfile)
varstr = create_common_vars(config, 'begblock')
dagfh.write('%s\n' % varstr)
dagfh.write('SCRIPT pre begblock %s/libexec/logpre.py %s %s j $JOB\n' % \
(pfwdir, configfile, blockname))
dagfh.write('SCRIPT post begblock %s/libexec/logpost.py %s %s j $JOB $RETURN\n' % \
(pfwdir, configfile, blockname))
dagfh.write('\n')
dagfh.write('JOB jobmngr %s.condor.sub\n' % jobmngr)
dagfh.write('SCRIPT pre jobmngr %s/libexec/logpre.py %s %s j $JOB\n' % \
(pfwdir, configfile, blockname))
dagfh.write('SCRIPT post jobmngr %s/libexec/logpost.py %s %s j $JOB $RETURN\n' % \
(pfwdir, configfile, blockname))
dagfh.write('\n')
dagfh.write('JOB endblock blocktask.condor\n')
dagfh.write('VARS endblock exec="%s/libexec/endblock.py"\n' % pfwdir)
dagfh.write('VARS endblock args="%s"\n' % configfile)
varstr = create_common_vars(config, 'endblock')
dagfh.write('%s\n' % varstr)
dagfh.write('SCRIPT pre endblock %s/libexec/logpre.py %s %s j $JOB\n' % \
(pfwdir, configfile, blockname))
dagfh.write('SCRIPT post endblock %s/libexec/logpost.py %s %s j $JOB $RETURN\n' % \
(pfwdir, configfile, blockname))
dagfh.write('\nPARENT begblock CHILD jobmngr\n')
dagfh.write('PARENT jobmngr CHILD endblock\n')
dagfh.close()
pfwcondor.add2dag(dag, config.get_dag_cmd_opts(),
config.get_condor_attributes(blockname, "blockmngr"),
blkdir, debugfh)
os.chdir(cwd)
return dag
def copyfiles(filelist, tstats, verify=False):
""" Copies files in given src,dst in filelist
Parameters
----------
filelist : list
List of files to transfer
tstats : object
Class for tracking the statistics of the transfer
verify : bool
Whether to verify the file transfer (if possible) (True) or
not (False), default is False
Returns
-------
tuple
The exit status of the transfer and a dictionary of
the transfer results
"""
status = 0
for filename, fdict in filelist.items():
fsize = 0
try:
src = fdict['src']
dst = fdict['dst']
if 'filesize' in fdict:
fsize = fdict['filesize']
elif os.path.exists(src):
fsize = os.path.getsize(src)
if not os.path.exists(dst):
if tstats is not None:
tstats.stat_beg_file(filename)
path = os.path.dirname(dst)
if path and not os.path.exists(path):
miscutils.coremakedirs(path)
shutil.copy(src, dst)
if tstats is not None:
tstats.stat_end_file(0, fsize)
if verify:
newfsize = os.path.getsize(dst)
if newfsize != fsize:
raise Exception(
"Incorrect files size for file %s (%i vs %i)" %
(filename, newfsize, fsize))
except Exception:
status = 1
if tstats is not None:
tstats.stat_end_file(1, fsize)
(_, value, _) = sys.exc_info()
filelist[filename]['err'] = str(value)
return (status, filelist)
def move_file_to_archive(config, delivery_fullname, archive_rel_path, dts_md5sum):
""" Move file to its location in the archive """
basename = os.path.basename(delivery_fullname)
root = config['archive'][config['archive_name']]['root']
path = "%s/%s" % (root, archive_rel_path)
dst = "%s/%s" % (path, basename)
print delivery_fullname, dst
miscutils.coremakedirs(path)
#shutil.move(delivery_fullname, dst) replace move by cp+unlink
max_cp_tries = 5
cp_cnt = 1
copied = False
fileinfo = {}
while cp_cnt <= max_cp_tries and not copied:
shutil.copy2(delivery_fullname, dst) # similar to cp -p
starttime = datetime.now()
fileinfo = diskutils.get_single_file_disk_info(dst, True, root)
endtime = datetime.now()
print "%s: md5sum after move %s (%0.2f secs)" % (delivery_fullname,
fileinfo['md5sum'],
endtime-starttime)
if dts_md5sum is None:
copied = True
elif dts_md5sum != fileinfo['md5sum']:
print "Warning: md5 doesn't match after cp (%s, %s)" % (delivery_fullname, dst)
time.sleep(5)
os.unlink(dst) # remove bad file from archive
cp_cnt += 1
else:
copied = True
if not copied:
raise Exception("Error: cannot cp file without md5sum problems.")
os.unlink(delivery_fullname)
#(path, filename, compress) = miscutils.parse_fullname(dst, miscutils.CU_PARSE_PATH | \
# miscutils.CU_PARSE_FILENAME | \
# miscutils.CU_PARSE_EXTENSION)
#
#fileinfo = {'fullname': dst,
# 'filename' : filename,
# 'compression': compress,
# 'path': path,
# 'filesize': os.path.getsize(dst),
# 'md5sum': md5_after_move }
return fileinfo
def create_log_fullname(config):
""" Create the log's full filename """
today = datetime.datetime.now()
datepath = "%04d/%02d" % (today.year, today.month)
logdir = "%s/%s" % (config['accept_log_dir'], datepath)
miscutils.coremakedirs(logdir)
log_fullname = "%s/%04d%02d%02d_dts_accept.log" % (logdir, today.year,
today.month, today.day)
return log_fullname
def write_workflow_taskfile(config, jobnum, tasks):
""" Write the list of wrapper executions for a single job to a file """
taskfile = config.get_filename(
'jobtasklist', {
pfwdefs.PF_CURRVALS: {
'jobnum': jobnum
},
'required': True,
intgdefs.REPLACE_VARS: True
})
tjpad = pfwutils.pad_jobnum(jobnum)
miscutils.coremakedirs(tjpad)
with open(f"{tjpad}/{taskfile}", 'w') as tasksfh:
for task in sorted(tasks, key=lambda singletask: int(singletask[0])):
tasksfh.write(
f"{task[0]}, {task[1]}, {task[2]}, {task[3]}, {task[4]}\n")
return taskfile
def notify_file_delivered(fullname, config, md5sum):
""" Create a file denoting that a file has been delivered """
miscutils.coremakedirs(config['delivery_notice_dir'])
print "%s - accept file - %s" % (create_timestamp(), fullname)
notifyfile = "%s/%s.dts" % (config['delivery_notice_dir'],
os.path.basename(fullname))
print "%s - notify file - %s=%s" % (create_timestamp(), fullname,
notifyfile)
if md5sum is None:
touch_file(notifyfile)
else:
with open(notifyfile, "w") as notifyfh:
notifyfh.write("md5sum = %s\n" % md5sum)
def submit(submitfile, logdir):
"""Call dessubmit on the specific submit file.
Calls dessubmit on the specific submit file that has mass submit variables
replaced.
"""
print("%s: Submitting %s" % (tsstr(), submitfile))
cwd = os.getcwd()
# create log filename
submitbase = os.path.basename(submitfile)
submitdir = os.path.dirname(submitfile)
prefix = os.path.splitext(submitbase)[0]
logfilename = "%s.log" % prefix
if logdir is not None and len(logdir) != 0:
miscutils.coremakedirs(logdir)
logfilename = "%s/%s" % (logdir, logfilename)
os.chdir(submitdir)
print("%s: dessubmit stdout/stderr - %s" % (tsstr(), logfilename))
cmd = "dessubmit %s" % (submitbase)
with open(logfilename, 'w') as logfh:
# call dessubmit
try:
process = subprocess.Popen(cmd.split(),
shell=False,
stdout=logfh,
stderr=subprocess.STDOUT)
except:
(extype, exvalue, _) = sys.exc_info()
print("********************")
print("Unexpected error: %s" % exvalue)
print("cmd> %s" % cmd)
print("Probably could not find %s in path" % cmd.split()[0])
raise
process.wait()
print("%s: dessubmit finished with exit code = %s" %
(tsstr(), process.returncode))
if process.returncode != 0:
raise Exception("Non-zero exit code from dessubmit")
os.chdir(cwd)
def move_file_to_archive(config, delivery_fullname, archive_rel_path, dts_md5sum):
""" Move file to its location in the archive """
basename = os.path.basename(delivery_fullname)
root = config['archive'][config['archive_name']]['root']
path = "%s/%s" % (root, archive_rel_path)
dst = "%s/%s" % (path, basename)
if miscutils.fwdebug_check(3, "DTSFILEHANDLER_DEBUG"):
miscutils.fwdebug_print("%s -> %s" % (delivery_fullname, dst))
# try a couple times to copy file to archive directory
max_cp_tries = 5
cp_cnt = 1
copied = False
fileinfo = {}
while cp_cnt <= max_cp_tries and not copied:
miscutils.coremakedirs(path)
shutil.copy2(delivery_fullname, dst) # similar to cp -p
starttime = datetime.now()
fileinfo = diskutils.get_single_file_disk_info(dst, True, root)
endtime = datetime.now()
miscutils.fwdebug_print("%s: md5sum after move %s (%0.2f secs)" % \
(delivery_fullname, fileinfo['md5sum'],
(endtime - starttime).total_seconds()))
if dts_md5sum is None:
copied = True
elif dts_md5sum != fileinfo['md5sum']:
miscutils.fwdebug_print("Warning: md5 doesn't match after cp (%s, %s)" % \
(delivery_fullname, dst))
time.sleep(5)
os.unlink(dst) # remove bad file from archive
cp_cnt += 1
else:
copied = True
if not copied:
raise IOError("Cannot cp file (%s->%s)" % (delivery_fullname, dst))
os.unlink(delivery_fullname)
return dst
def create_submitside_dirs(config):
""" Create directories for storage of pfw files on submit side """
# make local working dir
workdir = config.getfull('work_dir')
if miscutils.fwdebug_check(3, 'PFWSUBMIT_DEBUG'):
miscutils.fwdebug_print(f"workdir = {workdir}")
if os.path.exists(workdir):
raise Exception(
f"{workdir} subdirectory already exists.\nAborting submission")
print('\tMaking submit run directory...')
miscutils.coremakedirs(workdir)
uberdir = config.getfull('uberctrl_dir')
if miscutils.fwdebug_check(3, 'PFWSUBMIT_DEBUG'):
miscutils.fwdebug_print(f"uberdir = {uberdir}")
print('\tMaking submit uberctrl directory...')
miscutils.coremakedirs(uberdir)
def write_outputwcl(self, outfilename=None):
""" Write output wcl to file """
if outfilename is None:
outfilename = self.inputwcl['wrapper']['outputwcl']
if miscutils.fwdebug_check(3, 'BASICWRAP_DEBUG'):
miscutils.fwdebug_print(f"outfilename = {outfilename}",
WRAPPER_OUTPUT_PREFIX)
# create output wcl directory if needed
outwcldir = miscutils.parse_fullname(outfilename,
miscutils.CU_PARSE_PATH)
if miscutils.fwdebug_check(3, 'BASICWRAP_DEBUG'):
miscutils.fwdebug_print(f"outwcldir = {outwcldir}",
WRAPPER_OUTPUT_PREFIX)
miscutils.coremakedirs(outwcldir)
with open(outfilename, 'w') as wclfh:
self.outputwcl.write(wclfh, True)
def submit(submitfile, logdir):
""" Call dessubmit on the specific submit file that has mass submit variables replaced """
print(f"{tsstr()} Submitting {submitfile}")
cwd = os.getcwd()
# create log filename
submitbase = os.path.basename(submitfile)
submitdir = os.path.dirname(submitfile)
prefix = os.path.splitext(submitbase)[0]
logfilename = f"{prefix}.log"
if logdir is not None and logdir:
miscutils.coremakedirs(logdir)
logfilename = f"{logdir}/{logfilename}"
os.chdir(submitdir)
print(f"{tsstr()}: dessubmit stdout/stderr - {logfilename}")
cmd = f"dessubmit {submitbase}"
with open(logfilename, 'w') as logfh:
# call dessubmit
try:
process = subprocess.Popen(cmd.split(),
shell=False,
stdout=logfh,
stderr=subprocess.STDOUT)
except:
(_, exvalue, _) = sys.exc_info()
print("********************")
print(f"Unexpected error: {exvalue}")
print(f"cmd> {cmd}")
print(f"Probably could not find {cmd.split()[0]} in path")
raise
process.wait()
print(f"{tsstr()}: dessubmit finished with exit code = {process.returncode}")
if process.returncode != 0:
raise Exception("Non-zero exit code from dessubmit")
os.chdir(cwd)
def main(argv):
""" Program entry point """
args = parse_cmdline(argv)
origtname = args['templatewcl']
origtwcl = None
with open(origtname, 'r') as twclfh:
origtwcl = ''.join(twclfh.readlines())
with open(args['submitlist'], 'r') as sublistfh:
for line in sublistfh:
line = line.split('#')[0].strip()
if line == "": # skip comments or blank lines
continue
info = miscutils.fwsplit(line, args['delimiter'])
# update name
newtname = None
if args['outfilepat'] is not None:
newtname = args['outfilepat']
else:
newtname = origtname
if args['submitfiledir'] is not None:
newtname = '%s/%s' % (args['submitfiledir'], newtname)
logdir = ""
if args['logdir'] is not None:
logdir = args['logdir']
for i in range(0, len(info)):
newtname = newtname.replace('XXX%dXXX' % (i + 1), info[i])
logdir = logdir.replace('XXX%dXXX' % (i + 1), info[i])
if not newtname.endswith(".des"):
newtname += ".des"
if args['force'] or not os.path.exists(newtname):
submitdir = os.path.dirname(newtname)
if submitdir != "":
miscutils.coremakedirs(submitdir)
# can I submit?
if not args['nosubmit']:
while not can_submit(args):
print "%s: Shouldn't submit, sleeping %s seconds." % \
(tsstr(), args['delay_check'])
time.sleep(args['delay_check'])
newwcl = origtwcl
for i in range(0, len(info)):
newwcl = newwcl.replace('XXX%dXXX' % (i + 1), info[i])
newwcl += 'GROUP_SUBMIT_ID = %d\n' % args['group_submit_id']
print "%s: Writing submit wcl: %s" % (tsstr(), newtname)
with open(newtname, 'w') as ntwclfh:
ntwclfh.write(newwcl)
# submit it
if not args['nosubmit']:
submit(newtname, logdir)
print "%s: Sleeping %s seconds after submit." % (
tsstr(), args['delay'])
time.sleep(args['delay'])
else:
print "skipping %s" % newtname
def create_output_dirs(self, exwcl):
""" Make directories for output files """
self.start_exec_task('create_output_dirs')
if intgdefs.IW_OUTPUTS in exwcl:
for sect in miscutils.fwsplit(exwcl[intgdefs.IW_OUTPUTS]):
sectkeys = sect.split('.')
if miscutils.fwdebug_check(3, 'BASICWRAP_DEBUG'):
miscutils.fwdebug_print(f"INFO: sectkeys={sectkeys}",
WRAPPER_OUTPUT_PREFIX)
if sectkeys[0] == intgdefs.IW_FILE_SECT:
sectname = sectkeys[1]
if sectname in self.inputwcl[intgdefs.IW_FILE_SECT]:
if 'fullname' in self.inputwcl[
intgdefs.IW_FILE_SECT][sectname]:
fullnames = self.inputwcl[
intgdefs.IW_FILE_SECT][sectname]['fullname']
if miscutils.fwdebug_check(3, 'BASICWRAP_DEBUG'):
miscutils.fwdebug_print(
f"INFO: fullname = {fullnames}",
WRAPPER_OUTPUT_PREFIX)
if '$RNMLST{' in fullnames:
raise ValueError(
'Deprecated $RNMLST in output filename')
for fname in miscutils.fwsplit(fullnames, ','):
outdir = os.path.dirname(fname)
miscutils.coremakedirs(outdir)
elif sectkeys[0] == intgdefs.IW_LIST_SECT:
(_, _, filesect) = sect.split('.')
ldict = self.inputwcl[intgdefs.IW_LIST_SECT][sectkeys[1]]
# check list itself exists
listname = ldict['fullname']
if miscutils.fwdebug_check(3, 'BASICWRAP_DEBUG'):
miscutils.fwdebug_print(
f"\tINFO: Checking existence of '{listname}'",
WRAPPER_OUTPUT_PREFIX)
if not os.path.exists(listname):
miscutils.fwdebug_print(
f"\tError: list '{listname}' does not exist.",
WRAPPER_OUTPUT_PREFIX)
raise IOError(
f"List not found: {listname} does not exist")
# get list format: space separated, csv, wcl, etc
listfmt = intgdefs.DEFAULT_LIST_FORMAT
if intgdefs.LIST_FORMAT in ldict:
listfmt = ldict[intgdefs.LIST_FORMAT]
# read fullnames from list file
fullnames = intgmisc.read_fullnames_from_listfile(
listname, listfmt, ldict['columns'])
if miscutils.fwdebug_check(3, 'BASICWRAP_DEBUG'):
miscutils.fwdebug_print(
f"\tINFO: fullnames={fullnames}",
WRAPPER_OUTPUT_PREFIX)
for fname in fullnames[filesect]:
outdir = os.path.dirname(fname)
miscutils.coremakedirs(outdir)
self.end_exec_task(0)
def handle_bad_file(config, notify_file, delivery_fullname, archive_fullname,
dbh, ftype, metadata, disk_info, msg):
""" Perform steps required by any bad file """
dbh.rollback() # undo any db changes for this file
miscutils.fwdebug_print("delivery_fullname = %s" % delivery_fullname)
miscutils.fwdebug_print("archive_fullname = %s" % archive_fullname)
miscutils.fwdebug_print("filetype = %s" % ftype)
miscutils.fwdebug_print("msg = %s" % msg)
if metadata is None:
miscutils.fwdebug_print("metadata = None")
else:
miscutils.fwdebug_print("# metadata = %s" % len(metadata))
if disk_info is None:
miscutils.fwdebug_print("disk_info = None")
else:
miscutils.fwdebug_print("disk_info is not None")
if miscutils.fwdebug_check(6, "DTSFILEHANDLER_DEBUG"):
miscutils.fwdebug_print("metadata = %s" % metadata)
miscutils.fwdebug_print("disk_info = %s" % disk_info)
today = datetime.now()
datepath = "%04d/%02d" % (today.year, today.month)
# where is file now
if archive_fullname is None:
orig_fullname = delivery_fullname
else:
orig_fullname = archive_fullname
# create a uniq name for living in the "bad file" area
# contains relative path for storing in DB
uniq_fullname = "%s/%s.%s" % (datepath, os.path.basename(orig_fullname),
today.strftime("%Y%m%d%H%M%S%f")[:-3])
# absolute path
destbad = "%s/%s" % (config['bad_file_dir'], uniq_fullname)
if os.path.exists(destbad):
miscutils.fwdebug_print("WARNING: bad file already exists (%s)" % destbad)
os.remove(destbad)
# make directory in "bad file" area and move file there
miscutils.coremakedirs(os.path.dirname(destbad))
shutil.move(orig_fullname, destbad)
# save information in db about bad file
row = {}
# save extra metadata if it exists
if metadata is not None:
badcols = dbh.get_column_names('DTS_BAD_FILE')
for bcol in badcols:
if bcol in metadata:
row[bcol] = metadata[bcol]
row['task_id'] = config['dts_task_id']
notifyts = os.path.getmtime(notify_file)
row['delivery_date'] = datetime.fromtimestamp(notifyts)
row['orig_filename'] = os.path.basename(orig_fullname)
row['uniq_fullname'] = uniq_fullname
row['rejected_date'] = today
row['rejected_msg'] = msg
row['filesize'] = os.path.getsize(destbad)
if ftype is not None:
row['filetype'] = ftype
dbh.basic_insert_row('DTS_BAD_FILE', row)
dbh.commit()
os.unlink(notify_file)
def handle_bad_file(config, notify_file, delivery_fullname, dbh,
filetype, metadata, disk_info, prov, msg):
""" Perform steps required by any bad file """
dbh.rollback() # undo any db changes for this file
miscutils.fwdebug(0, "DTSFILEHANDLER_DEBUG", "delivery_fullname = %s" % delivery_fullname)
miscutils.fwdebug(0, "DTSFILEHANDLER_DEBUG", "filetype = %s" % filetype)
miscutils.fwdebug(0, "DTSFILEHANDLER_DEBUG", "msg = %s" % msg)
miscutils.fwdebug(0, "DTSFILEHANDLER_DEBUG", "metadata = %s" % metadata)
miscutils.fwdebug(0, "DTSFILEHANDLER_DEBUG", "disk_info = %s" % disk_info)
miscutils.fwdebug(0, "DTSFILEHANDLER_DEBUG", "prov = %s" % prov)
today = datetime.now()
datepath = "%04d/%02d" % (today.year, today.month)
# where is file now
if disk_info is None:
orig_fullname = delivery_fullname
else:
orig_fullname = disk_info['fullname']
# create a uniq name for living in the "bad file" area
# contains relative path for storing in DB
uniq_fullname = "%s/%s.%s" % (datepath, os.path.basename(orig_fullname),
today.strftime("%Y%m%d%H%M%S%f")[:-3])
# absolute path
destbad = "%s/%s" % (config['bad_file_dir'], uniq_fullname)
if os.path.exists(destbad):
miscutils.fwdebug(0, "DTSFILEHANDLER_DEBUG", "WARNING: bad file already exists (%s)" % destbad)
os.remove(destbad)
# make directory in "bad file" area and move file there
miscutils.coremakedirs(os.path.dirname(destbad))
shutil.move(orig_fullname, destbad)
# save information in db about bad file
row = {}
# save extra metadata if it exists
if metadata is not None:
badcols = dbh.get_column_names('DTS_BAD_FILE')
for c in badcols:
if c in metadata:
row[c] = metadata[c]
row['task_id'] = config['dts_task_id']
t = os.path.getmtime(notify_file)
row['delivery_date'] = datetime.fromtimestamp(t)
row['orig_filename'] = os.path.basename(orig_fullname)
row['uniq_fullname'] = uniq_fullname
row['rejected_date'] = today
row['rejected_msg'] = msg
row['filesize'] = os.path.getsize(destbad)
if filetype is not None:
row['filetype'] = filetype
dbh.basic_insert_row('DTS_BAD_FILE', row)
dbh.commit()
os.unlink(notify_file)
def main(argv):
""" Program entry point """
args = parse_cmdline(argv)
origtname = args['templatewcl']
origtwcl = None
with open(origtname, 'r') as twclfh:
origtwcl = ''.join(twclfh.readlines())
with open(args['submitlist'], 'r') as sublistfh:
for line in sublistfh:
line = line.split('#')[0].strip()
if line == "": # skip comments or blank lines
continue
info = miscutils.fwsplit(line, args['delimiter'])
# update name
newtname = None
if args['outfilepat'] is not None:
newtname = args['outfilepat']
else:
newtname = origtname
if args['submitfiledir'] is not None:
newtname = f"{args['submitfiledir']}/{newtname}"
logdir = ""
if args['logdir'] is not None:
logdir = args['logdir']
for i, val in enumerate(info):
newtname = newtname.replace(f'XXX{(i + 1):d}XXX', val)
logdir = logdir.replace(f'XXX{(i + 1):d}XXX', val)
if not newtname.endswith(".des"):
newtname += ".des"
if args['force'] or not os.path.exists(newtname):
submitdir = os.path.dirname(newtname)
if submitdir != "":
miscutils.coremakedirs(submitdir)
# can I submit?
if not args['nosubmit']:
while not can_submit(args):
print(f"{tsstr()}: Shouldn't submit, sleeping {args['delay_check']} seconds.")
time.sleep(args['delay_check'])
newwcl = origtwcl
for i, val in enumerate(info):
newwcl = newwcl.replace(f'XXX{(i + 1):d}XXX', val)
newwcl += f"GROUP_SUBMIT_ID = {args['group_submit_id']:d}\n"
print(f"{tsstr()}: Writing submit wcl: {newtname}")
with open(newtname, 'w') as ntwclfh:
ntwclfh.write(newwcl)
# submit it
if not args['nosubmit']:
submit(newtname, logdir)
print(f"{tsstr()}: Sleeping {args['delay']} seconds after submit.")
time.sleep(args['delay'])
else:
print(f"skipping {newtname}")
def copyfiles(self,
filelist,
tstats,
secondsBetweenRetriesC=30,
numTriesC=5,
verify=None):
""" Copies files in given src,dst in filelist
Parameters
----------
filelist : dict
Dictionary of the files to transfer along with their peoperties
tstats : object
secondsBetweenRetriesC : int
How long to wait between retries of the transfer in seconds
Default: 30
numTriesC : int
Maximum number of times to attempt the file transfer.
Default: 5
verify : str
What type of verification to do ("filesize", "md5sum", None)
Default: None
Returns
------
Tuple containing the status (int) and the file list
"""
num_copies_from_archive = 0
num_copies_to_archive = 0
total_copy_time_from_archive = 0.0
total_copy_time_to_archive = 0.0
status = 0
# loop over each file
for filename, fdict in filelist.items():
fsize = 0
if 'filesize' in fdict and fdict['filesize'] is not None:
fsize = fdict['filesize']
try:
(src, isurl_src) = self.check_url(fdict['src'])
(dst, isurl_dst) = self.check_url(fdict['dst'])
if (isurl_src and isurl_dst) or (not isurl_src
and not isurl_dst):
miscutils.fwdie(
"Exactly one of isurl_src and isurl_dst has to be true (values: %s %s %s %s)"
% (isurl_src, src, isurl_dst, dst),
fmdefs.FM_EXIT_FAILURE)
copy_time = None
# if local file and file doesn't already exist
if not isurl_dst and not os.path.exists(dst):
if tstats is not None:
tstats.stat_beg_file(filename)
# make the path
path = os.path.dirname(dst)
if path and not os.path.exists(path):
miscutils.coremakedirs(path)
# getting some non-zero curl exit codes, double check path exists
if path and not os.path.exists(path):
raise Exception(
"Error: path still missing after coremakedirs (%s)"
% path)
copy_time = self.move_file(
src,
dst,
secondsBetweenRetries=secondsBetweenRetriesC,
numTries=numTriesC,
verify=verify,
fdict=fdict)
if tstats is not None:
tstats.stat_end_file(0, fsize)
elif isurl_dst: # if remote file
if tstats is not None:
tstats.stat_beg_file(filename)
# create remote paths
self.create_http_intermediate_dirs(dst)
copy_time = self.move_file(
src,
dst,
secondsBetweenRetries=secondsBetweenRetriesC,
numTries=numTriesC,
verify=verify,
fdict=fdict)
if tstats is not None:
tstats.stat_end_file(0, fsize)
# Print some debugging info:
if miscutils.fwdebug_check(9, "HTTP_UTILS_DEBUG"):
miscutils.fwdebug_print("\n")
for lines in traceback.format_stack():
for L in lines.split('\n'):
if L.strip() != '':
miscutils.fwdebug_print("call stack: %s" % L)
if miscutils.fwdebug_check(3, "HTTP_UTILS_DEBUG"):
miscutils.fwdebug_print(
"Copy info: %s %s %s %s %s %s" %
(EwdUtils.copyfiles_called, fdict['filename'], fsize,
copy_time, time.time(),
'toarchive' if isurl_dst else 'fromarchive'))
if copy_time is None:
copy_time = 0
if isurl_dst:
num_copies_to_archive += 1
total_copy_time_to_archive += copy_time
else:
num_copies_from_archive += 1
total_copy_time_from_archive += copy_time
except Exception as err:
status = 1
if tstats is not None:
tstats.stat_end_file(1, fsize)
filelist[filename]['err'] = str(err)
miscutils.fwdebug_print(str(err))
if tstats is None:
print "[Copy summary] copy_batch:%d file_copies_to_archive:%d time_to_archive:%.3f copies_from_archive:%d time_from_archive:%.3f end_time_for_batch:%.3f" % \
(EwdUtils.copyfiles_called, num_copies_to_archive, total_copy_time_to_archive, num_copies_from_archive, total_copy_time_from_archive, time.time())
EwdUtils.copyfiles_called += 1
return (status, filelist)
def copyfiles(self,
filelist,
tstats,
secondsBetweenRetriesC=30,
numTriesC=5,
verify=False):
""" Copies files in given src,dst in filelist
Parameters
----------
filelist : list
The files to transfer
tstats : dict
Data for tracking transfer statistics
secondsBetweenRetriesC : int
How many seconds between each retry
numtriesC : int
The number of times to retry a transfer, default is 5
verify : bool
Whether or not to verify the file size after transfer, default is
False (do not verify)
Returns
-------
tuple of the status ans file list
"""
num_copies_from_archive = 0
num_copies_to_archive = 0
total_copy_time_from_archive = 0.0
total_copy_time_to_archive = 0.0
status = 0
self.secondsBetweenRetries = secondsBetweenRetriesC
self.numtries = numTriesC
try:
for filename, fdict in filelist.items():
self.filesize = 0
if 'filesize' in fdict and fdict['filesize'] is not None:
self.filesize = fdict['filesize']
try:
(self.src, isurl_src) = self.check_url(fdict['src'])
(self.dst, isurl_dst) = self.check_url(fdict['dst'])
if (isurl_src and isurl_dst) or (not isurl_src
and not isurl_dst):
miscutils.fwdie(
"Exactly one of isurl_src and isurl_dst has to be true (values: %s %s %s %s)"
% (isurl_src, self.src, isurl_dst, self.dst),
fmdefs.FM_EXIT_FAILURE)
copy_time = None
# if local file and file doesn't already exist
if not isurl_dst and not os.path.exists(self.dst):
if tstats is not None:
tstats.stat_beg_file(filename)
# make the path
path = os.path.dirname(self.dst)
if path and not os.path.exists(path):
miscutils.coremakedirs(path)
# getting some non-zero curl exit codes, double check path exists
if path and not os.path.exists(path):
raise Exception(
"Error: path still missing after coremakedirs (%s)"
% path)
copy_time = self.get(verify)
if tstats is not None:
tstats.stat_end_file(0, self.filesize)
elif isurl_dst: # if remote file
if tstats is not None:
tstats.stat_beg_file(filename)
# create remote paths
self.create_http_intermediate_dirs(self.dst)
copy_time = self.put(verify)
if tstats is not None:
tstats.stat_end_file(0, self.filesize)
# Print some debugging info:
if miscutils.fwdebug_check(9, "HTTP_UTILS_DEBUG"):
miscutils.fwdebug_print("\n")
for lines in traceback.format_stack():
for L in lines.split('\n'):
if L.strip() != '':
miscutils.fwdebug_print("call stack: %s" %
L)
if miscutils.fwdebug_check(3, "HTTP_UTILS_DEBUG"):
miscutils.fwdebug_print(
"Copy info: %s %s %s %s %s %s" %
(HttpUtils.copyfiles_called, fdict['filename'],
self.filesize, copy_time, time.time(),
'toarchive' if isurl_dst else 'fromarchive'))
if copy_time is None:
copy_time = 0
if isurl_dst:
num_copies_to_archive += 1
total_copy_time_to_archive += copy_time
else:
num_copies_from_archive += 1
total_copy_time_from_archive += copy_time
except Exception as err:
status = 1
if tstats is not None:
tstats.stat_end_file(1, self.filesize)
filelist[filename]['err'] = str(err)
miscutils.fwdebug_print(str(err))
finally:
print "[Copy summary] copy_batch:%d file_copies_to_archive:%d time_to_archive:%.3f copies_from_archive:%d time_from_archive:%.3f end_time_for_batch:%.3f" % \
(HttpUtils.copyfiles_called, num_copies_to_archive, total_copy_time_to_archive, num_copies_from_archive, total_copy_time_from_archive, time.time())
HttpUtils.copyfiles_called += 1
return (status, filelist)
def copyfiles(self,
filelist,
tstats,
secondsBetweenRetriesC=30,
numTriesC=5,
verify=False):
""" Copies files in given src,dst in filelist """
num_copies_from_archive = 0
num_copies_to_archive = 0
total_copy_time_from_archive = 0.0
total_copy_time_to_archive = 0.0
status = 0
self.secondsBetweenRetries = secondsBetweenRetriesC
self.numtries = numTriesC
try:
for filename, fdict in filelist.items():
self.filesize = 0
if 'filesize' in fdict and fdict['filesize'] is not None:
self.filesize = fdict['filesize']
try:
(self.src, isurl_src) = self.check_url(fdict['src'])
(self.dst, isurl_dst) = self.check_url(fdict['dst'])
if (isurl_src and isurl_dst) or (not isurl_src
and not isurl_dst):
miscutils.fwdie(
f"Exactly one of isurl_src and isurl_dst has to be true (values: {isurl_src}, {self.src}, {isurl_dst}, {self.dst}",
fmdefs.FM_EXIT_FAILURE)
copy_time = None
# if local file and file doesn't already exist
if not isurl_dst and not os.path.exists(self.dst):
if tstats is not None:
tstats.stat_beg_file(filename)
# make the path
path = os.path.dirname(self.dst)
if path and not os.path.exists(path):
miscutils.coremakedirs(path)
# getting some non-zero curl exit codes, double check path exists
if path and not os.path.exists(path):
raise Exception(
f"Error: path still missing after coremakedirs ({path})"
)
copy_time = self.get(verify)
if tstats is not None:
tstats.stat_end_file(0, self.filesize)
elif isurl_dst: # if remote file
if tstats is not None:
tstats.stat_beg_file(filename)
# create remote paths
self.create_http_intermediate_dirs(self.dst)
copy_time = self.put(verify)
if tstats is not None:
tstats.stat_end_file(0, self.filesize)
# Print some debugging info:
if miscutils.fwdebug_check(9, "HTTP_UTILS_DEBUG"):
miscutils.fwdebug_print("\n")
for lines in traceback.format_stack():
for L in lines.split('\n'):
if L.strip() != '':
miscutils.fwdebug_print(f"call stack: {L}")
if miscutils.fwdebug_check(3, "HTTP_UTILS_DEBUG"):
miscutils.fwdebug_print(
f"Copy info: {HttpUtils.copyfiles_called} {fdict['filename']} {self.filesize} {copy_time} {time.time()} {'toarchive' if isurl_dst else 'fromarchive'}"
)
if copy_time is None:
copy_time = 0
if isurl_dst:
num_copies_to_archive += 1
total_copy_time_to_archive += copy_time
else:
num_copies_from_archive += 1
total_copy_time_from_archive += copy_time
except Exception as err:
status = 1
if tstats is not None:
tstats.stat_end_file(1, self.filesize)
filelist[filename]['err'] = str(err)
miscutils.fwdebug_print(str(err))
finally:
print(
f"[Copy summary] copy_batch:{HttpUtils.copyfiles_called:d} file_copies_to_archive:{num_copies_to_archive:d} time_to_archive:{total_copy_time_to_archive:.3f} copies_from_archive:{num_copies_from_archive:d} time_from_archive:{total_copy_time_from_archive:.3f} end_time_for_batch:{time.time():3f}"
)
HttpUtils.copyfiles_called += 1
return (status, filelist)
