def copyfiles(filelist, tstats, verify=False): """ Copies files in given src,dst in filelist """ status = 0 for filename, fdict in filelist.items(): fsize = 0 try: src = fdict['src'] dst = fdict['dst'] if 'filesize' in fdict: fsize = fdict['filesize'] elif os.path.exists(src): fsize = os.path.getsize(src) if not os.path.exists(dst): if tstats is not None: tstats.stat_beg_file(filename) path = os.path.dirname(dst) if path and not os.path.exists(path): miscutils.coremakedirs(path) shutil.copy(src, dst) if tstats is not None: tstats.stat_end_file(0, fsize) if verify: newfsize = os.path.getsize(dst) if newfsize != fsize: raise Exception(f"Incorrect files size for file {filename} ({newfsize:d} vs {fsize:d})") except Exception: status = 1 if tstats is not None: tstats.stat_end_file(1, fsize) (_, value, _) = sys.exc_info() filelist[filename]['err'] = str(value) return (status, filelist)
def write_outputwcl(self, outfilename=None): """ Write output wcl to file Parameters ---------- outfilename : str, optional The anem of the output wcl file to write. Default is ``None`` which indicates that the file name is stored in the inputwcl. """ if outfilename is None: outfilename = self.inputwcl['wrapper']['outputwcl'] if miscutils.fwdebug_check(3, 'BASICWRAP_DEBUG'): miscutils.fwdebug_print("outfilename = %s" % outfilename, WRAPPER_OUTPUT_PREFIX) # create output wcl directory if needed outwcldir = miscutils.parse_fullname(outfilename, miscutils.CU_PARSE_PATH) if miscutils.fwdebug_check(3, 'BASICWRAP_DEBUG'): miscutils.fwdebug_print("outwcldir = %s" % outwcldir, WRAPPER_OUTPUT_PREFIX) miscutils.coremakedirs(outwcldir) with open(outfilename, 'w') as wclfh: self.outputwcl.write(wclfh, True)
def write_block_dag(config, blkdir, blockname, debugfh=None): """ writes block dag file """ if not debugfh: debugfh = sys.stderr debugfh.write('write_block_dag pwd: %s\n' % os.getcwd()) pfwdir = config.getfull('processingfw_dir') cwd = os.getcwd() miscutils.coremakedirs(blkdir) os.chdir(blkdir) print "curr dir = ", os.getcwd() configfile = "../uberctrl/config.des" jobmngr = write_stub_jobmngr_dag(config, blockname, blkdir, debugfh) dag = config.get_filename('blockdag', {intgdefs.REPLACE_VARS: True}) dagfh = open(dag, 'w') dagfh.write('JOB begblock blocktask.condor\n') dagfh.write('VARS begblock exec="$(pfwdir)/libexec/begblock.py"\n') dagfh.write('VARS begblock args="%s"\n' % configfile) varstr = create_common_vars(config, 'begblock') dagfh.write('%s\n' % varstr) dagfh.write('SCRIPT pre begblock %s/libexec/logpre.py %s %s j $JOB\n' % \ (pfwdir, configfile, blockname)) dagfh.write('SCRIPT post begblock %s/libexec/logpost.py %s %s j $JOB $RETURN\n' % \ (pfwdir, configfile, blockname)) dagfh.write('\n') dagfh.write('JOB jobmngr %s.condor.sub\n' % jobmngr) dagfh.write('SCRIPT pre jobmngr %s/libexec/logpre.py %s %s j $JOB\n' % \ (pfwdir, configfile, blockname)) dagfh.write('SCRIPT post jobmngr %s/libexec/logpost.py %s %s j $JOB $RETURN\n' % \ (pfwdir, configfile, blockname)) dagfh.write('\n') dagfh.write('JOB endblock blocktask.condor\n') dagfh.write('VARS endblock exec="%s/libexec/endblock.py"\n' % pfwdir) dagfh.write('VARS endblock args="%s"\n' % configfile) varstr = create_common_vars(config, 'endblock') dagfh.write('%s\n' % varstr) dagfh.write('SCRIPT pre endblock %s/libexec/logpre.py %s %s j $JOB\n' % \ (pfwdir, configfile, blockname)) dagfh.write('SCRIPT post endblock %s/libexec/logpost.py %s %s j $JOB $RETURN\n' % \ (pfwdir, configfile, blockname)) dagfh.write('\nPARENT begblock CHILD jobmngr\n') dagfh.write('PARENT jobmngr CHILD endblock\n') dagfh.close() pfwcondor.add2dag(dag, config.get_dag_cmd_opts(), config.get_condor_attributes(blockname, "blockmngr"), blkdir, debugfh) os.chdir(cwd) return dag
def copyfiles(filelist, tstats, verify=False): """ Copies files in given src,dst in filelist Parameters ---------- filelist : list List of files to transfer tstats : object Class for tracking the statistics of the transfer verify : bool Whether to verify the file transfer (if possible) (True) or not (False), default is False Returns ------- tuple The exit status of the transfer and a dictionary of the transfer results """ status = 0 for filename, fdict in filelist.items(): fsize = 0 try: src = fdict['src'] dst = fdict['dst'] if 'filesize' in fdict: fsize = fdict['filesize'] elif os.path.exists(src): fsize = os.path.getsize(src) if not os.path.exists(dst): if tstats is not None: tstats.stat_beg_file(filename) path = os.path.dirname(dst) if path and not os.path.exists(path): miscutils.coremakedirs(path) shutil.copy(src, dst) if tstats is not None: tstats.stat_end_file(0, fsize) if verify: newfsize = os.path.getsize(dst) if newfsize != fsize: raise Exception( "Incorrect files size for file %s (%i vs %i)" % (filename, newfsize, fsize)) except Exception: status = 1 if tstats is not None: tstats.stat_end_file(1, fsize) (_, value, _) = sys.exc_info() filelist[filename]['err'] = str(value) return (status, filelist)
def move_file_to_archive(config, delivery_fullname, archive_rel_path, dts_md5sum): """ Move file to its location in the archive """ basename = os.path.basename(delivery_fullname) root = config['archive'][config['archive_name']]['root'] path = "%s/%s" % (root, archive_rel_path) dst = "%s/%s" % (path, basename) print delivery_fullname, dst miscutils.coremakedirs(path) #shutil.move(delivery_fullname, dst) replace move by cp+unlink max_cp_tries = 5 cp_cnt = 1 copied = False fileinfo = {} while cp_cnt <= max_cp_tries and not copied: shutil.copy2(delivery_fullname, dst) # similar to cp -p starttime = datetime.now() fileinfo = diskutils.get_single_file_disk_info(dst, True, root) endtime = datetime.now() print "%s: md5sum after move %s (%0.2f secs)" % (delivery_fullname, fileinfo['md5sum'], endtime-starttime) if dts_md5sum is None: copied = True elif dts_md5sum != fileinfo['md5sum']: print "Warning: md5 doesn't match after cp (%s, %s)" % (delivery_fullname, dst) time.sleep(5) os.unlink(dst) # remove bad file from archive cp_cnt += 1 else: copied = True if not copied: raise Exception("Error: cannot cp file without md5sum problems.") os.unlink(delivery_fullname) #(path, filename, compress) = miscutils.parse_fullname(dst, miscutils.CU_PARSE_PATH | \ # miscutils.CU_PARSE_FILENAME | \ # miscutils.CU_PARSE_EXTENSION) # #fileinfo = {'fullname': dst, # 'filename' : filename, # 'compression': compress, # 'path': path, # 'filesize': os.path.getsize(dst), # 'md5sum': md5_after_move } return fileinfo
def create_log_fullname(config): """ Create the log's full filename """ today = datetime.datetime.now() datepath = "%04d/%02d" % (today.year, today.month) logdir = "%s/%s" % (config['accept_log_dir'], datepath) miscutils.coremakedirs(logdir) log_fullname = "%s/%04d%02d%02d_dts_accept.log" % (logdir, today.year, today.month, today.day) return log_fullname
def write_workflow_taskfile(config, jobnum, tasks): """ Write the list of wrapper executions for a single job to a file """ taskfile = config.get_filename( 'jobtasklist', { pfwdefs.PF_CURRVALS: { 'jobnum': jobnum }, 'required': True, intgdefs.REPLACE_VARS: True }) tjpad = pfwutils.pad_jobnum(jobnum) miscutils.coremakedirs(tjpad) with open(f"{tjpad}/{taskfile}", 'w') as tasksfh: for task in sorted(tasks, key=lambda singletask: int(singletask[0])): tasksfh.write( f"{task[0]}, {task[1]}, {task[2]}, {task[3]}, {task[4]}\n") return taskfile
def notify_file_delivered(fullname, config, md5sum): """ Create a file denoting that a file has been delivered """ miscutils.coremakedirs(config['delivery_notice_dir']) print "%s - accept file - %s" % (create_timestamp(), fullname) notifyfile = "%s/%s.dts" % (config['delivery_notice_dir'], os.path.basename(fullname)) print "%s - notify file - %s=%s" % (create_timestamp(), fullname, notifyfile) if md5sum is None: touch_file(notifyfile) else: with open(notifyfile, "w") as notifyfh: notifyfh.write("md5sum = %s\n" % md5sum)
def submit(submitfile, logdir): """Call dessubmit on the specific submit file. Calls dessubmit on the specific submit file that has mass submit variables replaced. """ print("%s: Submitting %s" % (tsstr(), submitfile)) cwd = os.getcwd() # create log filename submitbase = os.path.basename(submitfile) submitdir = os.path.dirname(submitfile) prefix = os.path.splitext(submitbase)[0] logfilename = "%s.log" % prefix if logdir is not None and len(logdir) != 0: miscutils.coremakedirs(logdir) logfilename = "%s/%s" % (logdir, logfilename) os.chdir(submitdir) print("%s: dessubmit stdout/stderr - %s" % (tsstr(), logfilename)) cmd = "dessubmit %s" % (submitbase) with open(logfilename, 'w') as logfh: # call dessubmit try: process = subprocess.Popen(cmd.split(), shell=False, stdout=logfh, stderr=subprocess.STDOUT) except: (extype, exvalue, _) = sys.exc_info() print("********************") print("Unexpected error: %s" % exvalue) print("cmd> %s" % cmd) print("Probably could not find %s in path" % cmd.split()[0]) raise process.wait() print("%s: dessubmit finished with exit code = %s" % (tsstr(), process.returncode)) if process.returncode != 0: raise Exception("Non-zero exit code from dessubmit") os.chdir(cwd)
def move_file_to_archive(config, delivery_fullname, archive_rel_path, dts_md5sum): """ Move file to its location in the archive """ basename = os.path.basename(delivery_fullname) root = config['archive'][config['archive_name']]['root'] path = "%s/%s" % (root, archive_rel_path) dst = "%s/%s" % (path, basename) if miscutils.fwdebug_check(3, "DTSFILEHANDLER_DEBUG"): miscutils.fwdebug_print("%s -> %s" % (delivery_fullname, dst)) # try a couple times to copy file to archive directory max_cp_tries = 5 cp_cnt = 1 copied = False fileinfo = {} while cp_cnt <= max_cp_tries and not copied: miscutils.coremakedirs(path) shutil.copy2(delivery_fullname, dst) # similar to cp -p starttime = datetime.now() fileinfo = diskutils.get_single_file_disk_info(dst, True, root) endtime = datetime.now() miscutils.fwdebug_print("%s: md5sum after move %s (%0.2f secs)" % \ (delivery_fullname, fileinfo['md5sum'], (endtime - starttime).total_seconds())) if dts_md5sum is None: copied = True elif dts_md5sum != fileinfo['md5sum']: miscutils.fwdebug_print("Warning: md5 doesn't match after cp (%s, %s)" % \ (delivery_fullname, dst)) time.sleep(5) os.unlink(dst) # remove bad file from archive cp_cnt += 1 else: copied = True if not copied: raise IOError("Cannot cp file (%s->%s)" % (delivery_fullname, dst)) os.unlink(delivery_fullname) return dst
def create_submitside_dirs(config): """ Create directories for storage of pfw files on submit side """ # make local working dir workdir = config.getfull('work_dir') if miscutils.fwdebug_check(3, 'PFWSUBMIT_DEBUG'): miscutils.fwdebug_print(f"workdir = {workdir}") if os.path.exists(workdir): raise Exception( f"{workdir} subdirectory already exists.\nAborting submission") print('\tMaking submit run directory...') miscutils.coremakedirs(workdir) uberdir = config.getfull('uberctrl_dir') if miscutils.fwdebug_check(3, 'PFWSUBMIT_DEBUG'): miscutils.fwdebug_print(f"uberdir = {uberdir}") print('\tMaking submit uberctrl directory...') miscutils.coremakedirs(uberdir)
def write_outputwcl(self, outfilename=None): """ Write output wcl to file """ if outfilename is None: outfilename = self.inputwcl['wrapper']['outputwcl'] if miscutils.fwdebug_check(3, 'BASICWRAP_DEBUG'): miscutils.fwdebug_print(f"outfilename = {outfilename}", WRAPPER_OUTPUT_PREFIX) # create output wcl directory if needed outwcldir = miscutils.parse_fullname(outfilename, miscutils.CU_PARSE_PATH) if miscutils.fwdebug_check(3, 'BASICWRAP_DEBUG'): miscutils.fwdebug_print(f"outwcldir = {outwcldir}", WRAPPER_OUTPUT_PREFIX) miscutils.coremakedirs(outwcldir) with open(outfilename, 'w') as wclfh: self.outputwcl.write(wclfh, True)
def submit(submitfile, logdir): """ Call dessubmit on the specific submit file that has mass submit variables replaced """ print(f"{tsstr()} Submitting {submitfile}") cwd = os.getcwd() # create log filename submitbase = os.path.basename(submitfile) submitdir = os.path.dirname(submitfile) prefix = os.path.splitext(submitbase)[0] logfilename = f"{prefix}.log" if logdir is not None and logdir: miscutils.coremakedirs(logdir) logfilename = f"{logdir}/{logfilename}" os.chdir(submitdir) print(f"{tsstr()}: dessubmit stdout/stderr - {logfilename}") cmd = f"dessubmit {submitbase}" with open(logfilename, 'w') as logfh: # call dessubmit try: process = subprocess.Popen(cmd.split(), shell=False, stdout=logfh, stderr=subprocess.STDOUT) except: (_, exvalue, _) = sys.exc_info() print("********************") print(f"Unexpected error: {exvalue}") print(f"cmd> {cmd}") print(f"Probably could not find {cmd.split()[0]} in path") raise process.wait() print(f"{tsstr()}: dessubmit finished with exit code = {process.returncode}") if process.returncode != 0: raise Exception("Non-zero exit code from dessubmit") os.chdir(cwd)
def main(argv): """ Program entry point """ args = parse_cmdline(argv) origtname = args['templatewcl'] origtwcl = None with open(origtname, 'r') as twclfh: origtwcl = ''.join(twclfh.readlines()) with open(args['submitlist'], 'r') as sublistfh: for line in sublistfh: line = line.split('#')[0].strip() if line == "": # skip comments or blank lines continue info = miscutils.fwsplit(line, args['delimiter']) # update name newtname = None if args['outfilepat'] is not None: newtname = args['outfilepat'] else: newtname = origtname if args['submitfiledir'] is not None: newtname = '%s/%s' % (args['submitfiledir'], newtname) logdir = "" if args['logdir'] is not None: logdir = args['logdir'] for i in range(0, len(info)): newtname = newtname.replace('XXX%dXXX' % (i + 1), info[i]) logdir = logdir.replace('XXX%dXXX' % (i + 1), info[i]) if not newtname.endswith(".des"): newtname += ".des" if args['force'] or not os.path.exists(newtname): submitdir = os.path.dirname(newtname) if submitdir != "": miscutils.coremakedirs(submitdir) # can I submit? if not args['nosubmit']: while not can_submit(args): print "%s: Shouldn't submit, sleeping %s seconds." % \ (tsstr(), args['delay_check']) time.sleep(args['delay_check']) newwcl = origtwcl for i in range(0, len(info)): newwcl = newwcl.replace('XXX%dXXX' % (i + 1), info[i]) newwcl += 'GROUP_SUBMIT_ID = %d\n' % args['group_submit_id'] print "%s: Writing submit wcl: %s" % (tsstr(), newtname) with open(newtname, 'w') as ntwclfh: ntwclfh.write(newwcl) # submit it if not args['nosubmit']: submit(newtname, logdir) print "%s: Sleeping %s seconds after submit." % ( tsstr(), args['delay']) time.sleep(args['delay']) else: print "skipping %s" % newtname
def create_output_dirs(self, exwcl): """ Make directories for output files """ self.start_exec_task('create_output_dirs') if intgdefs.IW_OUTPUTS in exwcl: for sect in miscutils.fwsplit(exwcl[intgdefs.IW_OUTPUTS]): sectkeys = sect.split('.') if miscutils.fwdebug_check(3, 'BASICWRAP_DEBUG'): miscutils.fwdebug_print(f"INFO: sectkeys={sectkeys}", WRAPPER_OUTPUT_PREFIX) if sectkeys[0] == intgdefs.IW_FILE_SECT: sectname = sectkeys[1] if sectname in self.inputwcl[intgdefs.IW_FILE_SECT]: if 'fullname' in self.inputwcl[ intgdefs.IW_FILE_SECT][sectname]: fullnames = self.inputwcl[ intgdefs.IW_FILE_SECT][sectname]['fullname'] if miscutils.fwdebug_check(3, 'BASICWRAP_DEBUG'): miscutils.fwdebug_print( f"INFO: fullname = {fullnames}", WRAPPER_OUTPUT_PREFIX) if '$RNMLST{' in fullnames: raise ValueError( 'Deprecated $RNMLST in output filename') for fname in miscutils.fwsplit(fullnames, ','): outdir = os.path.dirname(fname) miscutils.coremakedirs(outdir) elif sectkeys[0] == intgdefs.IW_LIST_SECT: (_, _, filesect) = sect.split('.') ldict = self.inputwcl[intgdefs.IW_LIST_SECT][sectkeys[1]] # check list itself exists listname = ldict['fullname'] if miscutils.fwdebug_check(3, 'BASICWRAP_DEBUG'): miscutils.fwdebug_print( f"\tINFO: Checking existence of '{listname}'", WRAPPER_OUTPUT_PREFIX) if not os.path.exists(listname): miscutils.fwdebug_print( f"\tError: list '{listname}' does not exist.", WRAPPER_OUTPUT_PREFIX) raise IOError( f"List not found: {listname} does not exist") # get list format: space separated, csv, wcl, etc listfmt = intgdefs.DEFAULT_LIST_FORMAT if intgdefs.LIST_FORMAT in ldict: listfmt = ldict[intgdefs.LIST_FORMAT] # read fullnames from list file fullnames = intgmisc.read_fullnames_from_listfile( listname, listfmt, ldict['columns']) if miscutils.fwdebug_check(3, 'BASICWRAP_DEBUG'): miscutils.fwdebug_print( f"\tINFO: fullnames={fullnames}", WRAPPER_OUTPUT_PREFIX) for fname in fullnames[filesect]: outdir = os.path.dirname(fname) miscutils.coremakedirs(outdir) self.end_exec_task(0)
def handle_bad_file(config, notify_file, delivery_fullname, archive_fullname, dbh, ftype, metadata, disk_info, msg): """ Perform steps required by any bad file """ dbh.rollback() # undo any db changes for this file miscutils.fwdebug_print("delivery_fullname = %s" % delivery_fullname) miscutils.fwdebug_print("archive_fullname = %s" % archive_fullname) miscutils.fwdebug_print("filetype = %s" % ftype) miscutils.fwdebug_print("msg = %s" % msg) if metadata is None: miscutils.fwdebug_print("metadata = None") else: miscutils.fwdebug_print("# metadata = %s" % len(metadata)) if disk_info is None: miscutils.fwdebug_print("disk_info = None") else: miscutils.fwdebug_print("disk_info is not None") if miscutils.fwdebug_check(6, "DTSFILEHANDLER_DEBUG"): miscutils.fwdebug_print("metadata = %s" % metadata) miscutils.fwdebug_print("disk_info = %s" % disk_info) today = datetime.now() datepath = "%04d/%02d" % (today.year, today.month) # where is file now if archive_fullname is None: orig_fullname = delivery_fullname else: orig_fullname = archive_fullname # create a uniq name for living in the "bad file" area # contains relative path for storing in DB uniq_fullname = "%s/%s.%s" % (datepath, os.path.basename(orig_fullname), today.strftime("%Y%m%d%H%M%S%f")[:-3]) # absolute path destbad = "%s/%s" % (config['bad_file_dir'], uniq_fullname) if os.path.exists(destbad): miscutils.fwdebug_print("WARNING: bad file already exists (%s)" % destbad) os.remove(destbad) # make directory in "bad file" area and move file there miscutils.coremakedirs(os.path.dirname(destbad)) shutil.move(orig_fullname, destbad) # save information in db about bad file row = {} # save extra metadata if it exists if metadata is not None: badcols = dbh.get_column_names('DTS_BAD_FILE') for bcol in badcols: if bcol in metadata: row[bcol] = metadata[bcol] row['task_id'] = config['dts_task_id'] notifyts = os.path.getmtime(notify_file) row['delivery_date'] = datetime.fromtimestamp(notifyts) row['orig_filename'] = os.path.basename(orig_fullname) row['uniq_fullname'] = uniq_fullname row['rejected_date'] = today row['rejected_msg'] = msg row['filesize'] = os.path.getsize(destbad) if ftype is not None: row['filetype'] = ftype dbh.basic_insert_row('DTS_BAD_FILE', row) dbh.commit() os.unlink(notify_file)
def handle_bad_file(config, notify_file, delivery_fullname, dbh, filetype, metadata, disk_info, prov, msg): """ Perform steps required by any bad file """ dbh.rollback() # undo any db changes for this file miscutils.fwdebug(0, "DTSFILEHANDLER_DEBUG", "delivery_fullname = %s" % delivery_fullname) miscutils.fwdebug(0, "DTSFILEHANDLER_DEBUG", "filetype = %s" % filetype) miscutils.fwdebug(0, "DTSFILEHANDLER_DEBUG", "msg = %s" % msg) miscutils.fwdebug(0, "DTSFILEHANDLER_DEBUG", "metadata = %s" % metadata) miscutils.fwdebug(0, "DTSFILEHANDLER_DEBUG", "disk_info = %s" % disk_info) miscutils.fwdebug(0, "DTSFILEHANDLER_DEBUG", "prov = %s" % prov) today = datetime.now() datepath = "%04d/%02d" % (today.year, today.month) # where is file now if disk_info is None: orig_fullname = delivery_fullname else: orig_fullname = disk_info['fullname'] # create a uniq name for living in the "bad file" area # contains relative path for storing in DB uniq_fullname = "%s/%s.%s" % (datepath, os.path.basename(orig_fullname), today.strftime("%Y%m%d%H%M%S%f")[:-3]) # absolute path destbad = "%s/%s" % (config['bad_file_dir'], uniq_fullname) if os.path.exists(destbad): miscutils.fwdebug(0, "DTSFILEHANDLER_DEBUG", "WARNING: bad file already exists (%s)" % destbad) os.remove(destbad) # make directory in "bad file" area and move file there miscutils.coremakedirs(os.path.dirname(destbad)) shutil.move(orig_fullname, destbad) # save information in db about bad file row = {} # save extra metadata if it exists if metadata is not None: badcols = dbh.get_column_names('DTS_BAD_FILE') for c in badcols: if c in metadata: row[c] = metadata[c] row['task_id'] = config['dts_task_id'] t = os.path.getmtime(notify_file) row['delivery_date'] = datetime.fromtimestamp(t) row['orig_filename'] = os.path.basename(orig_fullname) row['uniq_fullname'] = uniq_fullname row['rejected_date'] = today row['rejected_msg'] = msg row['filesize'] = os.path.getsize(destbad) if filetype is not None: row['filetype'] = filetype dbh.basic_insert_row('DTS_BAD_FILE', row) dbh.commit() os.unlink(notify_file)
def main(argv): """ Program entry point """ args = parse_cmdline(argv) origtname = args['templatewcl'] origtwcl = None with open(origtname, 'r') as twclfh: origtwcl = ''.join(twclfh.readlines()) with open(args['submitlist'], 'r') as sublistfh: for line in sublistfh: line = line.split('#')[0].strip() if line == "": # skip comments or blank lines continue info = miscutils.fwsplit(line, args['delimiter']) # update name newtname = None if args['outfilepat'] is not None: newtname = args['outfilepat'] else: newtname = origtname if args['submitfiledir'] is not None: newtname = f"{args['submitfiledir']}/{newtname}" logdir = "" if args['logdir'] is not None: logdir = args['logdir'] for i, val in enumerate(info): newtname = newtname.replace(f'XXX{(i + 1):d}XXX', val) logdir = logdir.replace(f'XXX{(i + 1):d}XXX', val) if not newtname.endswith(".des"): newtname += ".des" if args['force'] or not os.path.exists(newtname): submitdir = os.path.dirname(newtname) if submitdir != "": miscutils.coremakedirs(submitdir) # can I submit? if not args['nosubmit']: while not can_submit(args): print(f"{tsstr()}: Shouldn't submit, sleeping {args['delay_check']} seconds.") time.sleep(args['delay_check']) newwcl = origtwcl for i, val in enumerate(info): newwcl = newwcl.replace(f'XXX{(i + 1):d}XXX', val) newwcl += f"GROUP_SUBMIT_ID = {args['group_submit_id']:d}\n" print(f"{tsstr()}: Writing submit wcl: {newtname}") with open(newtname, 'w') as ntwclfh: ntwclfh.write(newwcl) # submit it if not args['nosubmit']: submit(newtname, logdir) print(f"{tsstr()}: Sleeping {args['delay']} seconds after submit.") time.sleep(args['delay']) else: print(f"skipping {newtname}")
def copyfiles(self, filelist, tstats, secondsBetweenRetriesC=30, numTriesC=5, verify=None): """ Copies files in given src,dst in filelist Parameters ---------- filelist : dict Dictionary of the files to transfer along with their peoperties tstats : object secondsBetweenRetriesC : int How long to wait between retries of the transfer in seconds Default: 30 numTriesC : int Maximum number of times to attempt the file transfer. Default: 5 verify : str What type of verification to do ("filesize", "md5sum", None) Default: None Returns ------ Tuple containing the status (int) and the file list """ num_copies_from_archive = 0 num_copies_to_archive = 0 total_copy_time_from_archive = 0.0 total_copy_time_to_archive = 0.0 status = 0 # loop over each file for filename, fdict in filelist.items(): fsize = 0 if 'filesize' in fdict and fdict['filesize'] is not None: fsize = fdict['filesize'] try: (src, isurl_src) = self.check_url(fdict['src']) (dst, isurl_dst) = self.check_url(fdict['dst']) if (isurl_src and isurl_dst) or (not isurl_src and not isurl_dst): miscutils.fwdie( "Exactly one of isurl_src and isurl_dst has to be true (values: %s %s %s %s)" % (isurl_src, src, isurl_dst, dst), fmdefs.FM_EXIT_FAILURE) copy_time = None # if local file and file doesn't already exist if not isurl_dst and not os.path.exists(dst): if tstats is not None: tstats.stat_beg_file(filename) # make the path path = os.path.dirname(dst) if path and not os.path.exists(path): miscutils.coremakedirs(path) # getting some non-zero curl exit codes, double check path exists if path and not os.path.exists(path): raise Exception( "Error: path still missing after coremakedirs (%s)" % path) copy_time = self.move_file( src, dst, secondsBetweenRetries=secondsBetweenRetriesC, numTries=numTriesC, verify=verify, fdict=fdict) if tstats is not None: tstats.stat_end_file(0, fsize) elif isurl_dst: # if remote file if tstats is not None: tstats.stat_beg_file(filename) # create remote paths self.create_http_intermediate_dirs(dst) copy_time = self.move_file( src, dst, secondsBetweenRetries=secondsBetweenRetriesC, numTries=numTriesC, verify=verify, fdict=fdict) if tstats is not None: tstats.stat_end_file(0, fsize) # Print some debugging info: if miscutils.fwdebug_check(9, "HTTP_UTILS_DEBUG"): miscutils.fwdebug_print("\n") for lines in traceback.format_stack(): for L in lines.split('\n'): if L.strip() != '': miscutils.fwdebug_print("call stack: %s" % L) if miscutils.fwdebug_check(3, "HTTP_UTILS_DEBUG"): miscutils.fwdebug_print( "Copy info: %s %s %s %s %s %s" % (EwdUtils.copyfiles_called, fdict['filename'], fsize, copy_time, time.time(), 'toarchive' if isurl_dst else 'fromarchive')) if copy_time is None: copy_time = 0 if isurl_dst: num_copies_to_archive += 1 total_copy_time_to_archive += copy_time else: num_copies_from_archive += 1 total_copy_time_from_archive += copy_time except Exception as err: status = 1 if tstats is not None: tstats.stat_end_file(1, fsize) filelist[filename]['err'] = str(err) miscutils.fwdebug_print(str(err)) if tstats is None: print "[Copy summary] copy_batch:%d file_copies_to_archive:%d time_to_archive:%.3f copies_from_archive:%d time_from_archive:%.3f end_time_for_batch:%.3f" % \ (EwdUtils.copyfiles_called, num_copies_to_archive, total_copy_time_to_archive, num_copies_from_archive, total_copy_time_from_archive, time.time()) EwdUtils.copyfiles_called += 1 return (status, filelist)
def copyfiles(self, filelist, tstats, secondsBetweenRetriesC=30, numTriesC=5, verify=False): """ Copies files in given src,dst in filelist Parameters ---------- filelist : list The files to transfer tstats : dict Data for tracking transfer statistics secondsBetweenRetriesC : int How many seconds between each retry numtriesC : int The number of times to retry a transfer, default is 5 verify : bool Whether or not to verify the file size after transfer, default is False (do not verify) Returns ------- tuple of the status ans file list """ num_copies_from_archive = 0 num_copies_to_archive = 0 total_copy_time_from_archive = 0.0 total_copy_time_to_archive = 0.0 status = 0 self.secondsBetweenRetries = secondsBetweenRetriesC self.numtries = numTriesC try: for filename, fdict in filelist.items(): self.filesize = 0 if 'filesize' in fdict and fdict['filesize'] is not None: self.filesize = fdict['filesize'] try: (self.src, isurl_src) = self.check_url(fdict['src']) (self.dst, isurl_dst) = self.check_url(fdict['dst']) if (isurl_src and isurl_dst) or (not isurl_src and not isurl_dst): miscutils.fwdie( "Exactly one of isurl_src and isurl_dst has to be true (values: %s %s %s %s)" % (isurl_src, self.src, isurl_dst, self.dst), fmdefs.FM_EXIT_FAILURE) copy_time = None # if local file and file doesn't already exist if not isurl_dst and not os.path.exists(self.dst): if tstats is not None: tstats.stat_beg_file(filename) # make the path path = os.path.dirname(self.dst) if path and not os.path.exists(path): miscutils.coremakedirs(path) # getting some non-zero curl exit codes, double check path exists if path and not os.path.exists(path): raise Exception( "Error: path still missing after coremakedirs (%s)" % path) copy_time = self.get(verify) if tstats is not None: tstats.stat_end_file(0, self.filesize) elif isurl_dst: # if remote file if tstats is not None: tstats.stat_beg_file(filename) # create remote paths self.create_http_intermediate_dirs(self.dst) copy_time = self.put(verify) if tstats is not None: tstats.stat_end_file(0, self.filesize) # Print some debugging info: if miscutils.fwdebug_check(9, "HTTP_UTILS_DEBUG"): miscutils.fwdebug_print("\n") for lines in traceback.format_stack(): for L in lines.split('\n'): if L.strip() != '': miscutils.fwdebug_print("call stack: %s" % L) if miscutils.fwdebug_check(3, "HTTP_UTILS_DEBUG"): miscutils.fwdebug_print( "Copy info: %s %s %s %s %s %s" % (HttpUtils.copyfiles_called, fdict['filename'], self.filesize, copy_time, time.time(), 'toarchive' if isurl_dst else 'fromarchive')) if copy_time is None: copy_time = 0 if isurl_dst: num_copies_to_archive += 1 total_copy_time_to_archive += copy_time else: num_copies_from_archive += 1 total_copy_time_from_archive += copy_time except Exception as err: status = 1 if tstats is not None: tstats.stat_end_file(1, self.filesize) filelist[filename]['err'] = str(err) miscutils.fwdebug_print(str(err)) finally: print "[Copy summary] copy_batch:%d file_copies_to_archive:%d time_to_archive:%.3f copies_from_archive:%d time_from_archive:%.3f end_time_for_batch:%.3f" % \ (HttpUtils.copyfiles_called, num_copies_to_archive, total_copy_time_to_archive, num_copies_from_archive, total_copy_time_from_archive, time.time()) HttpUtils.copyfiles_called += 1 return (status, filelist)
def copyfiles(self, filelist, tstats, secondsBetweenRetriesC=30, numTriesC=5, verify=False): """ Copies files in given src,dst in filelist """ num_copies_from_archive = 0 num_copies_to_archive = 0 total_copy_time_from_archive = 0.0 total_copy_time_to_archive = 0.0 status = 0 self.secondsBetweenRetries = secondsBetweenRetriesC self.numtries = numTriesC try: for filename, fdict in filelist.items(): self.filesize = 0 if 'filesize' in fdict and fdict['filesize'] is not None: self.filesize = fdict['filesize'] try: (self.src, isurl_src) = self.check_url(fdict['src']) (self.dst, isurl_dst) = self.check_url(fdict['dst']) if (isurl_src and isurl_dst) or (not isurl_src and not isurl_dst): miscutils.fwdie( f"Exactly one of isurl_src and isurl_dst has to be true (values: {isurl_src}, {self.src}, {isurl_dst}, {self.dst}", fmdefs.FM_EXIT_FAILURE) copy_time = None # if local file and file doesn't already exist if not isurl_dst and not os.path.exists(self.dst): if tstats is not None: tstats.stat_beg_file(filename) # make the path path = os.path.dirname(self.dst) if path and not os.path.exists(path): miscutils.coremakedirs(path) # getting some non-zero curl exit codes, double check path exists if path and not os.path.exists(path): raise Exception( f"Error: path still missing after coremakedirs ({path})" ) copy_time = self.get(verify) if tstats is not None: tstats.stat_end_file(0, self.filesize) elif isurl_dst: # if remote file if tstats is not None: tstats.stat_beg_file(filename) # create remote paths self.create_http_intermediate_dirs(self.dst) copy_time = self.put(verify) if tstats is not None: tstats.stat_end_file(0, self.filesize) # Print some debugging info: if miscutils.fwdebug_check(9, "HTTP_UTILS_DEBUG"): miscutils.fwdebug_print("\n") for lines in traceback.format_stack(): for L in lines.split('\n'): if L.strip() != '': miscutils.fwdebug_print(f"call stack: {L}") if miscutils.fwdebug_check(3, "HTTP_UTILS_DEBUG"): miscutils.fwdebug_print( f"Copy info: {HttpUtils.copyfiles_called} {fdict['filename']} {self.filesize} {copy_time} {time.time()} {'toarchive' if isurl_dst else 'fromarchive'}" ) if copy_time is None: copy_time = 0 if isurl_dst: num_copies_to_archive += 1 total_copy_time_to_archive += copy_time else: num_copies_from_archive += 1 total_copy_time_from_archive += copy_time except Exception as err: status = 1 if tstats is not None: tstats.stat_end_file(1, self.filesize) filelist[filename]['err'] = str(err) miscutils.fwdebug_print(str(err)) finally: print( f"[Copy summary] copy_batch:{HttpUtils.copyfiles_called:d} file_copies_to_archive:{num_copies_to_archive:d} time_to_archive:{total_copy_time_to_archive:.3f} copies_from_archive:{num_copies_from_archive:d} time_from_archive:{total_copy_time_from_archive:.3f} end_time_for_batch:{time.time():3f}" ) HttpUtils.copyfiles_called += 1 return (status, filelist)