def _setup(self): db = config.load(self.db_config_file) self.logger.info('Connecting to %s@%s ...' % (db.reader, db.sid)) self.connection = cx_Oracle.connect(db.reader, db.phrase, db.sid) self.cursor = self.connection.cursor()
def iterate(): path = cfg.get('Input', 'path') _scratch_base = cfg.get('Output', 'scratch_base') _dqm_base = cfg.get('Output', 'dqm_base') _ecal_base = cfg.get('Output', 'ecal_base') _lookarea_base = cfg.get('Output', 'lookarea_base') _evd_base = cfg.get('Output', 'evd_base') _evd_eosbase = cfg.get('Output', 'evd_eosbase') _checksum_status = cfg.getboolean('Misc', 'checksum_status') setup_label = cfg.get('Input', 'setup_label') db_config = cfg.get('Bookkeeping', 'db_config') new_path_base = cfg.get('Output', 'new_path_base') db_cred = config.load(db_config) connection = cx_Oracle.connect(db_cred.db_user, db_cred.db_pwd, db_cred.db_sid) cursor = connection.cursor() _streams_with_scalers = cfg.getlist('Streams', 'streams_with_scalars') _streams_to_ecal = cfg.getlist('Streams', 'streams_to_ecal') _streams_to_evd = cfg.getlist('Streams', 'streams_to_evd') _streams_to_dqm = cfg.getlist('Streams', 'streams_to_dqm') _streams_to_lookarea = cfg.getlist('Streams', 'streams_to_lookarea') _streams_to_postpone = cfg.getlist('Streams', 'streams_to_postpone') _streams_to_ignore = cfg.getlist('Streams', 'streams_to_ignore') _renotify = cfg.getboolean('Misc', 'renotify') max_tier0_transfer_file_size = cfg.getint( 'Output', 'maximum_tier0_transfer_file_size_in_bytes') max_dqm_transfer_file_size = cfg.getint( 'Output', 'maximum_dqm_transfer_file_size_in_bytes') hostname = socket.gethostname() new_path = get_new_path(path, new_path_base) scratch_path = get_new_path(path, _scratch_base) rundirs, hltkeys = get_rundirs_and_hltkeys(path, new_path) # Just for the MiniEoR files for rundir in rundirs: logger.debug("Inspecting `%s' for EoR searching..." % rundir) jsns = sorted(glob.glob(os.path.join(rundir, '*EoR*.jsn'))) if not jsns: continue rundir_basename = os.path.basename(rundir) run_number = int(rundir_basename.replace('run', '')) new_rundir = os.path.join(new_path, rundir_basename) scratch_rundir = os.path.join(scratch_path, rundir_basename) jsns.sort() run_key = runinfo.get_run_key(run_number) for jsn_file in jsns: if ('BoLS' not in jsn_file and 'EoLS' not in jsn_file and 'index' not in jsn_file and 'EoR' in jsn_file): if run_key == 'TIER0_TRANSFER_OFF': maybe_move(jsn_file, scratch_rundir, force_overwrite=True) else: maybe_move(jsn_file, new_rundir, force_overwrite=True) continue # check if there are any stream directories check_rundirs = [] for rundir in rundirs: if not (glob.glob(os.path.join(rundir, 'stream*'))): rundir = rundir check_rundirs.append(rundir) else: for streamdir in glob.glob(os.path.join(rundir, 'stream*')): check_rundirs.append(streamdir) for rundir in check_rundirs: logger.debug("Inspecting `%s' ..." % rundir) jsns = sorted(glob.glob(os.path.join(rundir, '*.jsn'))) if not jsns: continue if 'stream' in rundir: rundir_basename = os.path.basename(os.path.dirname(rundir)) else: rundir_basename = os.path.basename(rundir) run_number = int(rundir_basename.replace('run', '')) logger.info('********** Run %d **********' % run_number) bookkeeper._run_number = run_number new_rundir = os.path.join(new_path, rundir_basename) rundir_bad = os.path.join(rundir, 'bad') new_rundir_bad = os.path.join(new_rundir, 'bad') scratch_rundir = os.path.join(scratch_path, rundir_basename) dqm_rundir_open = os.path.join(_dqm_base, rundir_basename, 'open') dqm_rundir = os.path.join(_dqm_base, rundir_basename) ecal_rundir_open = os.path.join(_ecal_base, rundir_basename, 'open') ecal_rundir = os.path.join(_ecal_base, rundir_basename) evd_rundir = os.path.join(_evd_base, rundir_basename) evd_rundir_open = os.path.join(_evd_base, rundir_basename, 'open') evd_eosrundir = os.path.join(_evd_eosbase, rundir_basename) lookarea_rundir_open = os.path.join(_lookarea_base, rundir_basename, 'open') lookarea_rundir = os.path.join(_lookarea_base, rundir_basename) run_key = runinfo.get_run_key(run_number) if not os.path.exists(scratch_rundir): mkdir(scratch_rundir) mkdir(os.path.join(scratch_rundir, 'bad')) if (not os.path.exists(new_rundir) and not run_key == 'TIER0_TRANSFER_OFF'): mkdir(new_rundir) mkdir(os.path.join(new_rundir, 'bad')) logger.info("Opening bookkeeping for run %d ..." % run_number) try: bookkeeper.open_run(cursor) connection.commit() except cx_Oracle.IntegrityError: logger.warning('Bookkeeping for run %d already open!' % run_number) appversion = runinfo.get_cmssw_version(run_number) if appversion == 'UNKNOWN': logger.warning('The CMSSW version is UNKNOWN for run %d' % run_number) hlt_key = runinfo.get_hlt_key(run_number) # Sort JSON files by filename, implying also by lumi. jsns.sort() logger.info("Processing {count} JSON file(s) in `{folder}':\n".format( count=len(jsns), folder=rundir) + pprint.pformat([os.path.basename(f) for f in jsns])) for jsn_file in jsns: if ('BoLS' not in jsn_file and 'EoLS' not in jsn_file and 'index' not in jsn_file): settings_textI = open(jsn_file, "r").read() try: settings = json.loads(settings_textI) except ValueError: logger.warning("The json file %s is corrupted!" % jsn_file) maybe_move(jsn_file, new_rundir_bad, suffix='Corrupted') continue if len(settings['data']) < 5: logger.warning("Failed to parse `%s'!" % jsn_file) maybe_move(jsn_file, scratch_rundir, force_overwrite=True) continue eventsNumber = int(settings['data'][1]) fileName = str(settings['data'][3]) fileSize = int(settings['data'][4]) lumiSection = int(fileName.split('_')[1].strip('ls')) #streamName = str(fileName.split('_')[2].strip('stream')) streamName = str(fileName.split('_')[2].split('stream')[1]) if (_checksum_status): checksum_int = int(settings['data'][5]) checksum = format(checksum_int, 'x').zfill( 8) #making sure it is 8 digits else: checksum = 0 dat_file = os.path.join(rundir, fileName) logger.info( "The hex format checksum of the file {0} is {1} ".format( dat_file, checksum)) if streamName in _streams_with_scalers: monitor_rates(jsn_file) if streamName in _streams_to_postpone: continue if streamName in _streams_to_dqm: ## TODO: Use some other temporary directory instead of ## scratch if (fileSize > max_dqm_transfer_file_size): logger.warning("`{0}' too large ({1} > {2})! ".format( dat_file, fileSize, max_dqm_transfer_file_size ) + "Moving it to bad area with the suffix `TooLarge' ..." ) maybe_move(jsn_file, new_rundir_bad, suffix='TooLarge') maybe_move(dat_file, new_rundir_bad, suffix='TooLarge') else: maybe_move(jsn_file, scratch_rundir) maybe_move(dat_file, scratch_rundir) jsn_file = jsn_file.replace(rundir, scratch_rundir) dat_file = dat_file.replace(rundir, scratch_rundir) args = [ dat_file, jsn_file, dqm_rundir_open, dqm_rundir, lookarea_rundir_open, lookarea_rundir ] dqm_pool.apply_async(double_p5_location, args) #args = [dat_file, jsn_file, dqm_rundir_open, dqm_rundir] #dqm_pool.apply_async(move_files, args) continue if streamName in _streams_to_ecal: ## TODO: Use some other temporary directory instead of ## scratch maybe_move(jsn_file, scratch_rundir) maybe_move(dat_file, scratch_rundir) jsn_file = jsn_file.replace(rundir, scratch_rundir) dat_file = dat_file.replace(rundir, scratch_rundir) args = [dat_file, jsn_file, ecal_rundir_open, ecal_rundir] ecal_pool.apply_async(move_files, args) continue #if streamName in _streams_to_lookarea: ## TODO: Use some other temporary directory instead of ## scratch # maybe_move(jsn_file, scratch_rundir) # maybe_move(dat_file, scratch_rundir) # jsn_file = jsn_file.replace(rundir, scratch_rundir) # dat_file = dat_file.replace(rundir, scratch_rundir) # args = [dat_file, jsn_file, lookarea_rundir_open, lookarea_rundir] # lookarea_pool.apply_async(move_files, args) # continue if streamName in _streams_to_evd: maybe_move(jsn_file, scratch_rundir, force_overwrite=True) maybe_move(dat_file, scratch_rundir, force_overwrite=True) jsn_file = jsn_file.replace(rundir, scratch_rundir) dat_file = dat_file.replace(rundir, scratch_rundir) # Dima said they don't need the open area args = [ dat_file, jsn_file, evd_rundir_open, evd_rundir, evd_eosrundir ] evd_pool.apply_async(copy_move_files, args) continue if (run_key == 'TIER0_TRANSFER_OFF' or streamName in (_streams_with_scalers + _streams_to_ignore)): maybe_move(jsn_file, scratch_rundir) maybe_move(dat_file, scratch_rundir) continue if (fileSize > max_tier0_transfer_file_size): logger.warning( "`{0}' too large ({1} > {2})! ".format( dat_file, fileSize, max_tier0_transfer_file_size) + "Moving it to bad area with the suffix `TooLarge' ...") maybe_move(jsn_file, new_rundir_bad, suffix='TooLarge') maybe_move(dat_file, new_rundir_bad, suffix='TooLarge') continue starttime = int(os.stat(dat_file).st_atime) stoptime = int(os.stat(jsn_file).st_ctime) maybe_move(dat_file, new_rundir) maybe_move(jsn_file, new_rundir) ## Call the actual inject script if eventsNumber == 0: number_of_files = 0 else: number_of_files = 1 args_insert = [ './insertFile.pl', '--FILENAME', fileName, '--FILECOUNTER', 0, '--NEVENTS', 0, '--FILESIZE', 0, '--STARTTIME', starttime, '--STOPTIME', 0, '--STATUS', 'open', '--RUNNUMBER', run_number, '--LUMISECTION', lumiSection, '--PATHNAME', rundir, '--HOSTNAME', hostname, '--SETUPLABEL', setup_label, '--STREAM', streamName, '--INSTANCE', 1, '--SAFETY', 0, '--APPVERSION', appversion, '--APPNAME', 'CMSSW', '--TYPE', 'streamer', '--CHECKSUM', checksum, '--CHECKSUMIND', 0, ] args_close = [ './closeFile.pl', '--FILENAME', fileName, '--FILECOUNTER', 0, '--NEVENTS', eventsNumber, '--FILESIZE', fileSize, '--STARTTIME', starttime, '--STOPTIME', stoptime, '--STATUS', 'closed', '--RUNNUMBER', run_number, '--LUMISECTION', lumiSection, '--PATHNAME', new_rundir, '--HOSTNAME', hostname, '--SETUPLABEL', setup_label, '--STREAM', streamName, '--INSTANCE', 1, '--SAFETY', 0, '--APPVERSION', appversion, '--APPNAME', 'CMSSW', '--TYPE', 'streamer', '--DEBUGCLOSE', 2, '--CHECKSUM', checksum, '--CHECKSUMIND', 0, ] inject_file_path = os.path.join( cfg.get('Output', 'inject_base'), '{date}-{hostname}.log'.format( date=date.today().strftime('%Y%m%d'), hostname=hostname, )) with open(inject_file_path, 'a') as inject_file: line = ' '.join(map(str, args_insert)) logger.info("Appending line `%s' to `%s' ..." % (line, inject_file_path)) inject_file.write(line + '\n') line = ' '.join(map(str, args_close)) logger.info("Appending line `%s' to `%s' ..." % (line, inject_file_path)) inject_file.write(line + '\n') try: bookkeeper.fill_number_of_files(cursor, streamName, lumiSection, number_of_files) connection.commit() except cx_Oracle.IntegrityError: print('WARNING: Failed to insert bookkeeping for ' + 'run {0}, stream {1}, ls {2}: #files = {3}').format( run_number, streamName, lumiSection, number_of_files) ## Move the bad area to new run dir so that we can check for run ## completeness for fname in sorted(glob.glob(os.path.join(rundir_bad, '*.jsn'))): try: jsn = metafile.File(fname) if jsn.type == metafile.Type.MacroMerger: dat_path = jsn.path.replace('.jsn', '.dat') maybe_move(jsn.path, new_rundir_bad) maybe_move(dat_path, new_rundir_bad) except ValueError: logger.warning("Illegal filename `%s'!" % fname) connection.close()
def iterate(): path = cfg.get('Input', 'path') _scratch_base = cfg.get('Output','scratch_base') _dqm_base = cfg.get('Output','dqm_base') _ecal_base = cfg.get('Output','ecal_base') _lookarea_base = cfg.get('Output','lookarea_base') _evd_base = cfg.get('Output','evd_base') _evd_eosbase = cfg.get('Output','evd_eosbase') _checksum_status = cfg.getboolean('Misc','checksum_status') setup_label=cfg.get('Input','setup_label') db_config = cfg.get('Bookkeeping', 'db_config') new_path_base = cfg.get('Output', 'new_path_base') db_cred = config.load(db_config) connection = cx_Oracle.connect(db_cred.db_user, db_cred.db_pwd, db_cred.db_sid) cursor = connection.cursor() _streams_with_scalers = cfg.getlist('Streams','streams_with_scalars') _streams_to_ecal = cfg.getlist('Streams','streams_to_ecal' ) _streams_to_evd = cfg.getlist('Streams','streams_to_evd' ) _streams_to_dqm = cfg.getlist('Streams','streams_to_dqm' ) _streams_to_lookarea = cfg.getlist('Streams','streams_to_lookarea' ) _streams_to_postpone = cfg.getlist('Streams','streams_to_postpone' ) _streams_to_ignore = cfg.getlist('Streams','streams_to_ignore' ) _renotify = cfg.getboolean('Misc','renotify') max_tier0_transfer_file_size = cfg.getint( 'Output', 'maximum_tier0_transfer_file_size_in_bytes' ) max_dqm_transfer_file_size = cfg.getint( 'Output', 'maximum_dqm_transfer_file_size_in_bytes' ) hostname = socket.gethostname() new_path = get_new_path(path, new_path_base) scratch_path = get_new_path(path, _scratch_base) rundirs, hltkeys = get_rundirs_and_hltkeys(path, new_path) # Just for the MiniEoR files for rundir in rundirs: logger.debug("Inspecting `%s' for EoR searching..." % rundir) jsns = sorted(glob.glob(os.path.join(rundir, '*EoR*.jsn'))) if not jsns: continue rundir_basename = os.path.basename(rundir) run_number = int(rundir_basename.replace('run', '')) new_rundir = os.path.join(new_path , rundir_basename) scratch_rundir = os.path.join(scratch_path, rundir_basename) jsns.sort() run_key = runinfo.get_run_key(run_number) for jsn_file in jsns: if ('BoLS' not in jsn_file and 'EoLS' not in jsn_file and 'index' not in jsn_file and 'EoR' in jsn_file): if run_key == 'TIER0_TRANSFER_OFF': maybe_move(jsn_file, scratch_rundir, force_overwrite=True) else: maybe_move(jsn_file, new_rundir, force_overwrite=True) continue # check if there are any stream directories check_rundirs = [] for rundir in rundirs: if not (glob.glob(os.path.join(rundir, 'stream*'))): rundir = rundir check_rundirs.append(rundir) else: for streamdir in glob.glob(os.path.join(rundir, 'stream*')): check_rundirs.append(streamdir) for rundir in check_rundirs: logger.debug("Inspecting `%s' ..." % rundir) jsns = sorted(glob.glob(os.path.join(rundir, '*.jsn'))) if not jsns: continue if 'stream' in rundir: rundir_basename = os.path.basename(os.path.dirname(rundir)) else: rundir_basename = os.path.basename(rundir) run_number = int(rundir_basename.replace('run', '')) logger.info('********** Run %d **********' % run_number) bookkeeper._run_number = run_number new_rundir = os.path.join(new_path , rundir_basename) rundir_bad = os.path.join(rundir , 'bad') new_rundir_bad = os.path.join(new_rundir , 'bad') scratch_rundir = os.path.join(scratch_path, rundir_basename) dqm_rundir_open = os.path.join(_dqm_base , rundir_basename, 'open') dqm_rundir = os.path.join(_dqm_base , rundir_basename) ecal_rundir_open = os.path.join(_ecal_base , rundir_basename, 'open') ecal_rundir = os.path.join(_ecal_base , rundir_basename) evd_rundir = os.path.join(_evd_base , rundir_basename) evd_rundir_open = os.path.join(_evd_base , rundir_basename, 'open') evd_eosrundir = os.path.join(_evd_eosbase, rundir_basename) lookarea_rundir_open = os.path.join(_lookarea_base , rundir_basename, 'open') lookarea_rundir = os.path.join(_lookarea_base , rundir_basename) run_key = runinfo.get_run_key(run_number) if not os.path.exists(scratch_rundir): mkdir(scratch_rundir) mkdir(os.path.join(scratch_rundir, 'bad')) if (not os.path.exists(new_rundir) and not run_key == 'TIER0_TRANSFER_OFF'): mkdir(new_rundir) mkdir(os.path.join(new_rundir, 'bad')) logger.info("Opening bookkeeping for run %d ..." % run_number) try: bookkeeper.open_run(cursor) connection.commit() except cx_Oracle.IntegrityError: logger.warning( 'Bookkeeping for run %d already open!' % run_number ) appversion = runinfo.get_cmssw_version(run_number) if appversion == 'UNKNOWN': logger.warning('The CMSSW version is UNKNOWN for run %d' %run_number) hlt_key = runinfo.get_hlt_key(run_number) # Sort JSON files by filename, implying also by lumi. jsns.sort() logger.info( "Processing {count} JSON file(s) in `{folder}':\n".format( count=len(jsns), folder=rundir ) + pprint.pformat([os.path.basename(f) for f in jsns]) ) for jsn_file in jsns: if ('BoLS' not in jsn_file and 'EoLS' not in jsn_file and 'index' not in jsn_file): settings_textI = open(jsn_file, "r").read() try: settings = json.loads(settings_textI) except ValueError: logger.warning("The json file %s is corrupted!" % jsn_file) maybe_move(jsn_file, new_rundir_bad, suffix='Corrupted') continue if len(settings['data']) < 5: logger.warning("Failed to parse `%s'!" % jsn_file) maybe_move(jsn_file, scratch_rundir, force_overwrite=True) continue eventsNumber = int(settings['data'][1]) fileName = str(settings['data'][3]) fileSize = int(settings['data'][4]) lumiSection = int(fileName.split('_')[1].strip('ls')) #streamName = str(fileName.split('_')[2].strip('stream')) streamName = str(fileName.split('_')[2].split('stream')[1]) if ( _checksum_status ): checksum_int = int(settings['data'][5]) checksum = format(checksum_int, 'x').zfill(8) #making sure it is 8 digits else: checksum = 0 dat_file = os.path.join(rundir, fileName) logger.info("The hex format checksum of the file {0} is {1} ".format(dat_file, checksum)) if streamName in _streams_with_scalers: monitor_rates(jsn_file) if streamName in _streams_to_postpone: continue if streamName in _streams_to_dqm: ## TODO: Use some other temporary directory instead of ## scratch if (fileSize > max_dqm_transfer_file_size): logger.warning( "`{0}' too large ({1} > {2})! ".format( dat_file, fileSize, max_dqm_transfer_file_size ) + "Moving it to bad area with the suffix `TooLarge' ..." ) maybe_move(jsn_file, new_rundir_bad, suffix='TooLarge') maybe_move(dat_file, new_rundir_bad, suffix='TooLarge') else: maybe_move(jsn_file, scratch_rundir) maybe_move(dat_file, scratch_rundir) jsn_file = jsn_file.replace(rundir, scratch_rundir) dat_file = dat_file.replace(rundir, scratch_rundir) args = [dat_file, jsn_file, dqm_rundir_open, dqm_rundir, lookarea_rundir_open,lookarea_rundir] dqm_pool.apply_async(double_p5_location, args) #args = [dat_file, jsn_file, dqm_rundir_open, dqm_rundir] #dqm_pool.apply_async(move_files, args) continue if streamName in _streams_to_ecal: ## TODO: Use some other temporary directory instead of ## scratch maybe_move(jsn_file, scratch_rundir) maybe_move(dat_file, scratch_rundir) jsn_file = jsn_file.replace(rundir, scratch_rundir) dat_file = dat_file.replace(rundir, scratch_rundir) args = [dat_file, jsn_file, ecal_rundir_open, ecal_rundir] ecal_pool.apply_async(move_files, args) continue #if streamName in _streams_to_lookarea: ## TODO: Use some other temporary directory instead of ## scratch # maybe_move(jsn_file, scratch_rundir) # maybe_move(dat_file, scratch_rundir) # jsn_file = jsn_file.replace(rundir, scratch_rundir) # dat_file = dat_file.replace(rundir, scratch_rundir) # args = [dat_file, jsn_file, lookarea_rundir_open, lookarea_rundir] # lookarea_pool.apply_async(move_files, args) # continue if streamName in _streams_to_evd: maybe_move(jsn_file, scratch_rundir, force_overwrite=True) maybe_move(dat_file, scratch_rundir, force_overwrite=True) jsn_file = jsn_file.replace(rundir, scratch_rundir) dat_file = dat_file.replace(rundir, scratch_rundir) # Dima said they don't need the open area args = [dat_file, jsn_file, evd_rundir_open, evd_rundir, evd_eosrundir] evd_pool.apply_async(copy_move_files, args) continue if (run_key == 'TIER0_TRANSFER_OFF' or streamName in (_streams_with_scalers + _streams_to_ignore)): maybe_move(jsn_file, scratch_rundir) maybe_move(dat_file, scratch_rundir) continue if (fileSize > max_tier0_transfer_file_size): logger.warning( "`{0}' too large ({1} > {2})! ".format( dat_file, fileSize, max_tier0_transfer_file_size ) + "Moving it to bad area with the suffix `TooLarge' ..." ) maybe_move(jsn_file, new_rundir_bad, suffix='TooLarge') maybe_move(dat_file, new_rundir_bad, suffix='TooLarge') continue starttime = int(os.stat(dat_file).st_atime) stoptime = int(os.stat(jsn_file).st_ctime) maybe_move(dat_file, new_rundir) maybe_move(jsn_file, new_rundir) ## Call the actual inject script if eventsNumber == 0: number_of_files = 0 else: number_of_files = 1 args_insert = [ './insertFile.pl', '--FILENAME' , fileName, '--FILECOUNTER' , 0, '--NEVENTS' , 0, '--FILESIZE' , 0, '--STARTTIME' , starttime, '--STOPTIME' , 0, '--STATUS' , 'open', '--RUNNUMBER' , run_number, '--LUMISECTION' , lumiSection, '--PATHNAME' , rundir, '--HOSTNAME' , hostname, '--SETUPLABEL' , setup_label, '--STREAM' , streamName, '--INSTANCE' , 1, '--SAFETY' , 0, '--APPVERSION' , appversion, '--APPNAME' , 'CMSSW', '--TYPE' , 'streamer', '--CHECKSUM' , checksum, '--CHECKSUMIND' , 0, ] args_close = [ './closeFile.pl', '--FILENAME' , fileName, '--FILECOUNTER' , 0, '--NEVENTS' , eventsNumber, '--FILESIZE' , fileSize, '--STARTTIME' , starttime, '--STOPTIME' , stoptime, '--STATUS' , 'closed', '--RUNNUMBER' , run_number, '--LUMISECTION' , lumiSection, '--PATHNAME' , new_rundir, '--HOSTNAME' , hostname, '--SETUPLABEL' , setup_label, '--STREAM' , streamName, '--INSTANCE' , 1, '--SAFETY' , 0, '--APPVERSION' , appversion, '--APPNAME' , 'CMSSW', '--TYPE' , 'streamer', '--DEBUGCLOSE' , 2, '--CHECKSUM' , checksum, '--CHECKSUMIND' , 0, ] inject_file_path = os.path.join( cfg.get('Output', 'inject_base'), '{date}-{hostname}.log'.format( date=date.today().strftime('%Y%m%d'), hostname=hostname, ) ) with open(inject_file_path, 'a') as inject_file: line = ' '.join(map(str, args_insert)) logger.info( "Appending line `%s' to `%s' ..." % ( line, inject_file_path ) ) inject_file.write(line + '\n') line = ' '.join(map(str, args_close)) logger.info( "Appending line `%s' to `%s' ..." % ( line, inject_file_path ) ) inject_file.write(line + '\n') try: bookkeeper.fill_number_of_files( cursor, streamName, lumiSection, number_of_files ) connection.commit() except cx_Oracle.IntegrityError: print ('WARNING: Failed to insert bookkeeping for ' + 'run {0}, stream {1}, ls {2}: #files = {3}').format( run_number, streamName, lumiSection, number_of_files ) ## Move the bad area to new run dir so that we can check for run ## completeness for fname in sorted(glob.glob(os.path.join(rundir_bad, '*.jsn'))): try: jsn = metafile.File(fname) if jsn.type == metafile.Type.MacroMerger: dat_path = jsn.path.replace('.jsn', '.dat') maybe_move(jsn.path, new_rundir_bad) maybe_move(dat_path, new_rundir_bad) except ValueError: logger.warning("Illegal filename `%s'!" % fname) connection.close()