def _setup(self):
db = config.load(self.db_config_file)
self.logger.info('Connecting to %s@%s ...' % (db.reader, db.sid))
self.connection = cx_Oracle.connect(db.reader, db.phrase, db.sid)
self.cursor = self.connection.cursor()
def iterate():
path = cfg.get('Input', 'path')
_scratch_base = cfg.get('Output', 'scratch_base')
_dqm_base = cfg.get('Output', 'dqm_base')
_ecal_base = cfg.get('Output', 'ecal_base')
_lookarea_base = cfg.get('Output', 'lookarea_base')
_evd_base = cfg.get('Output', 'evd_base')
_evd_eosbase = cfg.get('Output', 'evd_eosbase')
_checksum_status = cfg.getboolean('Misc', 'checksum_status')
setup_label = cfg.get('Input', 'setup_label')
db_config = cfg.get('Bookkeeping', 'db_config')
new_path_base = cfg.get('Output', 'new_path_base')
db_cred = config.load(db_config)
connection = cx_Oracle.connect(db_cred.db_user, db_cred.db_pwd,
db_cred.db_sid)
cursor = connection.cursor()
_streams_with_scalers = cfg.getlist('Streams', 'streams_with_scalars')
_streams_to_ecal = cfg.getlist('Streams', 'streams_to_ecal')
_streams_to_evd = cfg.getlist('Streams', 'streams_to_evd')
_streams_to_dqm = cfg.getlist('Streams', 'streams_to_dqm')
_streams_to_lookarea = cfg.getlist('Streams', 'streams_to_lookarea')
_streams_to_postpone = cfg.getlist('Streams', 'streams_to_postpone')
_streams_to_ignore = cfg.getlist('Streams', 'streams_to_ignore')
_renotify = cfg.getboolean('Misc', 'renotify')
max_tier0_transfer_file_size = cfg.getint(
'Output', 'maximum_tier0_transfer_file_size_in_bytes')
max_dqm_transfer_file_size = cfg.getint(
'Output', 'maximum_dqm_transfer_file_size_in_bytes')
hostname = socket.gethostname()
new_path = get_new_path(path, new_path_base)
scratch_path = get_new_path(path, _scratch_base)
rundirs, hltkeys = get_rundirs_and_hltkeys(path, new_path)
# Just for the MiniEoR files
for rundir in rundirs:
logger.debug("Inspecting `%s' for EoR searching..." % rundir)
jsns = sorted(glob.glob(os.path.join(rundir, '*EoR*.jsn')))
if not jsns:
continue
rundir_basename = os.path.basename(rundir)
run_number = int(rundir_basename.replace('run', ''))
new_rundir = os.path.join(new_path, rundir_basename)
scratch_rundir = os.path.join(scratch_path, rundir_basename)
jsns.sort()
run_key = runinfo.get_run_key(run_number)
for jsn_file in jsns:
if ('BoLS' not in jsn_file and 'EoLS' not in jsn_file
and 'index' not in jsn_file and 'EoR' in jsn_file):
if run_key == 'TIER0_TRANSFER_OFF':
maybe_move(jsn_file, scratch_rundir, force_overwrite=True)
else:
maybe_move(jsn_file, new_rundir, force_overwrite=True)
continue
# check if there are any stream directories
check_rundirs = []
for rundir in rundirs:
if not (glob.glob(os.path.join(rundir, 'stream*'))):
rundir = rundir
check_rundirs.append(rundir)
else:
for streamdir in glob.glob(os.path.join(rundir, 'stream*')):
check_rundirs.append(streamdir)
for rundir in check_rundirs:
logger.debug("Inspecting `%s' ..." % rundir)
jsns = sorted(glob.glob(os.path.join(rundir, '*.jsn')))
if not jsns:
continue
if 'stream' in rundir:
rundir_basename = os.path.basename(os.path.dirname(rundir))
else:
rundir_basename = os.path.basename(rundir)
run_number = int(rundir_basename.replace('run', ''))
logger.info('********** Run %d **********' % run_number)
bookkeeper._run_number = run_number
new_rundir = os.path.join(new_path, rundir_basename)
rundir_bad = os.path.join(rundir, 'bad')
new_rundir_bad = os.path.join(new_rundir, 'bad')
scratch_rundir = os.path.join(scratch_path, rundir_basename)
dqm_rundir_open = os.path.join(_dqm_base, rundir_basename, 'open')
dqm_rundir = os.path.join(_dqm_base, rundir_basename)
ecal_rundir_open = os.path.join(_ecal_base, rundir_basename, 'open')
ecal_rundir = os.path.join(_ecal_base, rundir_basename)
evd_rundir = os.path.join(_evd_base, rundir_basename)
evd_rundir_open = os.path.join(_evd_base, rundir_basename, 'open')
evd_eosrundir = os.path.join(_evd_eosbase, rundir_basename)
lookarea_rundir_open = os.path.join(_lookarea_base, rundir_basename,
'open')
lookarea_rundir = os.path.join(_lookarea_base, rundir_basename)
run_key = runinfo.get_run_key(run_number)
if not os.path.exists(scratch_rundir):
mkdir(scratch_rundir)
mkdir(os.path.join(scratch_rundir, 'bad'))
if (not os.path.exists(new_rundir)
and not run_key == 'TIER0_TRANSFER_OFF'):
mkdir(new_rundir)
mkdir(os.path.join(new_rundir, 'bad'))
logger.info("Opening bookkeeping for run %d ..." % run_number)
try:
bookkeeper.open_run(cursor)
connection.commit()
except cx_Oracle.IntegrityError:
logger.warning('Bookkeeping for run %d already open!' %
run_number)
appversion = runinfo.get_cmssw_version(run_number)
if appversion == 'UNKNOWN':
logger.warning('The CMSSW version is UNKNOWN for run %d' %
run_number)
hlt_key = runinfo.get_hlt_key(run_number)
# Sort JSON files by filename, implying also by lumi.
jsns.sort()
logger.info("Processing {count} JSON file(s) in `{folder}':\n".format(
count=len(jsns), folder=rundir) +
pprint.pformat([os.path.basename(f) for f in jsns]))
for jsn_file in jsns:
if ('BoLS' not in jsn_file and 'EoLS' not in jsn_file
and 'index' not in jsn_file):
settings_textI = open(jsn_file, "r").read()
try:
settings = json.loads(settings_textI)
except ValueError:
logger.warning("The json file %s is corrupted!" % jsn_file)
maybe_move(jsn_file, new_rundir_bad, suffix='Corrupted')
continue
if len(settings['data']) < 5:
logger.warning("Failed to parse `%s'!" % jsn_file)
maybe_move(jsn_file, scratch_rundir, force_overwrite=True)
continue
eventsNumber = int(settings['data'][1])
fileName = str(settings['data'][3])
fileSize = int(settings['data'][4])
lumiSection = int(fileName.split('_')[1].strip('ls'))
#streamName = str(fileName.split('_')[2].strip('stream'))
streamName = str(fileName.split('_')[2].split('stream')[1])
if (_checksum_status):
checksum_int = int(settings['data'][5])
checksum = format(checksum_int, 'x').zfill(
8) #making sure it is 8 digits
else:
checksum = 0
dat_file = os.path.join(rundir, fileName)
logger.info(
"The hex format checksum of the file {0} is {1} ".format(
dat_file, checksum))
if streamName in _streams_with_scalers:
monitor_rates(jsn_file)
if streamName in _streams_to_postpone:
continue
if streamName in _streams_to_dqm:
## TODO: Use some other temporary directory instead of
## scratch
if (fileSize > max_dqm_transfer_file_size):
logger.warning("`{0}' too large ({1} > {2})! ".format(
dat_file, fileSize, max_dqm_transfer_file_size
) + "Moving it to bad area with the suffix `TooLarge' ..."
)
maybe_move(jsn_file, new_rundir_bad, suffix='TooLarge')
maybe_move(dat_file, new_rundir_bad, suffix='TooLarge')
else:
maybe_move(jsn_file, scratch_rundir)
maybe_move(dat_file, scratch_rundir)
jsn_file = jsn_file.replace(rundir, scratch_rundir)
dat_file = dat_file.replace(rundir, scratch_rundir)
args = [
dat_file, jsn_file, dqm_rundir_open, dqm_rundir,
lookarea_rundir_open, lookarea_rundir
]
dqm_pool.apply_async(double_p5_location, args)
#args = [dat_file, jsn_file, dqm_rundir_open, dqm_rundir]
#dqm_pool.apply_async(move_files, args)
continue
if streamName in _streams_to_ecal:
## TODO: Use some other temporary directory instead of
## scratch
maybe_move(jsn_file, scratch_rundir)
maybe_move(dat_file, scratch_rundir)
jsn_file = jsn_file.replace(rundir, scratch_rundir)
dat_file = dat_file.replace(rundir, scratch_rundir)
args = [dat_file, jsn_file, ecal_rundir_open, ecal_rundir]
ecal_pool.apply_async(move_files, args)
continue
#if streamName in _streams_to_lookarea:
## TODO: Use some other temporary directory instead of
## scratch
# maybe_move(jsn_file, scratch_rundir)
# maybe_move(dat_file, scratch_rundir)
# jsn_file = jsn_file.replace(rundir, scratch_rundir)
# dat_file = dat_file.replace(rundir, scratch_rundir)
# args = [dat_file, jsn_file, lookarea_rundir_open, lookarea_rundir]
# lookarea_pool.apply_async(move_files, args)
# continue
if streamName in _streams_to_evd:
maybe_move(jsn_file, scratch_rundir, force_overwrite=True)
maybe_move(dat_file, scratch_rundir, force_overwrite=True)
jsn_file = jsn_file.replace(rundir, scratch_rundir)
dat_file = dat_file.replace(rundir, scratch_rundir)
# Dima said they don't need the open area
args = [
dat_file, jsn_file, evd_rundir_open, evd_rundir,
evd_eosrundir
]
evd_pool.apply_async(copy_move_files, args)
continue
if (run_key == 'TIER0_TRANSFER_OFF' or streamName
in (_streams_with_scalers + _streams_to_ignore)):
maybe_move(jsn_file, scratch_rundir)
maybe_move(dat_file, scratch_rundir)
continue
if (fileSize > max_tier0_transfer_file_size):
logger.warning(
"`{0}' too large ({1} > {2})! ".format(
dat_file, fileSize, max_tier0_transfer_file_size) +
"Moving it to bad area with the suffix `TooLarge' ...")
maybe_move(jsn_file, new_rundir_bad, suffix='TooLarge')
maybe_move(dat_file, new_rundir_bad, suffix='TooLarge')
continue
starttime = int(os.stat(dat_file).st_atime)
stoptime = int(os.stat(jsn_file).st_ctime)
maybe_move(dat_file, new_rundir)
maybe_move(jsn_file, new_rundir)
## Call the actual inject script
if eventsNumber == 0:
number_of_files = 0
else:
number_of_files = 1
args_insert = [
'./insertFile.pl',
'--FILENAME',
fileName,
'--FILECOUNTER',
0,
'--NEVENTS',
0,
'--FILESIZE',
0,
'--STARTTIME',
starttime,
'--STOPTIME',
0,
'--STATUS',
'open',
'--RUNNUMBER',
run_number,
'--LUMISECTION',
lumiSection,
'--PATHNAME',
rundir,
'--HOSTNAME',
hostname,
'--SETUPLABEL',
setup_label,
'--STREAM',
streamName,
'--INSTANCE',
1,
'--SAFETY',
0,
'--APPVERSION',
appversion,
'--APPNAME',
'CMSSW',
'--TYPE',
'streamer',
'--CHECKSUM',
checksum,
'--CHECKSUMIND',
0,
]
args_close = [
'./closeFile.pl',
'--FILENAME',
fileName,
'--FILECOUNTER',
0,
'--NEVENTS',
eventsNumber,
'--FILESIZE',
fileSize,
'--STARTTIME',
starttime,
'--STOPTIME',
stoptime,
'--STATUS',
'closed',
'--RUNNUMBER',
run_number,
'--LUMISECTION',
lumiSection,
'--PATHNAME',
new_rundir,
'--HOSTNAME',
hostname,
'--SETUPLABEL',
setup_label,
'--STREAM',
streamName,
'--INSTANCE',
1,
'--SAFETY',
0,
'--APPVERSION',
appversion,
'--APPNAME',
'CMSSW',
'--TYPE',
'streamer',
'--DEBUGCLOSE',
2,
'--CHECKSUM',
checksum,
'--CHECKSUMIND',
0,
]
inject_file_path = os.path.join(
cfg.get('Output', 'inject_base'),
'{date}-{hostname}.log'.format(
date=date.today().strftime('%Y%m%d'),
hostname=hostname,
))
with open(inject_file_path, 'a') as inject_file:
line = ' '.join(map(str, args_insert))
logger.info("Appending line `%s' to `%s' ..." %
(line, inject_file_path))
inject_file.write(line + '\n')
line = ' '.join(map(str, args_close))
logger.info("Appending line `%s' to `%s' ..." %
(line, inject_file_path))
inject_file.write(line + '\n')
try:
bookkeeper.fill_number_of_files(cursor, streamName,
lumiSection,
number_of_files)
connection.commit()
except cx_Oracle.IntegrityError:
print('WARNING: Failed to insert bookkeeping for ' +
'run {0}, stream {1}, ls {2}: #files = {3}').format(
run_number, streamName, lumiSection,
number_of_files)
## Move the bad area to new run dir so that we can check for run
## completeness
for fname in sorted(glob.glob(os.path.join(rundir_bad, '*.jsn'))):
try:
jsn = metafile.File(fname)
if jsn.type == metafile.Type.MacroMerger:
dat_path = jsn.path.replace('.jsn', '.dat')
maybe_move(jsn.path, new_rundir_bad)
maybe_move(dat_path, new_rundir_bad)
except ValueError:
logger.warning("Illegal filename `%s'!" % fname)
connection.close()
def _setup(self):
db = config.load(self.db_config_file)
self.logger.info('Connecting to %s@%s ...' % (db.reader, db.sid))
self.connection = cx_Oracle.connect(db.reader, db.phrase, db.sid)
self.cursor = self.connection.cursor()
def iterate():
path = cfg.get('Input', 'path')
_scratch_base = cfg.get('Output','scratch_base')
_dqm_base = cfg.get('Output','dqm_base')
_ecal_base = cfg.get('Output','ecal_base')
_lookarea_base = cfg.get('Output','lookarea_base')
_evd_base = cfg.get('Output','evd_base')
_evd_eosbase = cfg.get('Output','evd_eosbase')
_checksum_status = cfg.getboolean('Misc','checksum_status')
setup_label=cfg.get('Input','setup_label')
db_config = cfg.get('Bookkeeping', 'db_config')
new_path_base = cfg.get('Output', 'new_path_base')
db_cred = config.load(db_config)
connection = cx_Oracle.connect(db_cred.db_user, db_cred.db_pwd,
db_cred.db_sid)
cursor = connection.cursor()
_streams_with_scalers = cfg.getlist('Streams','streams_with_scalars')
_streams_to_ecal = cfg.getlist('Streams','streams_to_ecal' )
_streams_to_evd = cfg.getlist('Streams','streams_to_evd' )
_streams_to_dqm = cfg.getlist('Streams','streams_to_dqm' )
_streams_to_lookarea = cfg.getlist('Streams','streams_to_lookarea' )
_streams_to_postpone = cfg.getlist('Streams','streams_to_postpone' )
_streams_to_ignore = cfg.getlist('Streams','streams_to_ignore' )
_renotify = cfg.getboolean('Misc','renotify')
max_tier0_transfer_file_size = cfg.getint(
'Output', 'maximum_tier0_transfer_file_size_in_bytes'
)
max_dqm_transfer_file_size = cfg.getint(
'Output', 'maximum_dqm_transfer_file_size_in_bytes'
)
hostname = socket.gethostname()
new_path = get_new_path(path, new_path_base)
scratch_path = get_new_path(path, _scratch_base)
rundirs, hltkeys = get_rundirs_and_hltkeys(path, new_path)
# Just for the MiniEoR files
for rundir in rundirs:
logger.debug("Inspecting `%s' for EoR searching..." % rundir)
jsns = sorted(glob.glob(os.path.join(rundir, '*EoR*.jsn')))
if not jsns:
continue
rundir_basename = os.path.basename(rundir)
run_number = int(rundir_basename.replace('run', ''))
new_rundir = os.path.join(new_path , rundir_basename)
scratch_rundir = os.path.join(scratch_path, rundir_basename)
jsns.sort()
run_key = runinfo.get_run_key(run_number)
for jsn_file in jsns:
if ('BoLS' not in jsn_file and
'EoLS' not in jsn_file and
'index' not in jsn_file and
'EoR' in jsn_file):
if run_key == 'TIER0_TRANSFER_OFF':
maybe_move(jsn_file, scratch_rundir,
force_overwrite=True)
else:
maybe_move(jsn_file, new_rundir, force_overwrite=True)
continue
# check if there are any stream directories
check_rundirs = []
for rundir in rundirs:
if not (glob.glob(os.path.join(rundir, 'stream*'))):
rundir = rundir
check_rundirs.append(rundir)
else:
for streamdir in glob.glob(os.path.join(rundir, 'stream*')):
check_rundirs.append(streamdir)
for rundir in check_rundirs:
logger.debug("Inspecting `%s' ..." % rundir)
jsns = sorted(glob.glob(os.path.join(rundir, '*.jsn')))
if not jsns:
continue
if 'stream' in rundir:
rundir_basename = os.path.basename(os.path.dirname(rundir))
else:
rundir_basename = os.path.basename(rundir)
run_number = int(rundir_basename.replace('run', ''))
logger.info('********** Run %d **********' % run_number)
bookkeeper._run_number = run_number
new_rundir = os.path.join(new_path , rundir_basename)
rundir_bad = os.path.join(rundir , 'bad')
new_rundir_bad = os.path.join(new_rundir , 'bad')
scratch_rundir = os.path.join(scratch_path, rundir_basename)
dqm_rundir_open = os.path.join(_dqm_base , rundir_basename, 'open')
dqm_rundir = os.path.join(_dqm_base , rundir_basename)
ecal_rundir_open = os.path.join(_ecal_base , rundir_basename, 'open')
ecal_rundir = os.path.join(_ecal_base , rundir_basename)
evd_rundir = os.path.join(_evd_base , rundir_basename)
evd_rundir_open = os.path.join(_evd_base , rundir_basename, 'open')
evd_eosrundir = os.path.join(_evd_eosbase, rundir_basename)
lookarea_rundir_open = os.path.join(_lookarea_base , rundir_basename,
'open')
lookarea_rundir = os.path.join(_lookarea_base , rundir_basename)
run_key = runinfo.get_run_key(run_number)
if not os.path.exists(scratch_rundir):
mkdir(scratch_rundir)
mkdir(os.path.join(scratch_rundir, 'bad'))
if (not os.path.exists(new_rundir) and
not run_key == 'TIER0_TRANSFER_OFF'):
mkdir(new_rundir)
mkdir(os.path.join(new_rundir, 'bad'))
logger.info("Opening bookkeeping for run %d ..." % run_number)
try:
bookkeeper.open_run(cursor)
connection.commit()
except cx_Oracle.IntegrityError:
logger.warning(
'Bookkeeping for run %d already open!' % run_number
)
appversion = runinfo.get_cmssw_version(run_number)
if appversion == 'UNKNOWN':
logger.warning('The CMSSW version is UNKNOWN for run %d' %run_number)
hlt_key = runinfo.get_hlt_key(run_number)
# Sort JSON files by filename, implying also by lumi.
jsns.sort()
logger.info(
"Processing {count} JSON file(s) in `{folder}':\n".format(
count=len(jsns), folder=rundir
) + pprint.pformat([os.path.basename(f) for f in jsns])
)
for jsn_file in jsns:
if ('BoLS' not in jsn_file and
'EoLS' not in jsn_file and
'index' not in jsn_file):
settings_textI = open(jsn_file, "r").read()
try:
settings = json.loads(settings_textI)
except ValueError:
logger.warning("The json file %s is corrupted!" % jsn_file)
maybe_move(jsn_file, new_rundir_bad, suffix='Corrupted')
continue
if len(settings['data']) < 5:
logger.warning("Failed to parse `%s'!" % jsn_file)
maybe_move(jsn_file, scratch_rundir, force_overwrite=True)
continue
eventsNumber = int(settings['data'][1])
fileName = str(settings['data'][3])
fileSize = int(settings['data'][4])
lumiSection = int(fileName.split('_')[1].strip('ls'))
#streamName = str(fileName.split('_')[2].strip('stream'))
streamName = str(fileName.split('_')[2].split('stream')[1])
if ( _checksum_status ):
checksum_int = int(settings['data'][5])
checksum = format(checksum_int, 'x').zfill(8) #making sure it is 8 digits
else:
checksum = 0
dat_file = os.path.join(rundir, fileName)
logger.info("The hex format checksum of the file {0} is {1} ".format(dat_file, checksum))
if streamName in _streams_with_scalers:
monitor_rates(jsn_file)
if streamName in _streams_to_postpone:
continue
if streamName in _streams_to_dqm:
## TODO: Use some other temporary directory instead of
## scratch
if (fileSize > max_dqm_transfer_file_size):
logger.warning(
"`{0}' too large ({1} > {2})! ".format(
dat_file, fileSize, max_dqm_transfer_file_size
) +
"Moving it to bad area with the suffix `TooLarge' ..."
)
maybe_move(jsn_file, new_rundir_bad, suffix='TooLarge')
maybe_move(dat_file, new_rundir_bad, suffix='TooLarge')
else:
maybe_move(jsn_file, scratch_rundir)
maybe_move(dat_file, scratch_rundir)
jsn_file = jsn_file.replace(rundir, scratch_rundir)
dat_file = dat_file.replace(rundir, scratch_rundir)
args = [dat_file, jsn_file, dqm_rundir_open, dqm_rundir, lookarea_rundir_open,lookarea_rundir]
dqm_pool.apply_async(double_p5_location, args)
#args = [dat_file, jsn_file, dqm_rundir_open, dqm_rundir]
#dqm_pool.apply_async(move_files, args)
continue
if streamName in _streams_to_ecal:
## TODO: Use some other temporary directory instead of
## scratch
maybe_move(jsn_file, scratch_rundir)
maybe_move(dat_file, scratch_rundir)
jsn_file = jsn_file.replace(rundir, scratch_rundir)
dat_file = dat_file.replace(rundir, scratch_rundir)
args = [dat_file, jsn_file, ecal_rundir_open, ecal_rundir]
ecal_pool.apply_async(move_files, args)
continue
#if streamName in _streams_to_lookarea:
## TODO: Use some other temporary directory instead of
## scratch
# maybe_move(jsn_file, scratch_rundir)
# maybe_move(dat_file, scratch_rundir)
# jsn_file = jsn_file.replace(rundir, scratch_rundir)
# dat_file = dat_file.replace(rundir, scratch_rundir)
# args = [dat_file, jsn_file, lookarea_rundir_open, lookarea_rundir]
# lookarea_pool.apply_async(move_files, args)
# continue
if streamName in _streams_to_evd:
maybe_move(jsn_file, scratch_rundir, force_overwrite=True)
maybe_move(dat_file, scratch_rundir, force_overwrite=True)
jsn_file = jsn_file.replace(rundir, scratch_rundir)
dat_file = dat_file.replace(rundir, scratch_rundir)
# Dima said they don't need the open area
args = [dat_file, jsn_file, evd_rundir_open, evd_rundir,
evd_eosrundir]
evd_pool.apply_async(copy_move_files, args)
continue
if (run_key == 'TIER0_TRANSFER_OFF' or
streamName in (_streams_with_scalers +
_streams_to_ignore)):
maybe_move(jsn_file, scratch_rundir)
maybe_move(dat_file, scratch_rundir)
continue
if (fileSize > max_tier0_transfer_file_size):
logger.warning(
"`{0}' too large ({1} > {2})! ".format(
dat_file, fileSize, max_tier0_transfer_file_size
) +
"Moving it to bad area with the suffix `TooLarge' ..."
)
maybe_move(jsn_file, new_rundir_bad, suffix='TooLarge')
maybe_move(dat_file, new_rundir_bad, suffix='TooLarge')
continue
starttime = int(os.stat(dat_file).st_atime)
stoptime = int(os.stat(jsn_file).st_ctime)
maybe_move(dat_file, new_rundir)
maybe_move(jsn_file, new_rundir)
## Call the actual inject script
if eventsNumber == 0:
number_of_files = 0
else:
number_of_files = 1
args_insert = [
'./insertFile.pl',
'--FILENAME' , fileName,
'--FILECOUNTER' , 0,
'--NEVENTS' , 0,
'--FILESIZE' , 0,
'--STARTTIME' , starttime,
'--STOPTIME' , 0,
'--STATUS' , 'open',
'--RUNNUMBER' , run_number,
'--LUMISECTION' , lumiSection,
'--PATHNAME' , rundir,
'--HOSTNAME' , hostname,
'--SETUPLABEL' , setup_label,
'--STREAM' , streamName,
'--INSTANCE' , 1,
'--SAFETY' , 0,
'--APPVERSION' , appversion,
'--APPNAME' , 'CMSSW',
'--TYPE' , 'streamer',
'--CHECKSUM' , checksum,
'--CHECKSUMIND' , 0,
]
args_close = [
'./closeFile.pl',
'--FILENAME' , fileName,
'--FILECOUNTER' , 0,
'--NEVENTS' , eventsNumber,
'--FILESIZE' , fileSize,
'--STARTTIME' , starttime,
'--STOPTIME' , stoptime,
'--STATUS' , 'closed',
'--RUNNUMBER' , run_number,
'--LUMISECTION' , lumiSection,
'--PATHNAME' , new_rundir,
'--HOSTNAME' , hostname,
'--SETUPLABEL' , setup_label,
'--STREAM' , streamName,
'--INSTANCE' , 1,
'--SAFETY' , 0,
'--APPVERSION' , appversion,
'--APPNAME' , 'CMSSW',
'--TYPE' , 'streamer',
'--DEBUGCLOSE' , 2,
'--CHECKSUM' , checksum,
'--CHECKSUMIND' , 0,
]
inject_file_path = os.path.join(
cfg.get('Output', 'inject_base'),
'{date}-{hostname}.log'.format(
date=date.today().strftime('%Y%m%d'),
hostname=hostname,
)
)
with open(inject_file_path, 'a') as inject_file:
line = ' '.join(map(str, args_insert))
logger.info(
"Appending line `%s' to `%s' ..." % (
line, inject_file_path
)
)
inject_file.write(line + '\n')
line = ' '.join(map(str, args_close))
logger.info(
"Appending line `%s' to `%s' ..." % (
line, inject_file_path
)
)
inject_file.write(line + '\n')
try:
bookkeeper.fill_number_of_files(
cursor, streamName, lumiSection, number_of_files
)
connection.commit()
except cx_Oracle.IntegrityError:
print ('WARNING: Failed to insert bookkeeping for ' +
'run {0}, stream {1}, ls {2}: #files = {3}').format(
run_number, streamName, lumiSection,
number_of_files
)
## Move the bad area to new run dir so that we can check for run
## completeness
for fname in sorted(glob.glob(os.path.join(rundir_bad, '*.jsn'))):
try:
jsn = metafile.File(fname)
if jsn.type == metafile.Type.MacroMerger:
dat_path = jsn.path.replace('.jsn', '.dat')
maybe_move(jsn.path, new_rundir_bad)
maybe_move(dat_path, new_rundir_bad)
except ValueError:
logger.warning("Illegal filename `%s'!" % fname)
connection.close()
