def lumi_calc(opts, work_dn, jobnum_list, reader):
# Lumi filter calculations
(map_sample2run_info_dict, map_sample2input_events, map_sample2output_events) = process_jobs(opts,
work_dn, jobnum_list, reader)
activity = Activity('Simplifying lumi sections')
map_sample2run_lumi_range = {}
for sample in map_sample2run_info_dict:
for run in map_sample2run_info_dict[sample]:
for lumi in map_sample2run_info_dict[sample][run]:
map_sample2run_lumi_range.setdefault(sample, []).append(([run, lumi], [run, lumi]))
for sample in map_sample2run_info_dict:
map_sample2run_lumi_range[sample] = merge_lumi_list(map_sample2run_lumi_range[sample])
activity.finish()
for sample, lumi_list in map_sample2run_lumi_range.items():
if opts.job_events:
if map_sample2output_events.get(sample):
LOG.info('')
display_dict_list = lmap(lambda pfn: {0: pfn, 1: map_sample2output_events[sample][pfn]},
map_sample2output_events.get(sample, {}))
if display_dict_list:
display_dict_list.append('=')
display_dict_list += [{0: 'Processed in total', 1: map_sample2input_events.get(sample)}]
ConsoleTable.create([(0, ''), (1, '#Events')], display_dict_list,
title='Sample: %s' % sample)
if opts.job_json:
json_fn = os.path.join(opts.output_dir or work_dn, 'processed_%s.json' % sample)
with_file(SafeFile(json_fn, 'w'), partial(write_lumi_json, lumi_list))
LOG.info('Saved processed lumi sections in %s', json_fn)
if opts.job_gc:
LOG.info('\nList of processed lumisections\n' + '-' * 30)
write_lumi_gc(lumi_list)
LOG.info('')
def _get_psrc_user(self, pexpr, repository):
user_functions = {}
for cls_info in ParameterSource.get_class_info_list():
for cls_name in ifilter(lambda name: name != 'depth',
cls_info.keys()):
user_functions[cls_name] = partial(
ParameterSource.create_psrc_safe, cls_name,
self._parameter_config, repository)
try:
return eval(pexpr, dict(user_functions)) # pylint:disable=eval-used
except Exception:
self._log.warning('Available functions: %s',
sorted(user_functions.keys()))
raise
def lumi_calc(opts, work_dn, jobnum_list, reader):
# Lumi filter calculations
(map_sample2run_info_dict, map_sample2input_events,
map_sample2output_events) = process_jobs(opts, work_dn, jobnum_list,
reader)
activity = Activity('Simplifying lumi sections')
map_sample2run_lumi_range = {}
for sample in map_sample2run_info_dict:
for run in map_sample2run_info_dict[sample]:
for lumi in map_sample2run_info_dict[sample][run]:
map_sample2run_lumi_range.setdefault(sample, []).append(
([run, lumi], [run, lumi]))
for sample in map_sample2run_info_dict:
map_sample2run_lumi_range[sample] = merge_lumi_list(
map_sample2run_lumi_range[sample])
activity.finish()
for sample, lumi_list in map_sample2run_lumi_range.items():
if opts.job_events:
if map_sample2output_events.get(sample):
LOG.info('')
display_dict_list = lmap(
lambda pfn: {
0: pfn,
1: map_sample2output_events[sample][pfn]
}, map_sample2output_events.get(sample, {}))
if display_dict_list:
display_dict_list.append('=')
display_dict_list += [{
0: 'Processed in total',
1: map_sample2input_events.get(sample)
}]
ConsoleTable.create([(0, ''), (1, '#Events')],
display_dict_list,
title='Sample: %s' % sample)
if opts.job_json:
json_fn = os.path.join(opts.output_dir or work_dn,
'processed_%s.json' % sample)
with_file(SafeFile(json_fn, 'w'),
partial(write_lumi_json, lumi_list))
LOG.info('Saved processed lumi sections in %s', json_fn)
if opts.job_gc:
LOG.info('\nList of processed lumisections\n' + '-' * 30)
write_lumi_gc(lumi_list)
LOG.info('')
def save_to_disk(self, filename):
with_file(SafeFile(filename, 'w'), partial(pickle.dump, self))
def dump_dbs3_json(dn, block_dump_iter):
for block_dump in block_dump_iter:
block_dump_fn = block_dump['block']['block_name'].strip('/').replace(
'/', '_') + '.json'
with_file(SafeFile(os.path.join(dn, block_dump_fn), 'w'),
partial(json.dump, block_dump))
def save_to_disk(self, filename): with_file(SafeFile(filename, 'w'), partial(pickle.dump, self))
def dump_dbs3_json(dn, block_dump_iter):
for block_dump in block_dump_iter:
block_dump_fn = block_dump['block']['block_name'].strip('/').replace('/', '_') + '.json'
with_file(SafeFile(os.path.join(dn, block_dump_fn), 'w'), partial(json.dump, block_dump))