def submit_batch_for_peds_aver(self) :
self.exportLocalPars() # export run_number to cp.str_run_number
if not self.job_can_be_submitted(self.job_id_peds_str, self.time_peds_job_submitted, 'peds') : return
self.time_peds_job_submitted = gu.get_time_sec()
self.command_for_peds_scan()
if not cfg.make_psana_cfg_file_for_peds_aver() :
self.stop_auto_processing(is_stop_on_button_click=False)
logger.warning('BATCH JOB IS NOT SUBMITTED !!!', __name__)
return False
if not self.is_good_lsf() :
self.stop_auto_processing(is_stop_on_button_click=False)
logger.warning('BATCH JOB IS NOT SUBMITTED !!!', __name__)
return False
#command = 'psana -c ' + fnm.path_peds_aver_psana_cfg() + ' ' + fnm.path_to_xtc_files_for_run() # fnm.path_dark_xtc_cond()
command = 'psana -c ' + fnm.path_peds_aver_psana_cfg() + self.opt + ' ' + fnm.path_to_xtc_files_for_run() # fnm.path_dark_xtc_cond()
queue = self.queue.value()
bat_log_file = fnm.path_peds_aver_batch_log()
self.job_id_peds_str, out, err = gu.batch_job_submit(command, queue, bat_log_file)
self.procDarkStatus ^= 2 # set bit to 1
if err != '' :
self.stop_auto_processing(is_stop_on_button_click=False)
logger.warning('Autoprocessing for run %s is stopped due to batch submission error!!!' % self.str_run_number, __name__)
logger.warning('BATCH JOB IS NOT SUBMITTED !!!', __name__)
return False
return True
def submit_batch_for_peds_aver(self):
self.exportLocalPars() # export run_number to cp.str_run_number
if not self.job_can_be_submitted(self.job_id_peds_str,
self.time_peds_job_submitted, 'peds'):
return
self.time_peds_job_submitted = gu.get_time_sec()
self.command_for_peds_scan()
cfg.make_psana_cfg_file_for_peds_aver()
command = 'psana -c ' + fnm.path_peds_aver_psana_cfg(
) + ' ' + fnm.path_to_xtc_files_for_run() # fnm.path_dark_xtc_cond()
queue = self.queue.value()
bat_log_file = fnm.path_peds_aver_batch_log()
self.job_id_peds_str, out, err = gu.batch_job_submit(
command, queue, bat_log_file)
self.procDarkStatus ^= 2 # set bit to 1
if err != '':
self.stop_auto_processing(is_stop_on_button_click=False)
logger.warning(
'Autoprocessing for run %s is stopped due to batch submission error!!!'
% self.str_run_number, __name__)
def make_psana_cfg_file_for_peds_aver(self):
self.path_in = apputils.AppDataPath(
'CorAna/scripts/psana-peds-aver.cfg').path()
self.path_out = fnm.path_peds_aver_psana_cfg()
self.d_subs = {
'FNAME_XTC': fnm.path_dark_xtc_cond(),
'SKIP': str(cp.bat_dark_start.value() - 1),
'EVENTS':
str(cp.bat_dark_end.value() - cp.bat_dark_start.value() + 1),
'IMG_REC_MODULE': str(cp.bat_img_rec_mod.value()),
'DETINFO': str(cp.bat_det_info.value()),
'FNAME_PEDS_AVE': fnm.path_pedestals_ave(),
'FNAME_PEDS_RMS': fnm.path_pedestals_rms()
}
self.d_subs['FNAME_HOTPIX_MASK'] = fnm.path_hotpix_mask()
self.d_subs['HOTPIX_THRESHOLD_ADU'] = str(cp.mask_hot_thr.value())
#if cp.mask_hot_is_used.value() :
# self.d_subs['FNAME_HOTPIX_MASK' ] = fnm.path_hotpix_mask()
# self.d_subs['HOTPIX_THRESHOLD_ADU'] = str( cp.mask_hot_thr.value() )
#else :
# self.d_subs['FNAME_HOTPIX_MASK' ] = ''
# self.d_subs['HOTPIX_THRESHOLD_ADU'] = '10000'
self.print_substitution_dict()
self.make_cfg_file()
def make_psana_cfg_file_for_peds_aver_v1 (self) :
self.path_in = apputils.AppDataPath('CorAna/scripts/psana-peds-aver.cfg').path()
self.path_out = fnm.path_peds_aver_psana_cfg()
self.d_subs = {'FNAME_XTC' : fnm.path_dark_xtc_cond(),
'SKIP' : str( cp.bat_dark_start.value() - 1 ),
'EVENTS' : str( cp.bat_dark_end.value() - cp.bat_dark_start.value() + 1 ),
'IMG_REC_MODULE' : str( cp.bat_img_rec_mod.value() ),
'DETINFO' : str( cp.bat_det_info.value() ),
'FNAME_PEDS_AVE' : fnm.path_pedestals_ave(),
'FNAME_PEDS_RMS' : fnm.path_pedestals_rms()
}
self.d_subs['FNAME_HOTPIX_MASK' ] = fnm.path_hotpix_mask()
self.d_subs['HOTPIX_THRESHOLD_ADU'] = str( cp.mask_hot_thr.value() )
#if cp.mask_hot_is_used.value() :
# self.d_subs['FNAME_HOTPIX_MASK' ] = fnm.path_hotpix_mask()
# self.d_subs['HOTPIX_THRESHOLD_ADU'] = str( cp.mask_hot_thr.value() )
#else :
# self.d_subs['FNAME_HOTPIX_MASK' ] = ''
# self.d_subs['HOTPIX_THRESHOLD_ADU'] = '10000'
#self.print_substitution_dict()
#self.make_cfg_file()
txt_cfg = self.text_for_section()
self.save_cfg_file(txt_cfg, self.path_out)
def command_for_peds_aver(self):
if not cfg.make_psana_cfg_file_for_peds_aver():
logger.warning('INTERACTIVE JOB IS NOT STARTED !!!', __name__)
return False
command = 'psana -c ' + fnm.path_peds_aver_psana_cfg(
) # + ' ' + fnm.path_to_xtc_files_for_run() # fnm.path_dark_xtc_cond()
command_seq = command.split()
msg = 'Avereging xtc file(s) using command:\n%s' % command \
+ '\nand save results in the log-file: %s' % fnm.path_peds_aver_batch_log()
logger.info(msg, __name__)
err = gu.subproc_in_log(
command_seq, fnm.path_peds_aver_batch_log()) # , shell=True)
if err != '':
logger.warning(
'\nWarning/error message from subprocess:\n%s' % (err),
__name__)
return False
else:
logger.info(
'Avereging for run %s is completed' % self.str_run_number,
__name__)
return True
def make_psana_cfg_file_for_peds_aver (self) :
self.path_in = apputils.AppDataPath('CorAna/scripts/psana-peds-aver.cfg').path()
self.path_out = fnm.path_peds_aver_psana_cfg()
self.d_subs = {'SKIP' : str( cp.bat_dark_start.value() - 1 ),
'EVENTS' : str( cp.bat_dark_end.value() - cp.bat_dark_start.value() + 1 ),
'IMG_REC_MODULE' : str( cp.bat_img_rec_mod.value() ),
'DETINFO' : str( cp.bat_det_info.value() ),
'FILE_AVE' : fnm.path_pedestals_ave(),
'FILE_RMS' : fnm.path_pedestals_rms()
}
self.print_substitution_dict()
self.make_cfg_file()
def make_psana_cfg_file_for_peds_aver (self) :
self.str_of_modules = ''
self.txt_cfg_header = '# Autogenerated config file for psana\n' \
+ '# Useful command:\n' \
+ '# psana -m EventKeys -n 5 ' + self.path_to_data_files() \
+ '\n'
self.txt_cfg_body = '\n'
self.add_cfg_module_tahometer()
self.cfg_file_body_for_peds_aver()
self.cfg_file_header()
self.save_cfg_file(self.txt_cfg_header + self.txt_cfg_body, fnm.path_peds_aver_psana_cfg())
def make_psana_cfg_file_for_peds_aver (self) :
self.path_in = apputils.AppDataPath('CorAna/scripts/psana-peds-aver.cfg').path()
self.path_out = fnm.path_peds_aver_psana_cfg()
self.d_subs = {'FNAME_XTC' : fnm.path_dark_xtc_cond(),
'SKIP' : str( cp.bat_dark_start.value() - 1 ),
'EVENTS' : str( cp.bat_dark_end.value() - cp.bat_dark_start.value() + 1 ),
'IMG_REC_MODULE' : str( cp.bat_img_rec_mod.value() ),
'DETINFO' : str( cp.bat_det_info.value() ),
'FNAME_PEDS_AVE' : fnm.path_pedestals_ave(),
'FNAME_PEDS_RMS' : fnm.path_pedestals_rms()
}
self.print_substitution_dict()
self.make_cfg_file()
def make_psana_cfg_file_for_peds_aver (self) :
self.str_of_modules = ''
self.txt_cfg_header = '# Autogenerated config file for psana\n' \
+ '# Useful command:\n' \
+ '# psana -m EventKeys -n 5 ' + self.path_to_data_files() \
+ '\n'
self.txt_cfg_body = '\n'
self.add_cfg_module_tahometer()
self.cfg_file_body_for_peds_aver()
self.cfg_file_header()
self.save_cfg_file(self.txt_cfg_header + self.txt_cfg_body, fnm.path_peds_aver_psana_cfg())
def make_psana_cfg_file_for_peds_aver (self) :
self.str_of_modules = ''
self.txt_cfg_header = '# Autogenerated config file for dark average\n' \
+ '# Useful command:\n' \
+ '# psana -m EventKeys -n 5 ' + fnm.path_data_xtc_cond() \
+ '\n'
self.txt_cfg_body = '\n\n'
self.add_cfg_module_tahometer()
self.add_cfg_module_img_producer()
self.add_cfg_module_ndarraverage_for_peds_aver()
self.cfg_file_header_for_peds_aver()
self.save_cfg_file(self.txt_cfg_header + self.txt_cfg_body, fnm.path_peds_aver_psana_cfg())
def make_psana_cfg_file_for_peds_aver(self):
self.str_of_modules = ''
self.txt_cfg_header = '# Autogenerated config file for dark average\n' \
+ '# Useful command:\n' \
+ '# psana -m EventKeys -n 5 ' + fnm.path_data_xtc_cond() \
+ '\n'
self.txt_cfg_body = '\n\n'
self.add_cfg_module_tahometer()
self.add_cfg_module_img_producer()
self.add_cfg_module_ndarraverage_for_peds_aver()
self.cfg_file_header_for_peds_aver()
self.save_cfg_file(self.txt_cfg_header + self.txt_cfg_body,
fnm.path_peds_aver_psana_cfg())
def make_psana_cfg_file_for_peds_aver_cspad(
self, module='cspad_mod.CsPadPedestals'):
self.path_in = apputils.AppDataPath(
'CalibManager/scripts/psana-section-psana.cfg').path()
self.path_out = fnm.path_peds_aver_psana_cfg()
modules = ''
for i in range(len(cp.blsp.list_of_sources)):
modules += '%s:%i ' % (module, i)
#print 'List of modules: %s' % modules
self.d_subs = {
'FNAME_XTC': str(fnm.path_to_xtc_files_for_run()),
'SKIP': str(cp.bat_dark_start.value() - 1),
'EVENTS':
str(cp.bat_dark_end.value() - cp.bat_dark_start.value() + 1),
'MODULES': modules
}
self.print_substitution_dict()
self.make_cfg_file()
#Add a few similar modules in loop
self.path_in = apputils.AppDataPath(
'CalibManager/scripts/psana-section-module-cspad-peds.cfg').path()
list_of_ave = cp.blsp.get_list_of_files_for_all_sources(
fnm.path_peds_ave())
list_of_rms = cp.blsp.get_list_of_files_for_all_sources(
fnm.path_peds_rms())
for i, (source, fname_ave, fname_rms) in enumerate(
zip(cp.blsp.list_of_sources, list_of_ave, list_of_rms)):
mod = '%s:%i' % (module, i)
#print ' Add module with pars: ', mod, source, fname_ave, fname_rms
self.d_subs = {
'MODULE': mod,
'DETINFO': source,
'FNAME_PEDS_AVE': fname_ave,
'FNAME_PEDS_RMS': fname_rms
}
#for item in self.d_subs.items() :
# print '%20s : %s' % item
self.make_cfg_file(fout_mode='a')
def submit_batch_for_peds_aver(self):
self.exportLocalPars() # export run_number to cp.str_run_number
if not self.job_can_be_submitted(self.job_id_peds_str,
self.time_peds_job_submitted, 'peds'):
return
self.time_peds_job_submitted = gu.get_time_sec()
self.command_for_peds_scan()
if not cfg.make_psana_cfg_file_for_peds_aver():
self.stop_auto_processing(is_stop_on_button_click=False)
logger.warning('BATCH JOB IS NOT SUBMITTED !!!', __name__)
return False
if not self.is_good_lsf():
self.stop_auto_processing(is_stop_on_button_click=False)
logger.warning('BATCH JOB IS NOT SUBMITTED !!!', __name__)
return False
#command = 'psana -c ' + fnm.path_peds_aver_psana_cfg() + ' ' + fnm.path_to_xtc_files_for_run() # fnm.path_dark_xtc_cond()
command = 'psana -c ' + fnm.path_peds_aver_psana_cfg(
) + self.opt + ' ' + fnm.path_to_xtc_files_for_run(
) # fnm.path_dark_xtc_cond()
queue = self.queue.value()
bat_log_file = fnm.path_peds_aver_batch_log()
self.job_id_peds_str, out, err = gu.batch_job_submit(
command, queue, bat_log_file)
self.procDarkStatus ^= 2 # set bit to 1
if err != 'Warning: job being submitted without an AFS token.':
logger.warning(
'This job is running on LCLS NFS, it does not need in AFS, ignore warning and continue.',
__name__)
return True
elif err != '':
self.stop_auto_processing(is_stop_on_button_click=False)
logger.warning(
'Autoprocessing for run %s is stopped due to batch submission error!!!'
% self.str_run_number, __name__)
logger.warning('BATCH JOB IS NOT SUBMITTED !!!', __name__)
return False
return True
def submit_batch_for_peds_aver(self) :
self.exportLocalPars() # export run_number to cp.str_run_number
if not self.job_can_be_submitted(self.job_id_peds_str, self.time_peds_job_submitted, 'peds') : return
self.time_peds_job_submitted = gu.get_time_sec()
self.make_psana_cfg_file_for_peds_aver()
command = 'psana -c ' + fnm.path_peds_aver_psana_cfg() + ' ' + fnm.path_to_xtc_files_for_run() # fnm.path_dark_xtc_cond()
queue = self.queue
bat_log_file = fnm.path_peds_aver_batch_log()
self.job_id_peds_str, out, err = gu.batch_job_submit(command, queue, bat_log_file)
self.procDarkStatus ^= 2 # set bit to 1
if err != '' :
self.stop_auto_processing(is_stop_on_button_click=False)
logger.warning('Autoprocessing is stopped due to batch submission error!!!', __name__)
def command_for_peds_aver(self) :
if not cfg.make_psana_cfg_file_for_peds_aver() :
logger.warning('INTERACTIVE JOB IS NOT STARTED !!!', __name__)
return False
command = 'psana -c ' + fnm.path_peds_aver_psana_cfg() # + ' ' + fnm.path_to_xtc_files_for_run() # fnm.path_dark_xtc_cond()
command_seq = command.split()
msg = 'Avereging xtc file(s) using command:\n%s' % command \
+ '\nand save results in the log-file: %s' % fnm.path_peds_aver_batch_log()
logger.info(msg, __name__)
err = gu.subproc_in_log(command_seq, fnm.path_peds_aver_batch_log()) # , shell=True)
if err != '' :
logger.warning('\nWarning/error message from subprocess:\n%s' % (err), __name__)
return False
else :
logger.info('Avereging for run %s is completed' % self.str_run_number, __name__)
return True
def make_psana_cfg_file_for_peds_aver_cspad (self, module='cspad_mod.CsPadPedestals') :
self.path_in = apputils.AppDataPath('CalibManager/scripts/psana-section-psana.cfg').path()
self.path_out = fnm.path_peds_aver_psana_cfg()
modules = ''
for i in range( len(cp.blsp.list_of_sources) ) : modules += '%s:%i ' % (module, i)
#print 'List of modules: %s' % modules
self.d_subs = {'FNAME_XTC' : str(fnm.path_to_xtc_files_for_run()),
'SKIP' : str( cp.bat_dark_start.value() - 1 ),
'EVENTS' : str( cp.bat_dark_end.value() - cp.bat_dark_start.value() + 1 ),
'MODULES' : modules
}
self.print_substitution_dict()
self.make_cfg_file()
#Add a few similar modules in loop
self.path_in = apputils.AppDataPath('CalibManager/scripts/psana-section-module-cspad-peds.cfg').path()
list_of_ave = cp.blsp.get_list_of_files_for_all_sources(fnm.path_peds_ave())
list_of_rms = cp.blsp.get_list_of_files_for_all_sources(fnm.path_peds_rms())
for i, (source, fname_ave, fname_rms) in enumerate( zip(cp.blsp.list_of_sources, list_of_ave, list_of_rms) ) :
mod = '%s:%i' % (module, i)
#print ' Add module with pars: ', mod, source, fname_ave, fname_rms
self.d_subs = {'MODULE' : mod,
'DETINFO' : source,
'FNAME_PEDS_AVE' : fname_ave,
'FNAME_PEDS_RMS' : fname_rms
}
#for item in self.d_subs.items() :
# print '%20s : %s' % item
self.make_cfg_file(fout_mode='a')
def make_psana_cfg_file_for_peds_aver_princeton (self) :
self.path_in = apputils.AppDataPath('CalibManager/scripts/psana-peds-aver-princeton.cfg').path()
self.path_out = fnm.path_peds_aver_psana_cfg()
self.d_subs = {'FNAME_XTC' : str(fnm.path_to_xtc_files_for_run()),
'SKIP' : str( cp.bat_dark_start.value() - 1 ),
'EVENTS' : str( cp.bat_dark_end.value() - cp.bat_dark_start.value() + 1 ),
'IMG_REC_MODULE' : str( cp.bat_img_rec_mod.value() ),
'DETINFO' : str( cp.bat_det_info.value() ),
'FNAME_PEDS_AVE' : fnm.path_peds_ave(),
'FNAME_PEDS_RMS' : fnm.path_peds_rms()
}
self.d_subs['FNAME_HOTPIX_MASK' ] = fnm.path_hotpix_mask()
self.d_subs['HOTPIX_THRESHOLD_ADU'] = str( cp.mask_hot_thr.value() )
#if cp.mask_hot_is_used.value() :
# self.d_subs['FNAME_HOTPIX_MASK' ] = fnm.path_hotpix_mask()
# self.d_subs['HOTPIX_THRESHOLD_ADU'] = str( cp.mask_hot_thr.value() )
#else :
# self.d_subs['FNAME_HOTPIX_MASK' ] = ''
# self.d_subs['HOTPIX_THRESHOLD_ADU'] = '10000'
self.print_substitution_dict()
self.make_cfg_file()
