def merge_cleanup_final(self):
script_dir = self.config_prep['script_dir']
# start with script_dir; separate tar files for CPU* and TRAINOPT*
prefix = (f"CPU")
util.compress_files(+1, script_dir, f"{prefix}*", prefix, "")
prefix = (f"{TRAINOPT_STRING}")
util.compress_files(+1, script_dir, f"{prefix}*", prefix, "")
def merge_cleanup_final(self):
# every makeDataFiles succeeded, so here we simply compress output.
submit_info_yaml = self.config_prep['submit_info_yaml']
output_dir = self.config_prep['output_dir']
script_dir = submit_info_yaml['SCRIPT_DIR']
cwd = submit_info_yaml['CWD']
output_format = submit_info_yaml['OUTPUT_FORMAT']
isfmt_snana = (output_format == OUTPUT_FORMAT_SNANA)
isfmt_lsst_alert = (output_format == OUTPUT_FORMAT_LSST_ALERTS)
msgerr = []
if isfmt_snana:
command_list = [
'makeDataFiles.sh', '--outdir_snana', output_dir, '--merge'
]
ret = subprocess.run(command_list, capture_output=False, text=True)
elif isfmt_lsst_alert:
wildcard_base = f"{BASE_PREFIX}*.csv.gz"
wildcard = f"{script_dir}/{wildcard_base}"
combined_file = f"{output_dir}/ALERTS_TRUTH.csv.gz"
util.combine_csv_files(wildcard, combined_file, True)
# xxx nothing to compress after combining csv files
#print(f"\t Compress {wildcard_base}")
#sys.stdout.flush()
#util.compress_files(+1, script_dir, wildcard_base, "csv", "" )
else:
msgerr.append(f"Unknown format '{output_format}")
util.log_assert(False, msgerr) # just abort, no done stamp
#print(ret,' debugging')
wildcard_list = [
'MAKEDATA',
'CPU',
]
for w in wildcard_list:
wstar = f"{w}*"
tmp_list = glob.glob1(script_dir, wstar)
if len(tmp_list) == 0: continue
print(f"\t Compress {wstar}")
sys.stdout.flush()
util.compress_files(+1, script_dir, wstar, w, "")
# - - - -
# tar up entire script dir
util.compress_subdir(+1, script_dir)
def merge_cleanup_final(self):
output_dir = self.config_prep['output_dir']
submit_info_yaml = self.config_prep['submit_info_yaml']
script_dir = submit_info_yaml['SCRIPT_DIR']
jobfile_wildcard = submit_info_yaml['JOBFILE_WILDCARD']
script_subdir = SUBDIR_SCRIPTS_WFIT
self.make_wfit_summary()
logging.info(f" wfit cleanup: compress {JOB_SUFFIX_TAR_LIST}")
for suffix in JOB_SUFFIX_TAR_LIST :
wildcard = (f"{jobfile_wildcard}*.{suffix}")
util.compress_files(+1, script_dir, wildcard, suffix, "" )
logging.info("")
self.merge_cleanup_script_dir()
def merge_cleanup_final(self):
# every snlc_fit job succeeded, so here we simply compress output.
submit_info_yaml = self.config_prep['submit_info_yaml']
output_dir = self.config_prep['output_dir']
script_dir = submit_info_yaml['SCRIPT_DIR']
wildcard_list = ['TRAINOPT', 'CPU', 'CALIB_SHIFT']
for w in wildcard_list:
wstar = f"{w}*"
tmp_list = glob.glob1(script_dir, wstar)
if len(tmp_list) == 0: continue
print(f"\t Compress {wstar}")
util.compress_files(+1, script_dir, wstar, w, "")
# - - - -
# tar up entire script dir
util.compress_subdir(+1, script_dir)