def test_mkdir_check_args():
dirs = [test_file_path, "path_test", "nested", "and_then"]
u.rm_check(os.path.join(test_file_path, "path_test"))
path_out = os.path.join(*dirs)
path_test = u.mkdir_check_args(*dirs)
assert path_out == path_test
assert os.path.isdir(path_test)
shutil.rmtree(os.path.join(test_file_path, "path_test"))
def test_mkdir_check():
path = os.path.join(test_file_path, "path_test")
u.rm_check(path)
u.mkdir_check(path)
assert os.path.isdir(path)
u.rmtree_check(path)
paths = [
os.path.join(test_file_path, "test_path_1"),
os.path.join(test_file_path, "test_path_2"),
os.path.join(test_file_path, "test_path_3")
]
u.mkdir_check_args(*paths)
for path in paths:
assert os.path.isdir(path)
u.rmtree_check(path)
def test_mkdir_check_nested():
u.debug_level = 2
path_test = os.path.join(test_file_path, "path_test", "nested", "and_then")
u.rm_check(os.path.join(test_file_path, "path_test"))
u.mkdir_check_nested(path_test, remove_last=False)
assert os.path.isdir(path_test)
u.rmtree_check(os.path.join(test_file_path, "path_test"))
path_test = os.path.join(test_file_path, "path_test", "nested", "and_then", "/")
u.mkdir_check_nested(path_test)
assert os.path.isdir(path_test)
u.rmtree_check(os.path.join(test_file_path, "path_test"))
path_test = os.path.join(test_file_path, "path_test", "nested", "and_then", "gaia.csv")
u.mkdir_check_nested(path_test)
assert os.path.isdir(os.path.split(path_test)[0])
assert not os.path.isfile(path_test)
u.rmtree_check(os.path.join(test_file_path, "path_test"))
def main(data_title: 'str'):
print("\nExecuting Python script pipeline_fors2/1-initial.py, with:")
print(f"\tepoch {data_title}")
print()
epoch_params = p.object_params_fors2(obj=data_title)
data_dir = epoch_params['data_dir']
output_dir = data_dir + "/0-data_with_raw_calibs/"
# Write tables of fits files to main directory; firstly, science images only:
table = fits_table(input_path=output_dir,
output_path=data_dir + data_title + "_fits_table_science.csv",
science_only=True)
# Then including all calibration files
table_full = fits_table(input_path=output_dir,
output_path=data_dir + data_title + "_fits_table_all.csv",
science_only=False)
fits_table_all(input_path=output_dir,
output_path=data_dir + data_title + "_fits_table_detailled.csv",
science_only=False)
# Clear output files for fresh start.
u.rm_check(data_dir + '/output_values.yaml')
u.rm_check(data_dir + '/output_values.json')
# Collect list of filters used:
filters = []
columns = []
for j in [1, 2, 3, 4, 5]:
column = 'filter' + str(j)
for name in table[column]:
if name != 'free':
if name not in filters:
filters.append(name)
columns.append(column)
# Collect pointings of standard-star observations.
std_ras = []
std_decs = []
std_pointings = []
# TODO: This is a horrible way to do this. Take the time to find a better one.
for ra in table_full[table_full['object'] == 'STD']['ref_ra']:
if ra not in std_ras:
std_ras.append(ra)
for dec in table_full[table_full['object'] == 'STD']['ref_dec']:
if dec not in std_decs:
std_decs.append(dec)
for i, ra in enumerate(std_ras):
std_pointings.append(f'RA{ra}_DEC{std_decs[i]}')
print(std_ras)
print(std_decs)
print(std_pointings)
# Collect and save some stats on those filters:
param_dict = {}
exp_times = []
ns_exposures = []
param_dict['filters'] = filters
param_dict['object'] = table['object'][0]
param_dict['obs_name'] = table['obs_name'][0]
mjd = param_dict['mjd_obs'] = float(table['mjd_obs'][0])
for i, f in enumerate(filters):
f_0 = f[0]
exp_time = table['exp_time'][table[columns[i]] == f]
exp_times.append(exp_time)
airmass_col = table['airmass'][table[columns[i]] == f]
n_frames = sum(table[columns[i]] == f)
n_exposures = n_frames / 2
ns_exposures.append(n_exposures)
airmass = float(np.nanmean(airmass_col))
param_dict[f_0 + '_exp_time_mean'] = float(np.nanmean(exp_time))
param_dict[f_0 + '_exp_time_err'] = float(2 * np.nanstd(exp_time))
param_dict[f_0 + '_airmass_mean'] = airmass
param_dict[f_0 + '_airmass_err'] = float(
max(np.nanmax(airmass_col) - airmass, airmass - np.nanmin(airmass_col)))
param_dict[f_0 + '_n_frames'] = float(n_frames)
param_dict[f_0 + '_n_exposures'] = float(n_exposures)
param_dict[f_0 + '_mjd_obs'] = float(np.nanmean(table['mjd_obs'][table[columns[i]] == f]))
std_filter_dir = f'{data_dir}calibration/std_star/{f}/'
u.mkdir_check(std_filter_dir)
print(f'Copying {f} calibration data to std_star folder...')
# Sort the STD files by filter, and within that by pointing.
for j, ra in enumerate(std_ras):
at_pointing = False
pointing = std_pointings[j]
pointing_dir = std_filter_dir + pointing + '/'
for file in \
table_full[
(table_full['object'] == 'STD') &
(table_full['ref_ra'] == ra) &
(table_full[columns[i]] == f)]['identifier']:
at_pointing = True
u.mkdir_check(pointing_dir)
shutil.copyfile(output_dir + file, pointing_dir + file)
if at_pointing:
for file in table_full[table_full['object'] == 'BIAS']['identifier']:
shutil.copyfile(output_dir + file, pointing_dir + file)
for file in table_full[(table_full['object'] == 'FLAT,SKY') & (table_full[columns[i]] == f)][
'identifier']:
shutil.copyfile(output_dir + file, pointing_dir + file)
p.add_output_values(obj=data_title, params=param_dict)
if "new_epoch" in data_dir:
mjd = f"MJD{int(float(mjd))}"
new_data_dir = data_dir.replace("new_epoch", mjd)
p.add_epoch_param(obj=data_title, params={"data_dir": new_data_dir})
def main(output_dir: 'str', data_title: 'str'):
data_dir = output_dir + "/0-data_with_raw_calibs/"
# Write tables of fits files to main directory; firstly, science images only:
table = fits_table_all(input_path=data_dir,
output_path=output_dir + data_title +
"_fits_table_science.csv",
science_only=True)
# Then including all calibration files
table_full = fits_table_all(input_path=data_dir,
output_path=output_dir + data_title +
"_fits_table_all.csv",
science_only=False)
# Clear output files for fresh start.
u.rm_check(output_dir + '/output_values.yaml')
u.rm_check(output_dir + '/output_values.json')
# Collect list of filters used:
filters = []
for name in table['ESO INS FILT1 NAME']:
if name != 'free':
if name not in filters:
filters.append(name)
# Collect pointings of standard-star observations.
std_ras = []
std_decs = []
std_pointings = []
# TODO: This is a horrible way to do this. Take the time to find a better one.
for ra in table_full[table_full['OBJECT'] == 'STD']['CRVAL1']:
if ra not in std_ras:
std_ras.append(ra)
for dec in table_full[table_full['OBJECT'] == 'STD']['CRVAL1']:
if dec not in std_decs:
std_decs.append(dec)
for i, ra in enumerate(std_ras):
std_pointings.append(f'RA{ra}_DEC{std_decs[i]}')
# Collect and save some stats on those filters:
param_dict = {}
exp_times = []
ns_exposures = []
param_dict['filters'] = filters
for i, f in enumerate(filters):
f_0 = f[0]
exp_time = table['EXPTIME'][table['ESO INS FILT1 NAME'] == f]
exp_times.append(exp_time)
airmass = table['AIRMASS'][table['ESO INS FILT1 NAME'] == f]
n_frames = sum(table['ESO INS FILT1 NAME'] == f)
n_exposures = n_frames / 2
ns_exposures.append(n_exposures)
param_dict[f_0 + '_exp_time_mean'] = float(np.nanmean(exp_time))
param_dict[f_0 + '_exp_time_err'] = float(2 * np.nanstd(exp_time))
param_dict[f_0 + '_airmass_mean'] = float(np.nanmean(airmass))
param_dict[f_0 + '_airmass_err'] = float(
max(
np.nanmax(airmass) - np.nanmean(airmass),
(np.nanmean(airmass) - np.nanmin(airmass))))
param_dict[f_0 + '_n_frames'] = float(n_frames)
param_dict[f_0 + '_n_exposures'] = float(n_exposures)
std_filter_dir = f'{output_dir}calibration/std_star/{f}/'
u.mkdir_check(std_filter_dir)
print(f'Copying {f} calibration data to std_star folder...')
# Sort the STD files by filter, and within that by pointing.
for j, ra in enumerate(std_ras):
at_pointing = False
pointing = std_pointings[j]
pointing_dir = std_filter_dir + pointing + '/'
for file in \
table_full[
(table_full['OBJECT'] == 'STD') &
(table_full['CRVAL1'] == ra) &
(table_full['ESO INS FILT1 NAME'] == f)]['ARCFILE']:
at_pointing = True
u.mkdir_check(pointing_dir)
shutil.copyfile(data_dir + file, pointing_dir + file)
if at_pointing:
for file in table_full[table_full['object'] ==
'BIAS']['ARCFILE']:
shutil.copyfile(data_dir + file, pointing_dir + file)
for file in table_full[(table_full['object'] == 'FLAT,SKY') & (
table_full['ESO INS FILT1 NAME'] == f)]['ARCFILE']:
shutil.copyfile(data_dir + file, pointing_dir + file)
p.add_params(output_dir + '/output_values', param_dict)