def compare_catalog_xmatch_between_reductions():
params = get_args()
log = logs.start_stage_log(params['log_dir'], 'compare_crossmatch')
meta1 = metadata.MetaData()
meta1.load_all_metadata(params['red_dir1'], 'pyDANDIA_metadata.fits')
log.info('Loaded metadata from ' + params['red_dir1'])
meta2 = metadata.MetaData()
meta2.load_all_metadata(params['red_dir2'], 'pyDANDIA_metadata.fits')
log.info('Loaded metadata from ' + params['red_dir2'])
# Crossmatch the catalogs by x, y pixel positions
(meta1_matches,
meta2_matches) = crossmatch_pixel_positions_per_star(meta1,
meta2,
log,
threshold=1.0)
matched_data = build_matched_arrays(meta1, meta2, meta1_matches,
meta2_matches, log)
matched_data = calculate_separations_on_sky(matched_data, log)
# For each matching star, compare RA, Dec and Gaia ID if available
compare_coordinates(params, matched_data, log)
compare_gaia_ids(params, matched_data, log)
logs.close_log(log)
def fetch_metadata(setup,params,log):
"""Function to extract the information necessary for the photometric
calibration from a metadata file, adding information to the params
dictionary"""
reduction_metadata = metadata.MetaData()
reduction_metadata.load_a_layer_from_file( setup.red_dir,
params['metadata'],
'data_architecture' )
reduction_metadata.load_a_layer_from_file( setup.red_dir,
params['metadata'],
'star_catalog' )
reduction_metadata.load_a_layer_from_file( setup.red_dir,
params['metadata'],
'phot_calib' )
params['refimage'] = reduction_metadata.data_architecture[1]['REF_IMAGE'][0]
log.info('Using data for reference image: '+params['refimage'])
star_catalog = Table()
star_catalog['star_index'] = reduction_metadata.star_catalog[1]['star_index']
star_catalog['mag'] = reduction_metadata.star_catalog[1]['ref_mag']
star_catalog['mag_err'] = reduction_metadata.star_catalog[1]['ref_mag_err']
star_catalog['cal_ref_mag'] = reduction_metadata.phot_calib[1]['cal_ref_mag']
star_catalog['cal_ref_mag_err'] = reduction_metadata.phot_calib[1]['cal_ref_mag_err']
log.info('Extracted star catalog')
return reduction_metadata, params, star_catalog
def has_ref_image_been_selected(data_dir):
stage2_log = path.join(data_dir, 'stage2.log')
found_log = path.isfile(stage2_log)
if found_log:
found_string = find_entry_in_log(log_file, 'No reference image found')
else:
found_string = False
reduction_metadata = metadata.MetaData()
reduction_metadata.load_all_metadata(setup.red_dir, 'pyDANDIA_metadata.fits')
found_meta = 'REF_PATH' in reduction_metadata.data_architecture[1].keys()
found_dir = path.isdir(data_dir, 'ref')
if found_dir:
image_list = glob.glob(data_dir, 'ref', '*fits')
found_image = len(image_list) >= 1
else:
found_image = False
if found_log and found_string and found_meta and found_dir and found_image:
return True
else:
return False
def create_or_load_the_reduction_metadata(
setup,
output_metadata_directory,
metadata_name='pyDANDIA_metadata.fits',
log=None):
'''
This creates (new reduction) or load (ongoing reduction) the metadata file linked to this reduction.
:param string output_metadata_directory: the directory where to place the metadata
:param string metadata_name: the name of the metadata file
:param boolean verbose: switch to True to have more informations
:return: the metadata object
:rtype: metadata object
'''
try:
meta_data_exist = [
i for i in os.listdir(output_metadata_directory)
if (i == metadata_name)
]
if meta_data_exist == []:
reduction_metadata = metadata.MetaData()
reduction_metadata.create_metadata_file(output_metadata_directory,
metadata_name)
logs.ifverbose(log, setup,
'Successfully created the reduction metadata file')
else:
reduction_metadata = metadata.MetaData()
reduction_metadata.load_all_metadata(output_metadata_directory,
metadata_name)
logs.ifverbose(log, setup,
'Successfully found the reduction metadata')
except:
logs.ifverbose(log, setup,
'No metadata created or loaded : check this!')
sys.exit(1)
return reduction_metadata
def set_psf_dimensions():
params = get_args()
reduction_metadata = metadata.MetaData()
reduction_metadata.load_all_metadata(
metadata_directory=params['red_dir'],
metadata_name='pyDANDIA_metadata.fits')
reduction_metadata.calc_psf_radii()
reduction_metadata.save_updated_metadata(params['red_dir'],
'pyDANDIA_metadata.fits')
def read_star_catalog(params):
"""Function to read in the star catalog layer of an existing metadata file"""
meta_file = params['metadata']
reduction_metadata = metadata.MetaData()
reduction_metadata.load_a_layer_from_file(path.dirname(meta_file),
path.basename(meta_file),
'star_catalog')
star_catalog = Table()
star_catalog['RA'] = reduction_metadata.star_catalog[1]['RA_J2000']
star_catalog['DEC'] = reduction_metadata.star_catalog[1]['DEC_J2000']
return star_catalog, reduction_metadata
def test_crossmatch_pixel_positions():
log = logs.start_stage_log( '.', 'test_compare_crossmatch' )
meta1 = metadata.MetaData()
nstars1 = 20000 # Must be greater than nstars2
data = [Column(name='x', data=np.arange(0,nstars1,1)),
Column(name='y', data=np.arange(0,nstars1,1))]
setattr(meta1,'star_catalog',([],Table(data)))
meta2 = metadata.MetaData()
nstars2 = 1000
data = [Column(name='x', data=np.arange(0.01,(nstars2+0.01),1)),
Column(name='y', data=np.arange(0.01,(nstars2+0.01),1))]
setattr(meta2,'star_catalog',([],Table(data)))
threshold = 0.02
# TEST 1: Find closest-matching stars
(idx1, idx2) = compare_xmatches.crossmatch_pixel_positions(meta1, meta2, log)
# As the number of stars in the two catalogs isn't necessarily the same,
# but the pixel positions are a monotonically increasing sequence,
# the nearest match to the later stars in the longer catalog will be the
# last star in the shorter catalog.
assert( idx1[0:nstars2] == idx2[0:nstars2] ).all()
assert( idx2[nstars2:] == nstars2-1 ).all()
# TEST 2: Require less than allowed separation for a match:
(idx1, idx2) = compare_xmatches.crossmatch_pixel_positions(meta1, meta2, log,
threshold)
assert( idx1[0:nstars2] == idx2[0:nstars2] ).all()
assert(len(idx1) == nstars2)
logs.close_log(log)
def read_meta_data(self):
"""Function to read in the meta data for each dataset"""
for d in ['gdir', 'rdir', 'idir']:
m = metadata.MetaData()
m.load_a_layer_from_file(getattr(self, d),
'pyDANDIA_metadata.fits', 'images_stats')
t = Table()
t['im_name'] = m.images_stats[1]['IM_NAME']
t['fwhm_x'] = m.images_stats[1]['FWHM_X']
t['fwhm_y'] = m.images_stats[1]['FWHM_Y']
t['sky'] = m.images_stats[1]['SKY']
setattr(self, d + '_stats', t)
def select_data_from_star_catalog(params):
"""Function to read the photometric and star catalog data from a metadata file"""
meta_file = params['metadata']
reduction_metadata = metadata.MetaData()
reduction_metadata.load_a_layer_from_file( path.dirname(meta_file), path.basename(meta_file), 'star_catalog' )
star_catalog = Table()
star_catalog['star_index'] = reduction_metadata.star_catalog[1]['star_index']
star_catalog['x_pixel'] = reduction_metadata.star_catalog[1]['x_pixel']
star_catalog['y_pixel'] = reduction_metadata.star_catalog[1]['y_pixel']
star_catalog['RA'] = reduction_metadata.star_catalog[1]['RA_J2000']
star_catalog['DEC'] = reduction_metadata.star_catalog[1]['DEC_J2000']
star_catalog['mag'] = reduction_metadata.star_catalog[1]['ref_mag']
star_catalog['mag_err'] = reduction_metadata.star_catalog[1]['ref_mag_err']
return star_catalog
def fetch_metadata(params):
"""Function to extract the information necessary for the photometric
calibration from a metadata file, adding information to the params
dictionary"""
(red_dir, meta_file) = path.split(params['metadata_file'])
reduction_metadata = metadata.MetaData()
reduction_metadata.load_a_layer_from_file(red_dir, meta_file,
'star_catalog')
star_catalog = Table()
star_catalog['star_index'] = reduction_metadata.star_catalog[1][
'star_index']
star_catalog['x'] = reduction_metadata.star_catalog[1]['x_pixel']
star_catalog['y'] = reduction_metadata.star_catalog[1]['y_pixel']
return reduction_metadata, star_catalog
def run_analysis():
(option, input_file, output_dir) = get_args()
if option == 'C':
xmatch = read_crossmatch_data(input_file)
(separations, masked_ra,
masked_dec) = calc_astrometric_residuals_xmatch(xmatch)
elif option == 'M':
meta = metadata.MetaData()
meta.load_a_layer_from_file(path.dirname(input_file),
path.basename(input_file), 'star_catalog')
(separations, masked_ra,
masked_dec) = calc_astrometric_residuals_metadata(meta)
plot_angular_separations(output_dir, separations, masked_ra, masked_dec)
def extract_star_catalog(params,filter_id,log):
"""Function to read the photometric and star catalog data from a metadata file"""
meta_file = params[filter_id]
reduction_metadata = metadata.MetaData()
reduction_metadata.load_a_layer_from_file( path.dirname(meta_file), path.basename(meta_file), 'star_catalog' )
reduction_metadata.load_a_layer_from_file( path.dirname(meta_file), path.basename(meta_file), 'phot_calib' )
reduction_metadata.load_a_layer_from_file( path.dirname(meta_file), path.basename(meta_file), 'images_stats' )
star_catalog = Table()
star_catalog['star_index'] = reduction_metadata.star_catalog[1]['star_index']
star_catalog['RA'] = reduction_metadata.star_catalog[1]['RA_J2000']
star_catalog['DEC'] = reduction_metadata.star_catalog[1]['DEC_J2000']
star_catalog['mag'] = reduction_metadata.star_catalog[1]['ref_mag']
star_catalog['mag_err'] = reduction_metadata.star_catalog[1]['ref_mag_err']
star_catalog['cal_ref_mag'] = reduction_metadata.phot_calib[1]['cal_ref_mag']
star_catalog['cal_ref_mag_err'] = reduction_metadata.phot_calib[1]['cal_ref_mag_err']
star_catalog['_RAJ2000'] = reduction_metadata.phot_calib[1]['_RAJ2000']
star_catalog['_DEJ2000'] = reduction_metadata.phot_calib[1]['_DEJ2000']
star_catalog['imag'] = reduction_metadata.phot_calib[1]['imag']
star_catalog['e_imag'] = reduction_metadata.phot_calib[1]['e_imag']
star_catalog['rmag'] = reduction_metadata.phot_calib[1]['rmag']
star_catalog['e_rmag'] = reduction_metadata.phot_calib[1]['e_rmag']
star_catalog['gmag'] = reduction_metadata.phot_calib[1]['gmag']
star_catalog['e_gmag'] = reduction_metadata.phot_calib[1]['e_gmag']
log.info('Extracted data for '+str(len(star_catalog))+\
' stars for dataset '+filter_id)
image_table = Table()
image_table['im_name'] = reduction_metadata.images_stats[1]['IM_NAME']
image_table['fwhm_x'] = reduction_metadata.images_stats[1]['FWHM_X']
image_table['fwhm_y'] = reduction_metadata.images_stats[1]['FWHM_Y']
image_table['sky'] = reduction_metadata.images_stats[1]['SKY']
log.info('Extracted data for '+str(len(image_table))+\
' images in the dataset')
return star_catalog, image_table
def read_star_catalogs(self):
for par in ['b_cat', 'g_cat', 'r_cat']:
meta_file = getattr(self, par + '_path')
if meta_file != None and path.isfile(meta_file):
m = metadata.MetaData()
m.load_a_layer_from_file(path.dirname(meta_file),
path.basename(meta_file),
'star_catalog')
star_catalog = Table()
star_catalog['star_index'] = m.star_catalog[1]['star_index']
star_catalog['x'] = m.star_catalog[1]['x_pixel']
star_catalog['y'] = m.star_catalog[1]['y_pixel']
star_catalog['RA'] = m.star_catalog[1]['RA_J2000']
star_catalog['DEC'] = m.star_catalog[1]['DEC_J2000']
setattr(self, par + '_meta', m)
setattr(self, par, star_catalog)
def load_pydandia_star_catalog(params):
"""Function to read the photometric and star catalog data from a metadata file"""
meta_file = params['metadata']
reduction_metadata = metadata.MetaData()
reduction_metadata.load_a_layer_from_file( path.dirname(meta_file), path.basename(meta_file), 'star_catalog' )
star_catalog = Table()
star_catalog['star_index'] = reduction_metadata.star_catalog[1]['star_index']
star_catalog['x'] = reduction_metadata.star_catalog[1]['x_pixel']
star_catalog['y'] = reduction_metadata.star_catalog[1]['y_pixel']
star_catalog['RA'] = reduction_metadata.star_catalog[1]['RA_J2000']
star_catalog['DEC'] = reduction_metadata.star_catalog[1]['DEC_J2000']
star_catalog['mag'] = reduction_metadata.star_catalog[1]['ref_mag']
star_catalog['mag_err'] = reduction_metadata.star_catalog[1]['ref_mag_err']
star_catalog['flux'] = reduction_metadata.star_catalog[1]['ref_flux']
star_catalog['flux_err'] = reduction_metadata.star_catalog[1]['ref_flux_err']
print('Extracted data for '+str(len(star_catalog))+\
' stars from the pyDANDIA analysis')
return star_catalog
def extract_star_catalog(params,filter_id,log):
"""Function to read the photometric and star catalog data from a metadata file"""
meta_file = params[filter_id]
reduction_metadata = metadata.MetaData()
reduction_metadata.load_a_layer_from_file( path.dirname(meta_file), path.basename(meta_file), 'star_catalog' )
star_catalog = Table()
star_catalog['star_index'] = reduction_metadata.star_catalog[1]['star_index']
star_catalog['x'] = reduction_metadata.star_catalog[1]['x_pixel']
star_catalog['y'] = reduction_metadata.star_catalog[1]['y_pixel']
star_catalog['RA'] = reduction_metadata.star_catalog[1]['RA_J2000']
star_catalog['DEC'] = reduction_metadata.star_catalog[1]['DEC_J2000']
star_catalog['mag'] = reduction_metadata.star_catalog[1]['ref_mag']
star_catalog['mag_err'] = reduction_metadata.star_catalog[1]['ref_mag_err']
star_catalog['flux'] = reduction_metadata.star_catalog[1]['ref_flux']
star_catalog['flux_err'] = reduction_metadata.star_catalog[1]['ref_flux_err']
log.info('Extracted data for '+str(len(star_catalog))+\
' stars for dataset '+filter_id)
return star_catalog
def fetch_metadata(setup, params, log):
"""Function to extract the information necessary for the photometric
calibration from a metadata file, adding information to the params
dictionary"""
reduction_metadata = metadata.MetaData()
reduction_metadata.load_a_layer_from_file(setup.red_dir,
params['metadata'],
'data_architecture')
reduction_metadata.load_a_layer_from_file(setup.red_dir,
params['metadata'],
'reduction_parameters')
reduction_metadata.load_a_layer_from_file(setup.red_dir,
params['metadata'],
'headers_summary')
reduction_metadata.load_a_layer_from_file(setup.red_dir,
params['metadata'],
'star_catalog')
try:
reduction_metadata.load_a_layer_from_file(setup.red_dir,
params['metadata'],
'phot_calib')
except KeyError:
pass
params['fov'] = reduction_metadata.reduction_parameters[1]['FOV'][0]
params['refimage'] = reduction_metadata.data_architecture[1]['REF_IMAGE'][
0]
iref = reduction_metadata.headers_summary[1]['IMAGES'].tolist().index(
params['refimage'])
params['ra'] = reduction_metadata.headers_summary[1]['RAKEY'][iref]
params['dec'] = reduction_metadata.headers_summary[1]['DECKEY'][iref]
params['filter'] = reduction_metadata.headers_summary[1]['FILTKEY'][iref]
params['cat_mag_col'] = params['filter'].replace('p', '') + 'mag'
params['cat_err_col'] = 'e_' + params['filter'].replace('p', '') + 'mag'
log.info('Gathered information from metadata file ' + params['metadata'] +
':')
log.info('Image field of view: ' + str(params['fov']) + 'sq. deg')
log.info('Reference image: '+params['refimage']+\
', index '+str(iref)+' in dataset')
log.info('Filter used for dataset: ' + params['filter'])
log.info('Pointing center coordinates: ' + params['ra'] + ' ' +
params['dec'])
star_catalog = Table()
star_catalog['star_index'] = reduction_metadata.star_catalog[1][
'star_index']
star_catalog['RA'] = reduction_metadata.star_catalog[1]['RA_J2000']
star_catalog['DEC'] = reduction_metadata.star_catalog[1]['DEC_J2000']
star_catalog['mag'] = reduction_metadata.star_catalog[1]['ref_mag']
star_catalog['mag_err'] = reduction_metadata.star_catalog[1]['ref_mag_err']
star_catalog['clean'] = np.zeros(
len(reduction_metadata.star_catalog[1]['ref_mag']))
star_catalog['cal_ref_mag'] = np.zeros(
len(reduction_metadata.star_catalog[1]['ref_mag']))
star_catalog['cal_ref_mag_err'] = np.zeros(
len(reduction_metadata.star_catalog[1]['ref_mag']))
log.info('Extracted star catalog')
return reduction_metadata, params, star_catalog
def backup_field_photometry_products(params):
"""Function to gather all the photometric data products for all datasets
for a given field and back them up to another disk"""
if params['output_dir'] != None and not os.path.isdir(params['output_dir']):
raise IOError('Cannot find output directory '+params['output_dir'])
if params['db_path'] and os.path.isfile(params['db_path']):
rsync_file(params['db_path'], os.path.join(params['output_dir'], os.path.basename(params['db_path'])))
elif not params['db_path']:
print('No photometry DB to be backed up')
else:
print('WARNING: Could not find field database at '+params['db_path'])
red_dirs = glob.glob(os.path.join(params['dir_path'],params['field_name']+'_???-dom?-1m0-??-f???_??'))
print('Found '+str(len(red_dirs))+' reduction directories')
for dir in red_dirs:
dset = os.path.basename(dir)
staging_dir = os.path.join(params['output_dir'],dset)
if not os.path.isdir(staging_dir):
os.mkdir(staging_dir)
# Backup the main photometry file
phot_source_file = os.path.join(dir,'photometry.hdf5')
phot_dest_file = os.path.join(staging_dir, 'photometry.hdf5')
if os.path.isfile(phot_source_file):
rsync_file(phot_source_file, phot_dest_file)
comp_phot_dest_file = compress_data_products.bzip2_file(phot_dest_file)
else:
print('WARNING: Could not find photometry file '+phot_source_file)
# Backup the metadata
meta_source_file = os.path.join(dir,'pyDANDIA_metadata.fits')
meta_dest_file = os.path.join(staging_dir, 'pyDANDIA_metadata.fits')
if os.path.isfile(meta_source_file):
rsync_file(meta_source_file, meta_dest_file)
else:
print('WARNING: Could not find metadata file '+meta_source_file)
# Backup the reference directory image data
if not os.path.isdir(os.path.join(staging_dir,'ref')):
os.mkdir(os.path.join(staging_dir,'ref'))
if os.path.isfile(meta_source_file) and os.path.isdir(os.path.join(dir, 'ref')):
mdata = metadata.MetaData()
mdata.load_a_layer_from_file(dir, 'pyDANDIA_metadata.fits', 'data_architecture')
source_files = []
dest_files = []
ref_source_file = os.path.join(mdata.data_architecture[1]['REF_PATH'][0],mdata.data_architecture[1]['REF_IMAGE'][0])
dref_source_file = ref_source_file.replace('.fits', '_res.fits')
psfstamp_source = os.path.join(mdata.data_architecture[1]['REF_PATH'][0],'final_psf_master_stamp.fits')
psfstampvar_source = os.path.join(mdata.data_architecture[1]['REF_PATH'][0],'final_psf_master_stamp_varience.fits')
mask_source = os.path.join(mdata.data_architecture[1]['REF_PATH'][0],'master_mask.fits')
maskedref_source = os.path.join(mdata.data_architecture[1]['REF_PATH'][0],'masked_ref_image.fits')
psfmodel_source = os.path.join(mdata.data_architecture[1]['REF_PATH'][0],'psf_model.fits')
psfmodelnorm_source = os.path.join(mdata.data_architecture[1]['REF_PATH'][0],'psf_model_normalized.fits')
psfmodelres_source = os.path.join(mdata.data_architecture[1]['REF_PATH'][0],'psf_model_residuals.fits')
flist = [ref_source_file, dref_source_file,
psfstamp_source, psfstampvar_source,
mask_source, maskedref_source,
psfmodel_source, psfmodelnorm_source, psfmodelres_source]
for f in flist:
source_files.append(f)
dest_files.append(os.path.join(staging_dir, 'ref', os.path.basename(f)))
for i,f in enumerate(source_files):
rsync_file(f, dest_files[i])
refpngs = glob.glob(os.path.join(mdata.data_architecture[1]['REF_PATH'][0],'ref','*.png'))
for f_source in refpngs:
f_dest = os.path.join(staging_dir, 'ref', os.path.basename(f_source))
rsync_file(f_source, f_dest)
# Backup the DS9 overlay datafiles
overlays = glob.glob(os.path.join(mdata.data_architecture[1]['REF_PATH'][0],'*.reg'))
for f_source in overlays:
f_dest = os.path.join(staging_dir, 'ref', os.path.basename(f_source))
rsync_file(f_source, f_dest)
# Compress reference image
for image in glob.glob(os.path.join(staging_dir, 'ref', '*.fits')):
if os.path.isfile(image+'.bz2') == False:
args = ['bzip2', image]
p = subprocess.Popen(args, stdout=subprocess.PIPE)
p.wait()
else:
print('Skipping compression of '+image+' (compressed product already exists)')
# Backup the reduction logs
log_list = glob.glob(os.path.join(dir,'*.log'))
for log_source in log_list:
log_dest = os.path.join(staging_dir,os.path.basename(log_source))
rsync_file(log_source, log_dest)
# Backup the output plots
png_list = glob.glob(os.path.join(dir,'*.png'))
for png_source in png_list:
png_dest = os.path.join(staging_dir,os.path.basename(png_source))
rsync_file(png_source, png_dest)
# Backup the configuration directory
config_source = os.path.join(params['dir_path'],'config')
config_dest = params['output_dir']
rsync_file(config_source, config_dest)
# Backup the field crossmatch file and log:
source_files = [ os.path.join(params['dir_path'], params['field_name']+'_field_crossmatch.fits'),
os.path.join(params['dir_path'], 'crossmatch.log'),
os.path.join(params['dir_path'], 'crossmatch_gaia.log'), ]
logs_dest = params['output_dir']
for f in source_files:
rsync_file(f, logs_dest)
if '.fits' in f:
comp_file = compress_data_products.bzip2_file(os.path.join(logs_dest, os.path.basename(f)))
# Backup the field photometry files:
source_files = []
for q in [1,2,3,4]:
source_files.append( os.path.join(params['dir_path'], params['field_name']+'_quad'+str(q)+'_photometry.hdf5') )
logs_dest = params['output_dir']
for f in source_files:
rsync_file(f, logs_dest)
comp_file = compress_data_products.bzip2_file(os.path.join(logs_dest, os.path.basename(f)))
# Backup the logs directory
logs_source = os.path.join(params['dir_path'],'logs')
logs_dest = params['output_dir']
rsync_file(logs_source, logs_dest)
print('Backed-up data products for '+dset+' to '+staging_dir)
upload_aws.upload_directory_nochecks(params['output_dir'],
params['local_root'],
params['aws_root'])
print('Backed-up field photometric data products to AWS Cloud')
def run_stage2(setup):
"""Main driver function to run stage 2: reference selection.
This stage is processing the metadata file, looks for the output of
stages0 and stage1 and checks if a reference file already
exists.
It creates a reference frame based on the selection criteria
defined in the configuration. If no such configuration exists, it
falls back to a standard configuration.
If stage1 has failed to produce output it selects a reference
based on header information.
It always re-runs when called, since it is a lightweight function
"""
stage2_version = 'stage2 v0.1'
log = logs.start_stage_log(setup.red_dir, 'stage2', version=stage2_version)
log.info('Setup:\n' + setup.summary() + '\n')
reduction_metadata = metadata.MetaData()
# Load all metadata
try:
reduction_metadata.load_all_metadata(
metadata_directory=setup.red_dir,
metadata_name='pyDANDIA_metadata.fits')
# Check data inventory on metadata
log.info('stage2 has loaded the reduction metadata')
except Exception as estr:
log.info('Could not load metadata!' + repr(estr))
status = 'FAILED'
report = 'Loading metadata failed:' + repr(estr)
return status, report
try:
n_images = len(reduction_metadata.images_stats)
except AttributeError:
log.info('stage2: data inventory missing.')
status = 'FAILED'
report = 'Data inventory (stage1) missing.'
logs.close_log(log)
return status, report
# All parameters are part of metadata
table_structure = [['IMAGE_NAME', 'MOON_STATUS', 'RANKING_KEY'],
['S100', 'S100', 'float'], ['degree', None, None]]
all_images = reduction_metadata.find_all_images(setup,
reduction_metadata,
os.path.join(
setup.red_dir, 'data'),
log=log)
reduction_metadata.create_a_new_layer(layer_name='reference_inventory',
data_structure=table_structure,
data_columns=None)
log.info('Create reference frame inventory table in metadata')
# Iterate over images that are in the stage inventory
reference_ranking = []
fwhm_max = 0.
for stats_entry in reduction_metadata.images_stats[1]:
if float(stats_entry['FWHM_X']) > fwhm_max:
fwhm_max = stats_entry['FWHM_X']
if float(stats_entry['FWHM_Y']) > fwhm_max:
fwhm_max = stats_entry['FWHM_Y']
# taking filenames from headers_summary (stage1 change pending)
filename_images = reduction_metadata.images_stats[1]['IM_NAME']
data_image_directory = reduction_metadata.data_architecture[1][
'IMAGES_PATH'][0]
max_adu = float(reduction_metadata.reduction_parameters[1]['MAXVAL'][0])
psf_size = int(
4. * float(reduction_metadata.reduction_parameters[1]['KER_RAD'][0]) *
fwhm_max)
empirical_psf_flag = False
if empirical_psf_flag == True:
for stats_entry in reduction_metadata.images_stats[1]:
image_filename = stats_entry[0]
row_idx = np.where(reduction_metadata.images_stats[1]['IM_NAME'] ==
image_filename)[0][0]
moon_status = 'dark'
# to be reactivated as soon as it is part of metadata
if 'MOONFKEY' in reduction_metadata.headers_summary[1].keys(
) and 'MOONDKEY' in reduction_metadata.headers_summary[1].keys():
moon_status = moon_brightness_header(
reduction_metadata.headers_summary[1], row_idx)
fwhm_arcsec = (float(stats_entry['FWHM_X'])**2 + float(
stats_entry['FWHM_Y'])**2)**0.5 * float(
reduction_metadata.reduction_parameters[1]['PIX_SCALE'])
# extract data inventory row for image and calculate sorting key
# if a sufficient number of stars has been detected at s1 (40)
if int(stats_entry['NSTARS']) > 34 and fwhm_arcsec < 3. and (
not 'bright' in moon_status):
hdulist = fits.open(os.path.join(data_image_directory,
image_filename),
memmap=True)
image = hdulist[0].data
ranking_key = empirical_psf_simple.empirical_snr_subframe(
image, psf_size, max_adu)
hdulist.close()
reference_ranking.append([image_filename, ranking_key])
entry = [image_filename, moon_status, ranking_key]
reduction_metadata.add_row_to_layer(
key_layer='reference_inventory', new_row=entry)
else:
for stats_entry in reduction_metadata.images_stats[1]:
image_filename = stats_entry[0]
row_idx = np.where(reduction_metadata.images_stats[1]['IM_NAME'] ==
image_filename)[0][0]
moon_status = 'dark'
# to be reactivated as soon as it is part of metadata
if 'MOONFKEY' in reduction_metadata.headers_summary[1].keys(
) and 'MOONDKEY' in reduction_metadata.headers_summary[1].keys():
moon_status = moon_brightness_header(
reduction_metadata.headers_summary[1], row_idx)
fwhm_arcsec = (float(stats_entry['FWHM_X'])**2 + float(
stats_entry['FWHM_Y'])**2)**0.5 * float(
reduction_metadata.reduction_parameters[1]['PIX_SCALE'])
# extract data inventory row for image and calculate sorting key
# if a sufficient number of stars has been detected at s1 (40)
if int(stats_entry['NSTARS']) > 34 and fwhm_arcsec < 3. and (
not 'bright' in moon_status):
ranking_key = add_stage1_rank(reduction_metadata, stats_entry)
reference_ranking.append([image_filename, ranking_key])
entry = [image_filename, moon_status, ranking_key]
reduction_metadata.add_row_to_layer(
key_layer='reference_inventory', new_row=entry)
#relax criteria...
if reference_ranking == []:
for stats_entry in reduction_metadata.images_stats[1]:
image_filename = stats_entry[0]
row_idx = np.where(reduction_metadata.images_stats[1]['IM_NAME'] ==
image_filename)[0][0]
moon_status = 'dark'
# to be reactivated as soon as it is part of metadata
if 'MOONFKEY' in reduction_metadata.headers_summary[1].keys(
) and 'MOONDKEY' in reduction_metadata.headers_summary[1].keys():
moon_status = moon_brightness_header(
reduction_metadata.headers_summary[1], row_idx)
fwhm_arcsec = (float(stats_entry['FWHM_X'])**2 + float(
stats_entry['FWHM_Y'])**2)**0.5 * float(
reduction_metadata.reduction_parameters[1]['PIX_SCALE'])
# extract data inventory row for image and calculate sorting key
if int(stats_entry['NSTARS']) > 20. and fwhm_arcsec < 3.:
ranking_key = add_stage1_rank(reduction_metadata, stats_entry)
reference_ranking.append([image_filename, ranking_key])
entry = [image_filename, moon_status, ranking_key]
reduction_metadata.add_row_to_layer(
key_layer='reference_inventory', new_row=entry)
# Save the updated layer to the metadata file
reduction_metadata.save_a_layer_to_file(
metadata_directory=setup.red_dir,
metadata_name='pyDANDIA_metadata.fits',
key_layer='reference_inventory')
if reference_ranking != []:
best_image = sorted(reference_ranking, key=itemgetter(1))[-1]
ref_directory_path = os.path.join(setup.red_dir, 'ref')
if not os.path.exists(ref_directory_path):
os.mkdir(ref_directory_path)
ref_img_path = os.path.join(
str(reduction_metadata.data_architecture[1]['IMAGES_PATH'][0]),
best_image[0])
print('New reference ' + best_image[0] + ' in ' + ref_img_path)
try:
copyfile(
reduction_metadata.data_architecture[1]['IMAGES_PATH'][0] +
'/' + best_image[0], ref_directory_path + '/' + best_image[0])
except:
print('copy ref failed: ', best_image[0])
if not 'REF_PATH' in reduction_metadata.data_architecture[1].keys():
reduction_metadata.add_column_to_layer('data_architecture',
'REF_PATH',
[ref_directory_path],
new_column_format=None,
new_column_unit=None)
else:
reduction_metadata.update_a_cell_to_layer('data_architecture', 0,
'REF_PATH',
ref_directory_path)
if not 'REF_IMAGE' in reduction_metadata.data_architecture[1].keys():
reduction_metadata.add_column_to_layer(
'data_architecture',
'REF_IMAGE', [os.path.basename(ref_img_path)],
new_column_format=None,
new_column_unit=None)
else:
reduction_metadata.update_a_cell_to_layer(
'data_architecture', 0, 'REF_IMAGE',
os.path.basename(ref_img_path))
# Update the REDUCTION_STATUS table in metadata for stage 2
reduction_metadata.update_reduction_metadata_reduction_status(
all_images, stage_number=1, status=1, log=log)
reduction_metadata.save_updated_metadata(
metadata_directory=setup.red_dir,
metadata_name='pyDANDIA_metadata.fits')
status = 'OK'
report = 'Completed successfully'
log.info('Updating metadata with info on new images...')
logs.close_log(log)
return status, report
else:
status = 'FAILED'
report = 'No suitable image found.'
log.info('No reference image found...')
logs.close_log(log)
return status, report
def run_stage6(setup):
"""Main driver function to run stage 6: image substraction and photometry.
This stage align the images to the reference frame!
:param object setup : an instance of the ReductionSetup class. See reduction_control.py
:return: [status, report, reduction_metadata], the stage4 status, the report, the metadata file
:rtype: array_like
"""
stage6_version = 'stage6 v0.1'
log = logs.start_stage_log(setup.red_dir, 'stage6', version=stage6_version)
log.info('Setup:\n' + setup.summary() + '\n')
# find the metadata
reduction_metadata = metadata.MetaData()
reduction_metadata.load_all_metadata(setup.red_dir, 'pyDANDIA_metadata.fits')
# find the images needed to treat
all_images = reduction_metadata.find_all_images(setup, reduction_metadata,
os.path.join(setup.red_dir, 'data'), log=log)
new_images = reduction_metadata.find_images_need_to_be_process(setup, all_images,
stage_number=6, rerun_all=None, log=log)
# find the starlist
starlist = reduction_metadata.star_catalog[1]
max_x = np.max(starlist['x_pixel'].data)
max_y = np.max(starlist['y_pixel'].data)
mask = (starlist['psf_star'].data == 1) & (starlist['x_pixel'].data<max_x-25) & (starlist['x_pixel'].data>25) & (starlist['y_pixel'].data<max_y-25) & (starlist['y_pixel'].data>25)
control_stars = starlist[mask][:10]
star_coordinates = np.c_[control_stars['star_index'].data,
control_stars['x_pixel'].data,
control_stars['y_pixel'].data]
for index,key in enumerate(starlist.columns.keys()):
if index != 0:
ref_star_catalog = np.c_[ref_star_catalog,starlist[key].data]
else:
ref_star_catalog = starlist[key].data
psf_model = fits.open(reduction_metadata.data_architecture[1]['REF_PATH'].data[0]+'/psf_model.fits')
psf_type = psf_model[0].header['PSFTYPE']
psf_parameters = [0, psf_model[0].header['Y_CENTER'],
psf_model[0].header['X_CENTER'],
psf_model[0].header['GAMMA'],
psf_model[0].header['ALPHA']]
sky_model = sky_background.model_sky_background(setup,
reduction_metadata,log,ref_star_catalog)
psf_model = psf.get_psf_object( psf_type )
psf_model.update_psf_parameters( psf_parameters)
ind = ((starlist['x_pixel']-150)**2<1) & ((starlist['y_pixel']-150)**2<1)
print (np.argmin(((starlist['x_pixel']-150)**2) + ((starlist['y_pixel']-150)**2)))
if len(new_images) > 0:
# find the reference image
try:
reference_image_name = reduction_metadata.data_architecture[1]['REF_IMAGE'].data[0]
reference_image_directory = reduction_metadata.data_architecture[1]['REF_PATH'].data[0]
reference_image,date = open_an_image(setup, reference_image_directory, reference_image_name, image_index=0,
log=None)
ref_image_name = reduction_metadata.data_architecture[1]['REF_IMAGE'].data[0]
index_reference = np.where(ref_image_name == reduction_metadata.headers_summary[1]['IMAGES'].data)[0][0]
ref_exposure_time = float(reduction_metadata.headers_summary[1]['EXPKEY'].data[index_reference])
logs.ifverbose(log, setup,
'I found the reference frame:' + reference_image_name)
except KeyError:
logs.ifverbose(log, setup,
'I can not find any reference image! Aboard stage6')
status = 'KO'
report = 'No reference frame found!'
return status, report
# find the kernels directory
try:
kernels_directory = reduction_metadata.data_architecture[1]['OUTPUT_DIRECTORY'].data[0]+'kernel/'
logs.ifverbose(log, setup,
'I found the kernels directory:' + kernels_directory)
except KeyError:
logs.ifverbose(log, setup,
'I can not find the kernels directory! Aboard stage6')
status = 'KO'
report = 'No kernels directory found!'
return status, report
data = []
diffim_directory = reduction_metadata.data_architecture[1]['OUTPUT_DIRECTORY'].data[0]+'diffim/'
images_directory = reduction_metadata.data_architecture[1]['IMAGES_PATH'].data[0]
phot = np.zeros((len(new_images),len(ref_star_catalog),16))
time = []
for idx,new_image in enumerate(new_images):
log.info('Starting difference photometry of '+new_image)
target_image,date = open_an_image(setup, images_directory, new_image, image_index=0, log=None)
kernel_image,kernel_error,kernel_bkg = find_the_associated_kernel(setup, kernels_directory, new_image)
difference_image = image_substraction(setup, reduction_metadata,reference_image, kernel_image, new_image)-kernel_bkg
time.append(date)
save_control_stars_of_the_difference_image(setup, new_image, difference_image, star_coordinates)
photometric_table, control_zone = photometry_on_the_difference_image(setup, reduction_metadata, log,ref_star_catalog,difference_image, psf_model, sky_model, kernel_image,kernel_error, ref_exposure_time)
phot[idx,:,:] = photometric_table
#save_control_zone_of_residuals(setup, new_image, control_zone)
#ingest_photometric_table_in_db(setup, photometric_table)
import pdb; pdb.set_trace()
import matplotlib.pyplot as plt
ind = ((starlist['x_pixel']-150)**2<1) & ((starlist['y_pixel']-150)**2<1)
plt.errorbar(time,phot[:,ind,8],fmt='.k')
plt.show()
import pdb; pdb.set_trace()
return status, report
def run_stage4(setup):
"""Main driver function to run stage 4: image alignement.
This stage align the images to the reference frame!
:param object setup : an instance of the ReductionSetup class. See reduction_control.py
:return: [status, report, reduction_metadata], the stage4 status, the report, the metadata file
:rtype: array_like
"""
stage4_version = 'stage4 v0.1'
log = logs.start_stage_log(setup.red_dir, 'stage4', version=stage4_version)
log.info('Setup:\n' + setup.summary() + '\n')
# find the metadata
reduction_metadata = metadata.MetaData()
reduction_metadata.load_all_metadata(setup.red_dir,
'pyDANDIA_metadata.fits')
# find the images needed to treat
all_images = reduction_metadata.find_all_images(setup,
reduction_metadata,
os.path.join(
setup.red_dir, 'data'),
log=log)
new_images = reduction_metadata.find_images_need_to_be_process(
setup, all_images, stage_number=4, rerun_all=None, log=log)
if len(new_images) > 0:
# find the reference image
try:
reference_image_name = reduction_metadata.data_architecture[1][
'REF_IMAGE'].data[0]
reference_image_directory = reduction_metadata.data_architecture[
1]['REF_PATH'].data[0]
reference_image = open_an_image(setup,
reference_image_directory,
reference_image_name,
image_index=0,
log=None)
logs.ifverbose(
log, setup,
'I found the reference frame:' + reference_image_name)
except KeyError:
logs.ifverbose(log, setup,
'I can not find any reference image! Abort stage4')
status = 'KO'
report = 'No reference frame found!'
return status, report
data = []
images_directory = reduction_metadata.data_architecture[1][
'IMAGES_PATH'].data[0]
for new_image in new_images:
target_image = open_an_image(setup,
images_directory,
new_image,
image_index=0,
log=None)
try:
x_new_center, y_new_center, x_shift, y_shift = find_x_y_shifts_from_the_reference_image(
setup,
reference_image,
target_image,
edgefraction=0.5,
log=None)
data.append([new_image, x_shift, y_shift])
logs.ifverbose(
log, setup,
'I found the image translation to the reference for frame:'
+ new_image)
except:
logs.ifverbose(
log, setup,
'I can not find the image translation to the reference for frame:'
+ new_image + '. Abort stage4!')
status = 'KO'
report = 'No shift found for image:' + new_image + ' !'
return status, report
if ('SHIFT_X' in reduction_metadata.images_stats[1].keys()) and (
'SHIFT_Y' in reduction_metadata.images_stats[1].keys()):
for index in range(len(data)):
target_image = data[index][0]
x_shift = data[index][1]
y_shift = data[index][2]
row_index = np.where(reduction_metadata.images_stats[1]
['IM_NAME'].data == new_image)[0][0]
reduction_metadata.update_a_cell_to_layer(
'images_stats', row_index, 'SHIFT_X', x_shift)
reduction_metadata.update_a_cell_to_layer(
'images_stats', row_index, 'SHIFT_Y', y_shift)
logs.ifverbose(log, setup,
'Updated metadata for image: ' + target_image)
else:
logs.ifverbose(log, setup,
'I have to construct SHIFT_X and SHIFT_Y columns')
sorted_data = np.copy(data)
for index in range(len(data)):
target_image = data[index][0]
row_index = np.where(reduction_metadata.images_stats[1]
['IM_NAME'].data == new_image)[0][0]
sorted_data[row_index] = data[index]
column_format = 'int'
column_unit = 'pix'
reduction_metadata.add_column_to_layer(
'images_stats',
'SHIFT_X',
sorted_data[:, 1],
new_column_format=column_format,
new_column_unit=column_unit)
reduction_metadata.add_column_to_layer(
'images_stats',
'SHIFT_Y',
sorted_data[:, 2],
new_column_format=column_format,
new_column_unit=column_unit)
reduction_metadata.update_reduction_metadata_reduction_status(
new_images, stage_number=4, status=1, log=log)
reduction_metadata.save_updated_metadata(
reduction_metadata.data_architecture[1]['OUTPUT_DIRECTORY'][0],
reduction_metadata.data_architecture[1]['METADATA_NAME'][0],
log=log)
logs.close_log(log)
status = 'OK'
report = 'Completed successfully'
return status, report
def run_stage5(setup):
"""Main driver function to run stage 5: kernel_solution
This stage finds the kernel solution and (optionally) subtracts the model
image
:param object setup : an instance of the ReductionSetup class. See reduction_control.py
:return: [status, report, reduction_metadata], stage5 status, report,
metadata file
:rtype: array_like
"""
stage5_version = 'stage5 v0.1'
log = logs.start_stage_log(setup.red_dir, 'stage5', version=stage5_version)
log.info('Setup:\n' + setup.summary() + '\n')
try:
from umatrix_routine import umatrix_construction, umatrix_bvector_construction, bvector_construction
except ImportError:
log.info(
'Uncompiled cython code, please run setup.py: e.g.\n python setup.py build_ext --inplace'
)
status = 'KO'
report = 'Uncompiled cython code, please run setup.py: e.g.\n python setup.py build_ext --inplace'
return status, report
# find the metadata
reduction_metadata = metadata.MetaData()
reduction_metadata.load_all_metadata(setup.red_dir,
'pyDANDIA_metadata.fits')
#determine kernel size based on maximum FWHM
fwhm_max = 0.
shift_max = 0
for stats_entry in reduction_metadata.images_stats[1]:
if float(stats_entry['FWHM_X']) > fwhm_max:
fwhm_max = stats_entry['FWHM_X']
if float(stats_entry['FWHM_Y']) > fwhm_max:
fwhm_max = stats_entry['FWHM_Y']
if abs(float(stats_entry['SHIFT_X'])) > shift_max:
shift_max = abs(float(stats_entry['SHIFT_X']))
if abs(float(stats_entry['SHIFT_Y'])) > shift_max:
shift_max = abs(float(stats_entry['SHIFT_Y']))
maxshift = int(shift_max) + 2
#image smaller or equal 500x500
large_format_image = False
sigma_max = fwhm_max / (2. * (2. * np.log(2.))**0.5)
# Factor 4 corresponds to the radius of 2*FWHM the old pipeline
kernel_size = int(
4. * float(reduction_metadata.reduction_parameters[1]['KER_RAD'][0]) *
fwhm_max)
if kernel_size:
if kernel_size % 2 == 0:
kernel_size = kernel_size + 1
# find the images that need to be processed
all_images = reduction_metadata.find_all_images(setup,
reduction_metadata,
os.path.join(
setup.red_dir, 'data'),
log=log)
new_images = reduction_metadata.find_images_need_to_be_process(
setup, all_images, stage_number=5, rerun_all=None, log=log)
kernel_directory_path = os.path.join(setup.red_dir, 'kernel')
diffim_directory_path = os.path.join(setup.red_dir, 'diffim')
if not os.path.exists(kernel_directory_path):
os.mkdir(kernel_directory_path)
if not os.path.exists(diffim_directory_path):
os.mkdir(diffim_directory_path)
reduction_metadata.update_column_to_layer('data_architecture',
'KERNEL_PATH',
kernel_directory_path)
# difference images are written for verbosity level > 0
reduction_metadata.update_column_to_layer('data_architecture',
'DIFFIM_PATH',
diffim_directory_path)
data_image_directory = reduction_metadata.data_architecture[1][
'IMAGES_PATH'][0]
ref_directory_path = '.'
#For a quick image subtraction, pre-calculate a sufficiently large u_matrix
#based on the largest FWHM and store it to disk -> needs config switch
try:
reference_image_name = str(
reduction_metadata.data_architecture[1]['REF_IMAGE'][0])
reference_image_directory = str(
reduction_metadata.data_architecture[1]['REF_PATH'][0])
max_adu = 0.3 * float(
reduction_metadata.reduction_parameters[1]['MAXVAL'][0])
ref_row_index = np.where(
reduction_metadata.images_stats[1]['IM_NAME'] == str(
reduction_metadata.data_architecture[1]['REF_IMAGE'][0]))[0][0]
ref_fwhm_x = reduction_metadata.images_stats[1][ref_row_index][
'FWHM_X']
ref_fwhm_y = reduction_metadata.images_stats[1][ref_row_index][
'FWHM_Y']
ref_sigma_x = ref_fwhm_x / (2. * (2. * np.log(2.))**0.5)
ref_sigma_y = ref_fwhm_y / (2. * (2. * np.log(2.))**0.5)
ref_stats = [ref_fwhm_x, ref_fwhm_y, ref_sigma_x, ref_sigma_y]
logs.ifverbose(log, setup,
'Using reference image:' + reference_image_name)
except Exception as e:
log.ifverbose(log, setup, 'Reference/Images ! Abort stage5' + str(e))
status = 'KO'
report = 'No reference image found!'
return status, report, reduction_metadata
if not ('SHIFT_X' in reduction_metadata.images_stats[1].keys()) and (
'SHIFT_Y' in reduction_metadata.images_stats[1].keys()):
log.ifverbose(log, setup, 'No xshift! run stage4 ! Abort stage5')
status = 'KO'
report = 'No alignment data found!'
return status, report, reduction_metadata
if large_format_image == False:
subtract_small_format_image(new_images,
reference_image_name,
reference_image_directory,
reduction_metadata,
setup,
data_image_directory,
kernel_size,
max_adu,
ref_stats,
maxshift,
kernel_directory_path,
diffim_directory_path,
log=log)
#append some metric for the kernel, perhaps its scale factor...
reduction_metadata.update_reduction_metadata_reduction_status(
new_images, stage_number=5, status=1, log=log)
logs.close_log(log)
status = 'OK'
report = 'Completed successfully'
return status, report
bb = astropy.coordinates.get_sun(ti)
moon.append([aa.ra.value,aa.dec.value])
illumination.append(microlsimulator.moon_illumination(bb,aa).value)
Moon = np.array(moon)
distance = np.arccos(np.sin(Moon[:,1]*3.14/180)*np.sin(dec*3.14/180)+np.cos(Moon[:,1]*3.14/180)*np.cos(dec*3.14/180)*np.cos((Moon[:,0]-ra)*3.14/180))*180/3.14
return Moon,distance,np.array(illumination)
### Setup paths etc
phot_path = '/home/ebachelet/Work/ROMEREA/LIA/ROME1/LSCA_ip/'
hdf_files = h5py.File(phot_path+'photometry.hdf5', 'r')
reduction_metadata = metadata.MetaData()
reduction_metadata.load_all_metadata(metadata_directory=phot_path,
metadata_name='pyDANDIA_metadata.fits')
target = SkyCoord(ra=267.83589537*units.deg,dec=-30.060817819*units.deg)
### Extract images quality metrics
pscale = reduction_metadata.images_stats[1]['PSCALE']
sky = reduction_metadata.images_stats[1]['SKY']
fwhm = reduction_metadata.images_stats[1]['FWHM']
nstars = reduction_metadata.images_stats[1]['NSTARS']