def compare_catalog_xmatch_between_reductions(): params = get_args() log = logs.start_stage_log(params['log_dir'], 'compare_crossmatch') meta1 = metadata.MetaData() meta1.load_all_metadata(params['red_dir1'], 'pyDANDIA_metadata.fits') log.info('Loaded metadata from ' + params['red_dir1']) meta2 = metadata.MetaData() meta2.load_all_metadata(params['red_dir2'], 'pyDANDIA_metadata.fits') log.info('Loaded metadata from ' + params['red_dir2']) # Crossmatch the catalogs by x, y pixel positions (meta1_matches, meta2_matches) = crossmatch_pixel_positions_per_star(meta1, meta2, log, threshold=1.0) matched_data = build_matched_arrays(meta1, meta2, meta1_matches, meta2_matches, log) matched_data = calculate_separations_on_sky(matched_data, log) # For each matching star, compare RA, Dec and Gaia ID if available compare_coordinates(params, matched_data, log) compare_gaia_ids(params, matched_data, log) logs.close_log(log)
def fetch_metadata(setup,params,log): """Function to extract the information necessary for the photometric calibration from a metadata file, adding information to the params dictionary""" reduction_metadata = metadata.MetaData() reduction_metadata.load_a_layer_from_file( setup.red_dir, params['metadata'], 'data_architecture' ) reduction_metadata.load_a_layer_from_file( setup.red_dir, params['metadata'], 'star_catalog' ) reduction_metadata.load_a_layer_from_file( setup.red_dir, params['metadata'], 'phot_calib' ) params['refimage'] = reduction_metadata.data_architecture[1]['REF_IMAGE'][0] log.info('Using data for reference image: '+params['refimage']) star_catalog = Table() star_catalog['star_index'] = reduction_metadata.star_catalog[1]['star_index'] star_catalog['mag'] = reduction_metadata.star_catalog[1]['ref_mag'] star_catalog['mag_err'] = reduction_metadata.star_catalog[1]['ref_mag_err'] star_catalog['cal_ref_mag'] = reduction_metadata.phot_calib[1]['cal_ref_mag'] star_catalog['cal_ref_mag_err'] = reduction_metadata.phot_calib[1]['cal_ref_mag_err'] log.info('Extracted star catalog') return reduction_metadata, params, star_catalog
def has_ref_image_been_selected(data_dir): stage2_log = path.join(data_dir, 'stage2.log') found_log = path.isfile(stage2_log) if found_log: found_string = find_entry_in_log(log_file, 'No reference image found') else: found_string = False reduction_metadata = metadata.MetaData() reduction_metadata.load_all_metadata(setup.red_dir, 'pyDANDIA_metadata.fits') found_meta = 'REF_PATH' in reduction_metadata.data_architecture[1].keys() found_dir = path.isdir(data_dir, 'ref') if found_dir: image_list = glob.glob(data_dir, 'ref', '*fits') found_image = len(image_list) >= 1 else: found_image = False if found_log and found_string and found_meta and found_dir and found_image: return True else: return False
def create_or_load_the_reduction_metadata( setup, output_metadata_directory, metadata_name='pyDANDIA_metadata.fits', log=None): ''' This creates (new reduction) or load (ongoing reduction) the metadata file linked to this reduction. :param string output_metadata_directory: the directory where to place the metadata :param string metadata_name: the name of the metadata file :param boolean verbose: switch to True to have more informations :return: the metadata object :rtype: metadata object ''' try: meta_data_exist = [ i for i in os.listdir(output_metadata_directory) if (i == metadata_name) ] if meta_data_exist == []: reduction_metadata = metadata.MetaData() reduction_metadata.create_metadata_file(output_metadata_directory, metadata_name) logs.ifverbose(log, setup, 'Successfully created the reduction metadata file') else: reduction_metadata = metadata.MetaData() reduction_metadata.load_all_metadata(output_metadata_directory, metadata_name) logs.ifverbose(log, setup, 'Successfully found the reduction metadata') except: logs.ifverbose(log, setup, 'No metadata created or loaded : check this!') sys.exit(1) return reduction_metadata
def set_psf_dimensions(): params = get_args() reduction_metadata = metadata.MetaData() reduction_metadata.load_all_metadata( metadata_directory=params['red_dir'], metadata_name='pyDANDIA_metadata.fits') reduction_metadata.calc_psf_radii() reduction_metadata.save_updated_metadata(params['red_dir'], 'pyDANDIA_metadata.fits')
def read_star_catalog(params): """Function to read in the star catalog layer of an existing metadata file""" meta_file = params['metadata'] reduction_metadata = metadata.MetaData() reduction_metadata.load_a_layer_from_file(path.dirname(meta_file), path.basename(meta_file), 'star_catalog') star_catalog = Table() star_catalog['RA'] = reduction_metadata.star_catalog[1]['RA_J2000'] star_catalog['DEC'] = reduction_metadata.star_catalog[1]['DEC_J2000'] return star_catalog, reduction_metadata
def test_crossmatch_pixel_positions(): log = logs.start_stage_log( '.', 'test_compare_crossmatch' ) meta1 = metadata.MetaData() nstars1 = 20000 # Must be greater than nstars2 data = [Column(name='x', data=np.arange(0,nstars1,1)), Column(name='y', data=np.arange(0,nstars1,1))] setattr(meta1,'star_catalog',([],Table(data))) meta2 = metadata.MetaData() nstars2 = 1000 data = [Column(name='x', data=np.arange(0.01,(nstars2+0.01),1)), Column(name='y', data=np.arange(0.01,(nstars2+0.01),1))] setattr(meta2,'star_catalog',([],Table(data))) threshold = 0.02 # TEST 1: Find closest-matching stars (idx1, idx2) = compare_xmatches.crossmatch_pixel_positions(meta1, meta2, log) # As the number of stars in the two catalogs isn't necessarily the same, # but the pixel positions are a monotonically increasing sequence, # the nearest match to the later stars in the longer catalog will be the # last star in the shorter catalog. assert( idx1[0:nstars2] == idx2[0:nstars2] ).all() assert( idx2[nstars2:] == nstars2-1 ).all() # TEST 2: Require less than allowed separation for a match: (idx1, idx2) = compare_xmatches.crossmatch_pixel_positions(meta1, meta2, log, threshold) assert( idx1[0:nstars2] == idx2[0:nstars2] ).all() assert(len(idx1) == nstars2) logs.close_log(log)
def read_meta_data(self): """Function to read in the meta data for each dataset""" for d in ['gdir', 'rdir', 'idir']: m = metadata.MetaData() m.load_a_layer_from_file(getattr(self, d), 'pyDANDIA_metadata.fits', 'images_stats') t = Table() t['im_name'] = m.images_stats[1]['IM_NAME'] t['fwhm_x'] = m.images_stats[1]['FWHM_X'] t['fwhm_y'] = m.images_stats[1]['FWHM_Y'] t['sky'] = m.images_stats[1]['SKY'] setattr(self, d + '_stats', t)
def select_data_from_star_catalog(params): """Function to read the photometric and star catalog data from a metadata file""" meta_file = params['metadata'] reduction_metadata = metadata.MetaData() reduction_metadata.load_a_layer_from_file( path.dirname(meta_file), path.basename(meta_file), 'star_catalog' ) star_catalog = Table() star_catalog['star_index'] = reduction_metadata.star_catalog[1]['star_index'] star_catalog['x_pixel'] = reduction_metadata.star_catalog[1]['x_pixel'] star_catalog['y_pixel'] = reduction_metadata.star_catalog[1]['y_pixel'] star_catalog['RA'] = reduction_metadata.star_catalog[1]['RA_J2000'] star_catalog['DEC'] = reduction_metadata.star_catalog[1]['DEC_J2000'] star_catalog['mag'] = reduction_metadata.star_catalog[1]['ref_mag'] star_catalog['mag_err'] = reduction_metadata.star_catalog[1]['ref_mag_err'] return star_catalog
def fetch_metadata(params): """Function to extract the information necessary for the photometric calibration from a metadata file, adding information to the params dictionary""" (red_dir, meta_file) = path.split(params['metadata_file']) reduction_metadata = metadata.MetaData() reduction_metadata.load_a_layer_from_file(red_dir, meta_file, 'star_catalog') star_catalog = Table() star_catalog['star_index'] = reduction_metadata.star_catalog[1][ 'star_index'] star_catalog['x'] = reduction_metadata.star_catalog[1]['x_pixel'] star_catalog['y'] = reduction_metadata.star_catalog[1]['y_pixel'] return reduction_metadata, star_catalog
def run_analysis(): (option, input_file, output_dir) = get_args() if option == 'C': xmatch = read_crossmatch_data(input_file) (separations, masked_ra, masked_dec) = calc_astrometric_residuals_xmatch(xmatch) elif option == 'M': meta = metadata.MetaData() meta.load_a_layer_from_file(path.dirname(input_file), path.basename(input_file), 'star_catalog') (separations, masked_ra, masked_dec) = calc_astrometric_residuals_metadata(meta) plot_angular_separations(output_dir, separations, masked_ra, masked_dec)
def extract_star_catalog(params,filter_id,log): """Function to read the photometric and star catalog data from a metadata file""" meta_file = params[filter_id] reduction_metadata = metadata.MetaData() reduction_metadata.load_a_layer_from_file( path.dirname(meta_file), path.basename(meta_file), 'star_catalog' ) reduction_metadata.load_a_layer_from_file( path.dirname(meta_file), path.basename(meta_file), 'phot_calib' ) reduction_metadata.load_a_layer_from_file( path.dirname(meta_file), path.basename(meta_file), 'images_stats' ) star_catalog = Table() star_catalog['star_index'] = reduction_metadata.star_catalog[1]['star_index'] star_catalog['RA'] = reduction_metadata.star_catalog[1]['RA_J2000'] star_catalog['DEC'] = reduction_metadata.star_catalog[1]['DEC_J2000'] star_catalog['mag'] = reduction_metadata.star_catalog[1]['ref_mag'] star_catalog['mag_err'] = reduction_metadata.star_catalog[1]['ref_mag_err'] star_catalog['cal_ref_mag'] = reduction_metadata.phot_calib[1]['cal_ref_mag'] star_catalog['cal_ref_mag_err'] = reduction_metadata.phot_calib[1]['cal_ref_mag_err'] star_catalog['_RAJ2000'] = reduction_metadata.phot_calib[1]['_RAJ2000'] star_catalog['_DEJ2000'] = reduction_metadata.phot_calib[1]['_DEJ2000'] star_catalog['imag'] = reduction_metadata.phot_calib[1]['imag'] star_catalog['e_imag'] = reduction_metadata.phot_calib[1]['e_imag'] star_catalog['rmag'] = reduction_metadata.phot_calib[1]['rmag'] star_catalog['e_rmag'] = reduction_metadata.phot_calib[1]['e_rmag'] star_catalog['gmag'] = reduction_metadata.phot_calib[1]['gmag'] star_catalog['e_gmag'] = reduction_metadata.phot_calib[1]['e_gmag'] log.info('Extracted data for '+str(len(star_catalog))+\ ' stars for dataset '+filter_id) image_table = Table() image_table['im_name'] = reduction_metadata.images_stats[1]['IM_NAME'] image_table['fwhm_x'] = reduction_metadata.images_stats[1]['FWHM_X'] image_table['fwhm_y'] = reduction_metadata.images_stats[1]['FWHM_Y'] image_table['sky'] = reduction_metadata.images_stats[1]['SKY'] log.info('Extracted data for '+str(len(image_table))+\ ' images in the dataset') return star_catalog, image_table
def read_star_catalogs(self): for par in ['b_cat', 'g_cat', 'r_cat']: meta_file = getattr(self, par + '_path') if meta_file != None and path.isfile(meta_file): m = metadata.MetaData() m.load_a_layer_from_file(path.dirname(meta_file), path.basename(meta_file), 'star_catalog') star_catalog = Table() star_catalog['star_index'] = m.star_catalog[1]['star_index'] star_catalog['x'] = m.star_catalog[1]['x_pixel'] star_catalog['y'] = m.star_catalog[1]['y_pixel'] star_catalog['RA'] = m.star_catalog[1]['RA_J2000'] star_catalog['DEC'] = m.star_catalog[1]['DEC_J2000'] setattr(self, par + '_meta', m) setattr(self, par, star_catalog)
def load_pydandia_star_catalog(params): """Function to read the photometric and star catalog data from a metadata file""" meta_file = params['metadata'] reduction_metadata = metadata.MetaData() reduction_metadata.load_a_layer_from_file( path.dirname(meta_file), path.basename(meta_file), 'star_catalog' ) star_catalog = Table() star_catalog['star_index'] = reduction_metadata.star_catalog[1]['star_index'] star_catalog['x'] = reduction_metadata.star_catalog[1]['x_pixel'] star_catalog['y'] = reduction_metadata.star_catalog[1]['y_pixel'] star_catalog['RA'] = reduction_metadata.star_catalog[1]['RA_J2000'] star_catalog['DEC'] = reduction_metadata.star_catalog[1]['DEC_J2000'] star_catalog['mag'] = reduction_metadata.star_catalog[1]['ref_mag'] star_catalog['mag_err'] = reduction_metadata.star_catalog[1]['ref_mag_err'] star_catalog['flux'] = reduction_metadata.star_catalog[1]['ref_flux'] star_catalog['flux_err'] = reduction_metadata.star_catalog[1]['ref_flux_err'] print('Extracted data for '+str(len(star_catalog))+\ ' stars from the pyDANDIA analysis') return star_catalog
def extract_star_catalog(params,filter_id,log): """Function to read the photometric and star catalog data from a metadata file""" meta_file = params[filter_id] reduction_metadata = metadata.MetaData() reduction_metadata.load_a_layer_from_file( path.dirname(meta_file), path.basename(meta_file), 'star_catalog' ) star_catalog = Table() star_catalog['star_index'] = reduction_metadata.star_catalog[1]['star_index'] star_catalog['x'] = reduction_metadata.star_catalog[1]['x_pixel'] star_catalog['y'] = reduction_metadata.star_catalog[1]['y_pixel'] star_catalog['RA'] = reduction_metadata.star_catalog[1]['RA_J2000'] star_catalog['DEC'] = reduction_metadata.star_catalog[1]['DEC_J2000'] star_catalog['mag'] = reduction_metadata.star_catalog[1]['ref_mag'] star_catalog['mag_err'] = reduction_metadata.star_catalog[1]['ref_mag_err'] star_catalog['flux'] = reduction_metadata.star_catalog[1]['ref_flux'] star_catalog['flux_err'] = reduction_metadata.star_catalog[1]['ref_flux_err'] log.info('Extracted data for '+str(len(star_catalog))+\ ' stars for dataset '+filter_id) return star_catalog
def fetch_metadata(setup, params, log): """Function to extract the information necessary for the photometric calibration from a metadata file, adding information to the params dictionary""" reduction_metadata = metadata.MetaData() reduction_metadata.load_a_layer_from_file(setup.red_dir, params['metadata'], 'data_architecture') reduction_metadata.load_a_layer_from_file(setup.red_dir, params['metadata'], 'reduction_parameters') reduction_metadata.load_a_layer_from_file(setup.red_dir, params['metadata'], 'headers_summary') reduction_metadata.load_a_layer_from_file(setup.red_dir, params['metadata'], 'star_catalog') try: reduction_metadata.load_a_layer_from_file(setup.red_dir, params['metadata'], 'phot_calib') except KeyError: pass params['fov'] = reduction_metadata.reduction_parameters[1]['FOV'][0] params['refimage'] = reduction_metadata.data_architecture[1]['REF_IMAGE'][ 0] iref = reduction_metadata.headers_summary[1]['IMAGES'].tolist().index( params['refimage']) params['ra'] = reduction_metadata.headers_summary[1]['RAKEY'][iref] params['dec'] = reduction_metadata.headers_summary[1]['DECKEY'][iref] params['filter'] = reduction_metadata.headers_summary[1]['FILTKEY'][iref] params['cat_mag_col'] = params['filter'].replace('p', '') + 'mag' params['cat_err_col'] = 'e_' + params['filter'].replace('p', '') + 'mag' log.info('Gathered information from metadata file ' + params['metadata'] + ':') log.info('Image field of view: ' + str(params['fov']) + 'sq. deg') log.info('Reference image: '+params['refimage']+\ ', index '+str(iref)+' in dataset') log.info('Filter used for dataset: ' + params['filter']) log.info('Pointing center coordinates: ' + params['ra'] + ' ' + params['dec']) star_catalog = Table() star_catalog['star_index'] = reduction_metadata.star_catalog[1][ 'star_index'] star_catalog['RA'] = reduction_metadata.star_catalog[1]['RA_J2000'] star_catalog['DEC'] = reduction_metadata.star_catalog[1]['DEC_J2000'] star_catalog['mag'] = reduction_metadata.star_catalog[1]['ref_mag'] star_catalog['mag_err'] = reduction_metadata.star_catalog[1]['ref_mag_err'] star_catalog['clean'] = np.zeros( len(reduction_metadata.star_catalog[1]['ref_mag'])) star_catalog['cal_ref_mag'] = np.zeros( len(reduction_metadata.star_catalog[1]['ref_mag'])) star_catalog['cal_ref_mag_err'] = np.zeros( len(reduction_metadata.star_catalog[1]['ref_mag'])) log.info('Extracted star catalog') return reduction_metadata, params, star_catalog
def backup_field_photometry_products(params): """Function to gather all the photometric data products for all datasets for a given field and back them up to another disk""" if params['output_dir'] != None and not os.path.isdir(params['output_dir']): raise IOError('Cannot find output directory '+params['output_dir']) if params['db_path'] and os.path.isfile(params['db_path']): rsync_file(params['db_path'], os.path.join(params['output_dir'], os.path.basename(params['db_path']))) elif not params['db_path']: print('No photometry DB to be backed up') else: print('WARNING: Could not find field database at '+params['db_path']) red_dirs = glob.glob(os.path.join(params['dir_path'],params['field_name']+'_???-dom?-1m0-??-f???_??')) print('Found '+str(len(red_dirs))+' reduction directories') for dir in red_dirs: dset = os.path.basename(dir) staging_dir = os.path.join(params['output_dir'],dset) if not os.path.isdir(staging_dir): os.mkdir(staging_dir) # Backup the main photometry file phot_source_file = os.path.join(dir,'photometry.hdf5') phot_dest_file = os.path.join(staging_dir, 'photometry.hdf5') if os.path.isfile(phot_source_file): rsync_file(phot_source_file, phot_dest_file) comp_phot_dest_file = compress_data_products.bzip2_file(phot_dest_file) else: print('WARNING: Could not find photometry file '+phot_source_file) # Backup the metadata meta_source_file = os.path.join(dir,'pyDANDIA_metadata.fits') meta_dest_file = os.path.join(staging_dir, 'pyDANDIA_metadata.fits') if os.path.isfile(meta_source_file): rsync_file(meta_source_file, meta_dest_file) else: print('WARNING: Could not find metadata file '+meta_source_file) # Backup the reference directory image data if not os.path.isdir(os.path.join(staging_dir,'ref')): os.mkdir(os.path.join(staging_dir,'ref')) if os.path.isfile(meta_source_file) and os.path.isdir(os.path.join(dir, 'ref')): mdata = metadata.MetaData() mdata.load_a_layer_from_file(dir, 'pyDANDIA_metadata.fits', 'data_architecture') source_files = [] dest_files = [] ref_source_file = os.path.join(mdata.data_architecture[1]['REF_PATH'][0],mdata.data_architecture[1]['REF_IMAGE'][0]) dref_source_file = ref_source_file.replace('.fits', '_res.fits') psfstamp_source = os.path.join(mdata.data_architecture[1]['REF_PATH'][0],'final_psf_master_stamp.fits') psfstampvar_source = os.path.join(mdata.data_architecture[1]['REF_PATH'][0],'final_psf_master_stamp_varience.fits') mask_source = os.path.join(mdata.data_architecture[1]['REF_PATH'][0],'master_mask.fits') maskedref_source = os.path.join(mdata.data_architecture[1]['REF_PATH'][0],'masked_ref_image.fits') psfmodel_source = os.path.join(mdata.data_architecture[1]['REF_PATH'][0],'psf_model.fits') psfmodelnorm_source = os.path.join(mdata.data_architecture[1]['REF_PATH'][0],'psf_model_normalized.fits') psfmodelres_source = os.path.join(mdata.data_architecture[1]['REF_PATH'][0],'psf_model_residuals.fits') flist = [ref_source_file, dref_source_file, psfstamp_source, psfstampvar_source, mask_source, maskedref_source, psfmodel_source, psfmodelnorm_source, psfmodelres_source] for f in flist: source_files.append(f) dest_files.append(os.path.join(staging_dir, 'ref', os.path.basename(f))) for i,f in enumerate(source_files): rsync_file(f, dest_files[i]) refpngs = glob.glob(os.path.join(mdata.data_architecture[1]['REF_PATH'][0],'ref','*.png')) for f_source in refpngs: f_dest = os.path.join(staging_dir, 'ref', os.path.basename(f_source)) rsync_file(f_source, f_dest) # Backup the DS9 overlay datafiles overlays = glob.glob(os.path.join(mdata.data_architecture[1]['REF_PATH'][0],'*.reg')) for f_source in overlays: f_dest = os.path.join(staging_dir, 'ref', os.path.basename(f_source)) rsync_file(f_source, f_dest) # Compress reference image for image in glob.glob(os.path.join(staging_dir, 'ref', '*.fits')): if os.path.isfile(image+'.bz2') == False: args = ['bzip2', image] p = subprocess.Popen(args, stdout=subprocess.PIPE) p.wait() else: print('Skipping compression of '+image+' (compressed product already exists)') # Backup the reduction logs log_list = glob.glob(os.path.join(dir,'*.log')) for log_source in log_list: log_dest = os.path.join(staging_dir,os.path.basename(log_source)) rsync_file(log_source, log_dest) # Backup the output plots png_list = glob.glob(os.path.join(dir,'*.png')) for png_source in png_list: png_dest = os.path.join(staging_dir,os.path.basename(png_source)) rsync_file(png_source, png_dest) # Backup the configuration directory config_source = os.path.join(params['dir_path'],'config') config_dest = params['output_dir'] rsync_file(config_source, config_dest) # Backup the field crossmatch file and log: source_files = [ os.path.join(params['dir_path'], params['field_name']+'_field_crossmatch.fits'), os.path.join(params['dir_path'], 'crossmatch.log'), os.path.join(params['dir_path'], 'crossmatch_gaia.log'), ] logs_dest = params['output_dir'] for f in source_files: rsync_file(f, logs_dest) if '.fits' in f: comp_file = compress_data_products.bzip2_file(os.path.join(logs_dest, os.path.basename(f))) # Backup the field photometry files: source_files = [] for q in [1,2,3,4]: source_files.append( os.path.join(params['dir_path'], params['field_name']+'_quad'+str(q)+'_photometry.hdf5') ) logs_dest = params['output_dir'] for f in source_files: rsync_file(f, logs_dest) comp_file = compress_data_products.bzip2_file(os.path.join(logs_dest, os.path.basename(f))) # Backup the logs directory logs_source = os.path.join(params['dir_path'],'logs') logs_dest = params['output_dir'] rsync_file(logs_source, logs_dest) print('Backed-up data products for '+dset+' to '+staging_dir) upload_aws.upload_directory_nochecks(params['output_dir'], params['local_root'], params['aws_root']) print('Backed-up field photometric data products to AWS Cloud')
def run_stage2(setup): """Main driver function to run stage 2: reference selection. This stage is processing the metadata file, looks for the output of stages0 and stage1 and checks if a reference file already exists. It creates a reference frame based on the selection criteria defined in the configuration. If no such configuration exists, it falls back to a standard configuration. If stage1 has failed to produce output it selects a reference based on header information. It always re-runs when called, since it is a lightweight function """ stage2_version = 'stage2 v0.1' log = logs.start_stage_log(setup.red_dir, 'stage2', version=stage2_version) log.info('Setup:\n' + setup.summary() + '\n') reduction_metadata = metadata.MetaData() # Load all metadata try: reduction_metadata.load_all_metadata( metadata_directory=setup.red_dir, metadata_name='pyDANDIA_metadata.fits') # Check data inventory on metadata log.info('stage2 has loaded the reduction metadata') except Exception as estr: log.info('Could not load metadata!' + repr(estr)) status = 'FAILED' report = 'Loading metadata failed:' + repr(estr) return status, report try: n_images = len(reduction_metadata.images_stats) except AttributeError: log.info('stage2: data inventory missing.') status = 'FAILED' report = 'Data inventory (stage1) missing.' logs.close_log(log) return status, report # All parameters are part of metadata table_structure = [['IMAGE_NAME', 'MOON_STATUS', 'RANKING_KEY'], ['S100', 'S100', 'float'], ['degree', None, None]] all_images = reduction_metadata.find_all_images(setup, reduction_metadata, os.path.join( setup.red_dir, 'data'), log=log) reduction_metadata.create_a_new_layer(layer_name='reference_inventory', data_structure=table_structure, data_columns=None) log.info('Create reference frame inventory table in metadata') # Iterate over images that are in the stage inventory reference_ranking = [] fwhm_max = 0. for stats_entry in reduction_metadata.images_stats[1]: if float(stats_entry['FWHM_X']) > fwhm_max: fwhm_max = stats_entry['FWHM_X'] if float(stats_entry['FWHM_Y']) > fwhm_max: fwhm_max = stats_entry['FWHM_Y'] # taking filenames from headers_summary (stage1 change pending) filename_images = reduction_metadata.images_stats[1]['IM_NAME'] data_image_directory = reduction_metadata.data_architecture[1][ 'IMAGES_PATH'][0] max_adu = float(reduction_metadata.reduction_parameters[1]['MAXVAL'][0]) psf_size = int( 4. * float(reduction_metadata.reduction_parameters[1]['KER_RAD'][0]) * fwhm_max) empirical_psf_flag = False if empirical_psf_flag == True: for stats_entry in reduction_metadata.images_stats[1]: image_filename = stats_entry[0] row_idx = np.where(reduction_metadata.images_stats[1]['IM_NAME'] == image_filename)[0][0] moon_status = 'dark' # to be reactivated as soon as it is part of metadata if 'MOONFKEY' in reduction_metadata.headers_summary[1].keys( ) and 'MOONDKEY' in reduction_metadata.headers_summary[1].keys(): moon_status = moon_brightness_header( reduction_metadata.headers_summary[1], row_idx) fwhm_arcsec = (float(stats_entry['FWHM_X'])**2 + float( stats_entry['FWHM_Y'])**2)**0.5 * float( reduction_metadata.reduction_parameters[1]['PIX_SCALE']) # extract data inventory row for image and calculate sorting key # if a sufficient number of stars has been detected at s1 (40) if int(stats_entry['NSTARS']) > 34 and fwhm_arcsec < 3. and ( not 'bright' in moon_status): hdulist = fits.open(os.path.join(data_image_directory, image_filename), memmap=True) image = hdulist[0].data ranking_key = empirical_psf_simple.empirical_snr_subframe( image, psf_size, max_adu) hdulist.close() reference_ranking.append([image_filename, ranking_key]) entry = [image_filename, moon_status, ranking_key] reduction_metadata.add_row_to_layer( key_layer='reference_inventory', new_row=entry) else: for stats_entry in reduction_metadata.images_stats[1]: image_filename = stats_entry[0] row_idx = np.where(reduction_metadata.images_stats[1]['IM_NAME'] == image_filename)[0][0] moon_status = 'dark' # to be reactivated as soon as it is part of metadata if 'MOONFKEY' in reduction_metadata.headers_summary[1].keys( ) and 'MOONDKEY' in reduction_metadata.headers_summary[1].keys(): moon_status = moon_brightness_header( reduction_metadata.headers_summary[1], row_idx) fwhm_arcsec = (float(stats_entry['FWHM_X'])**2 + float( stats_entry['FWHM_Y'])**2)**0.5 * float( reduction_metadata.reduction_parameters[1]['PIX_SCALE']) # extract data inventory row for image and calculate sorting key # if a sufficient number of stars has been detected at s1 (40) if int(stats_entry['NSTARS']) > 34 and fwhm_arcsec < 3. and ( not 'bright' in moon_status): ranking_key = add_stage1_rank(reduction_metadata, stats_entry) reference_ranking.append([image_filename, ranking_key]) entry = [image_filename, moon_status, ranking_key] reduction_metadata.add_row_to_layer( key_layer='reference_inventory', new_row=entry) #relax criteria... if reference_ranking == []: for stats_entry in reduction_metadata.images_stats[1]: image_filename = stats_entry[0] row_idx = np.where(reduction_metadata.images_stats[1]['IM_NAME'] == image_filename)[0][0] moon_status = 'dark' # to be reactivated as soon as it is part of metadata if 'MOONFKEY' in reduction_metadata.headers_summary[1].keys( ) and 'MOONDKEY' in reduction_metadata.headers_summary[1].keys(): moon_status = moon_brightness_header( reduction_metadata.headers_summary[1], row_idx) fwhm_arcsec = (float(stats_entry['FWHM_X'])**2 + float( stats_entry['FWHM_Y'])**2)**0.5 * float( reduction_metadata.reduction_parameters[1]['PIX_SCALE']) # extract data inventory row for image and calculate sorting key if int(stats_entry['NSTARS']) > 20. and fwhm_arcsec < 3.: ranking_key = add_stage1_rank(reduction_metadata, stats_entry) reference_ranking.append([image_filename, ranking_key]) entry = [image_filename, moon_status, ranking_key] reduction_metadata.add_row_to_layer( key_layer='reference_inventory', new_row=entry) # Save the updated layer to the metadata file reduction_metadata.save_a_layer_to_file( metadata_directory=setup.red_dir, metadata_name='pyDANDIA_metadata.fits', key_layer='reference_inventory') if reference_ranking != []: best_image = sorted(reference_ranking, key=itemgetter(1))[-1] ref_directory_path = os.path.join(setup.red_dir, 'ref') if not os.path.exists(ref_directory_path): os.mkdir(ref_directory_path) ref_img_path = os.path.join( str(reduction_metadata.data_architecture[1]['IMAGES_PATH'][0]), best_image[0]) print('New reference ' + best_image[0] + ' in ' + ref_img_path) try: copyfile( reduction_metadata.data_architecture[1]['IMAGES_PATH'][0] + '/' + best_image[0], ref_directory_path + '/' + best_image[0]) except: print('copy ref failed: ', best_image[0]) if not 'REF_PATH' in reduction_metadata.data_architecture[1].keys(): reduction_metadata.add_column_to_layer('data_architecture', 'REF_PATH', [ref_directory_path], new_column_format=None, new_column_unit=None) else: reduction_metadata.update_a_cell_to_layer('data_architecture', 0, 'REF_PATH', ref_directory_path) if not 'REF_IMAGE' in reduction_metadata.data_architecture[1].keys(): reduction_metadata.add_column_to_layer( 'data_architecture', 'REF_IMAGE', [os.path.basename(ref_img_path)], new_column_format=None, new_column_unit=None) else: reduction_metadata.update_a_cell_to_layer( 'data_architecture', 0, 'REF_IMAGE', os.path.basename(ref_img_path)) # Update the REDUCTION_STATUS table in metadata for stage 2 reduction_metadata.update_reduction_metadata_reduction_status( all_images, stage_number=1, status=1, log=log) reduction_metadata.save_updated_metadata( metadata_directory=setup.red_dir, metadata_name='pyDANDIA_metadata.fits') status = 'OK' report = 'Completed successfully' log.info('Updating metadata with info on new images...') logs.close_log(log) return status, report else: status = 'FAILED' report = 'No suitable image found.' log.info('No reference image found...') logs.close_log(log) return status, report
def run_stage6(setup): """Main driver function to run stage 6: image substraction and photometry. This stage align the images to the reference frame! :param object setup : an instance of the ReductionSetup class. See reduction_control.py :return: [status, report, reduction_metadata], the stage4 status, the report, the metadata file :rtype: array_like """ stage6_version = 'stage6 v0.1' log = logs.start_stage_log(setup.red_dir, 'stage6', version=stage6_version) log.info('Setup:\n' + setup.summary() + '\n') # find the metadata reduction_metadata = metadata.MetaData() reduction_metadata.load_all_metadata(setup.red_dir, 'pyDANDIA_metadata.fits') # find the images needed to treat all_images = reduction_metadata.find_all_images(setup, reduction_metadata, os.path.join(setup.red_dir, 'data'), log=log) new_images = reduction_metadata.find_images_need_to_be_process(setup, all_images, stage_number=6, rerun_all=None, log=log) # find the starlist starlist = reduction_metadata.star_catalog[1] max_x = np.max(starlist['x_pixel'].data) max_y = np.max(starlist['y_pixel'].data) mask = (starlist['psf_star'].data == 1) & (starlist['x_pixel'].data<max_x-25) & (starlist['x_pixel'].data>25) & (starlist['y_pixel'].data<max_y-25) & (starlist['y_pixel'].data>25) control_stars = starlist[mask][:10] star_coordinates = np.c_[control_stars['star_index'].data, control_stars['x_pixel'].data, control_stars['y_pixel'].data] for index,key in enumerate(starlist.columns.keys()): if index != 0: ref_star_catalog = np.c_[ref_star_catalog,starlist[key].data] else: ref_star_catalog = starlist[key].data psf_model = fits.open(reduction_metadata.data_architecture[1]['REF_PATH'].data[0]+'/psf_model.fits') psf_type = psf_model[0].header['PSFTYPE'] psf_parameters = [0, psf_model[0].header['Y_CENTER'], psf_model[0].header['X_CENTER'], psf_model[0].header['GAMMA'], psf_model[0].header['ALPHA']] sky_model = sky_background.model_sky_background(setup, reduction_metadata,log,ref_star_catalog) psf_model = psf.get_psf_object( psf_type ) psf_model.update_psf_parameters( psf_parameters) ind = ((starlist['x_pixel']-150)**2<1) & ((starlist['y_pixel']-150)**2<1) print (np.argmin(((starlist['x_pixel']-150)**2) + ((starlist['y_pixel']-150)**2))) if len(new_images) > 0: # find the reference image try: reference_image_name = reduction_metadata.data_architecture[1]['REF_IMAGE'].data[0] reference_image_directory = reduction_metadata.data_architecture[1]['REF_PATH'].data[0] reference_image,date = open_an_image(setup, reference_image_directory, reference_image_name, image_index=0, log=None) ref_image_name = reduction_metadata.data_architecture[1]['REF_IMAGE'].data[0] index_reference = np.where(ref_image_name == reduction_metadata.headers_summary[1]['IMAGES'].data)[0][0] ref_exposure_time = float(reduction_metadata.headers_summary[1]['EXPKEY'].data[index_reference]) logs.ifverbose(log, setup, 'I found the reference frame:' + reference_image_name) except KeyError: logs.ifverbose(log, setup, 'I can not find any reference image! Aboard stage6') status = 'KO' report = 'No reference frame found!' return status, report # find the kernels directory try: kernels_directory = reduction_metadata.data_architecture[1]['OUTPUT_DIRECTORY'].data[0]+'kernel/' logs.ifverbose(log, setup, 'I found the kernels directory:' + kernels_directory) except KeyError: logs.ifverbose(log, setup, 'I can not find the kernels directory! Aboard stage6') status = 'KO' report = 'No kernels directory found!' return status, report data = [] diffim_directory = reduction_metadata.data_architecture[1]['OUTPUT_DIRECTORY'].data[0]+'diffim/' images_directory = reduction_metadata.data_architecture[1]['IMAGES_PATH'].data[0] phot = np.zeros((len(new_images),len(ref_star_catalog),16)) time = [] for idx,new_image in enumerate(new_images): log.info('Starting difference photometry of '+new_image) target_image,date = open_an_image(setup, images_directory, new_image, image_index=0, log=None) kernel_image,kernel_error,kernel_bkg = find_the_associated_kernel(setup, kernels_directory, new_image) difference_image = image_substraction(setup, reduction_metadata,reference_image, kernel_image, new_image)-kernel_bkg time.append(date) save_control_stars_of_the_difference_image(setup, new_image, difference_image, star_coordinates) photometric_table, control_zone = photometry_on_the_difference_image(setup, reduction_metadata, log,ref_star_catalog,difference_image, psf_model, sky_model, kernel_image,kernel_error, ref_exposure_time) phot[idx,:,:] = photometric_table #save_control_zone_of_residuals(setup, new_image, control_zone) #ingest_photometric_table_in_db(setup, photometric_table) import pdb; pdb.set_trace() import matplotlib.pyplot as plt ind = ((starlist['x_pixel']-150)**2<1) & ((starlist['y_pixel']-150)**2<1) plt.errorbar(time,phot[:,ind,8],fmt='.k') plt.show() import pdb; pdb.set_trace() return status, report
def run_stage4(setup): """Main driver function to run stage 4: image alignement. This stage align the images to the reference frame! :param object setup : an instance of the ReductionSetup class. See reduction_control.py :return: [status, report, reduction_metadata], the stage4 status, the report, the metadata file :rtype: array_like """ stage4_version = 'stage4 v0.1' log = logs.start_stage_log(setup.red_dir, 'stage4', version=stage4_version) log.info('Setup:\n' + setup.summary() + '\n') # find the metadata reduction_metadata = metadata.MetaData() reduction_metadata.load_all_metadata(setup.red_dir, 'pyDANDIA_metadata.fits') # find the images needed to treat all_images = reduction_metadata.find_all_images(setup, reduction_metadata, os.path.join( setup.red_dir, 'data'), log=log) new_images = reduction_metadata.find_images_need_to_be_process( setup, all_images, stage_number=4, rerun_all=None, log=log) if len(new_images) > 0: # find the reference image try: reference_image_name = reduction_metadata.data_architecture[1][ 'REF_IMAGE'].data[0] reference_image_directory = reduction_metadata.data_architecture[ 1]['REF_PATH'].data[0] reference_image = open_an_image(setup, reference_image_directory, reference_image_name, image_index=0, log=None) logs.ifverbose( log, setup, 'I found the reference frame:' + reference_image_name) except KeyError: logs.ifverbose(log, setup, 'I can not find any reference image! Abort stage4') status = 'KO' report = 'No reference frame found!' return status, report data = [] images_directory = reduction_metadata.data_architecture[1][ 'IMAGES_PATH'].data[0] for new_image in new_images: target_image = open_an_image(setup, images_directory, new_image, image_index=0, log=None) try: x_new_center, y_new_center, x_shift, y_shift = find_x_y_shifts_from_the_reference_image( setup, reference_image, target_image, edgefraction=0.5, log=None) data.append([new_image, x_shift, y_shift]) logs.ifverbose( log, setup, 'I found the image translation to the reference for frame:' + new_image) except: logs.ifverbose( log, setup, 'I can not find the image translation to the reference for frame:' + new_image + '. Abort stage4!') status = 'KO' report = 'No shift found for image:' + new_image + ' !' return status, report if ('SHIFT_X' in reduction_metadata.images_stats[1].keys()) and ( 'SHIFT_Y' in reduction_metadata.images_stats[1].keys()): for index in range(len(data)): target_image = data[index][0] x_shift = data[index][1] y_shift = data[index][2] row_index = np.where(reduction_metadata.images_stats[1] ['IM_NAME'].data == new_image)[0][0] reduction_metadata.update_a_cell_to_layer( 'images_stats', row_index, 'SHIFT_X', x_shift) reduction_metadata.update_a_cell_to_layer( 'images_stats', row_index, 'SHIFT_Y', y_shift) logs.ifverbose(log, setup, 'Updated metadata for image: ' + target_image) else: logs.ifverbose(log, setup, 'I have to construct SHIFT_X and SHIFT_Y columns') sorted_data = np.copy(data) for index in range(len(data)): target_image = data[index][0] row_index = np.where(reduction_metadata.images_stats[1] ['IM_NAME'].data == new_image)[0][0] sorted_data[row_index] = data[index] column_format = 'int' column_unit = 'pix' reduction_metadata.add_column_to_layer( 'images_stats', 'SHIFT_X', sorted_data[:, 1], new_column_format=column_format, new_column_unit=column_unit) reduction_metadata.add_column_to_layer( 'images_stats', 'SHIFT_Y', sorted_data[:, 2], new_column_format=column_format, new_column_unit=column_unit) reduction_metadata.update_reduction_metadata_reduction_status( new_images, stage_number=4, status=1, log=log) reduction_metadata.save_updated_metadata( reduction_metadata.data_architecture[1]['OUTPUT_DIRECTORY'][0], reduction_metadata.data_architecture[1]['METADATA_NAME'][0], log=log) logs.close_log(log) status = 'OK' report = 'Completed successfully' return status, report
def run_stage5(setup): """Main driver function to run stage 5: kernel_solution This stage finds the kernel solution and (optionally) subtracts the model image :param object setup : an instance of the ReductionSetup class. See reduction_control.py :return: [status, report, reduction_metadata], stage5 status, report, metadata file :rtype: array_like """ stage5_version = 'stage5 v0.1' log = logs.start_stage_log(setup.red_dir, 'stage5', version=stage5_version) log.info('Setup:\n' + setup.summary() + '\n') try: from umatrix_routine import umatrix_construction, umatrix_bvector_construction, bvector_construction except ImportError: log.info( 'Uncompiled cython code, please run setup.py: e.g.\n python setup.py build_ext --inplace' ) status = 'KO' report = 'Uncompiled cython code, please run setup.py: e.g.\n python setup.py build_ext --inplace' return status, report # find the metadata reduction_metadata = metadata.MetaData() reduction_metadata.load_all_metadata(setup.red_dir, 'pyDANDIA_metadata.fits') #determine kernel size based on maximum FWHM fwhm_max = 0. shift_max = 0 for stats_entry in reduction_metadata.images_stats[1]: if float(stats_entry['FWHM_X']) > fwhm_max: fwhm_max = stats_entry['FWHM_X'] if float(stats_entry['FWHM_Y']) > fwhm_max: fwhm_max = stats_entry['FWHM_Y'] if abs(float(stats_entry['SHIFT_X'])) > shift_max: shift_max = abs(float(stats_entry['SHIFT_X'])) if abs(float(stats_entry['SHIFT_Y'])) > shift_max: shift_max = abs(float(stats_entry['SHIFT_Y'])) maxshift = int(shift_max) + 2 #image smaller or equal 500x500 large_format_image = False sigma_max = fwhm_max / (2. * (2. * np.log(2.))**0.5) # Factor 4 corresponds to the radius of 2*FWHM the old pipeline kernel_size = int( 4. * float(reduction_metadata.reduction_parameters[1]['KER_RAD'][0]) * fwhm_max) if kernel_size: if kernel_size % 2 == 0: kernel_size = kernel_size + 1 # find the images that need to be processed all_images = reduction_metadata.find_all_images(setup, reduction_metadata, os.path.join( setup.red_dir, 'data'), log=log) new_images = reduction_metadata.find_images_need_to_be_process( setup, all_images, stage_number=5, rerun_all=None, log=log) kernel_directory_path = os.path.join(setup.red_dir, 'kernel') diffim_directory_path = os.path.join(setup.red_dir, 'diffim') if not os.path.exists(kernel_directory_path): os.mkdir(kernel_directory_path) if not os.path.exists(diffim_directory_path): os.mkdir(diffim_directory_path) reduction_metadata.update_column_to_layer('data_architecture', 'KERNEL_PATH', kernel_directory_path) # difference images are written for verbosity level > 0 reduction_metadata.update_column_to_layer('data_architecture', 'DIFFIM_PATH', diffim_directory_path) data_image_directory = reduction_metadata.data_architecture[1][ 'IMAGES_PATH'][0] ref_directory_path = '.' #For a quick image subtraction, pre-calculate a sufficiently large u_matrix #based on the largest FWHM and store it to disk -> needs config switch try: reference_image_name = str( reduction_metadata.data_architecture[1]['REF_IMAGE'][0]) reference_image_directory = str( reduction_metadata.data_architecture[1]['REF_PATH'][0]) max_adu = 0.3 * float( reduction_metadata.reduction_parameters[1]['MAXVAL'][0]) ref_row_index = np.where( reduction_metadata.images_stats[1]['IM_NAME'] == str( reduction_metadata.data_architecture[1]['REF_IMAGE'][0]))[0][0] ref_fwhm_x = reduction_metadata.images_stats[1][ref_row_index][ 'FWHM_X'] ref_fwhm_y = reduction_metadata.images_stats[1][ref_row_index][ 'FWHM_Y'] ref_sigma_x = ref_fwhm_x / (2. * (2. * np.log(2.))**0.5) ref_sigma_y = ref_fwhm_y / (2. * (2. * np.log(2.))**0.5) ref_stats = [ref_fwhm_x, ref_fwhm_y, ref_sigma_x, ref_sigma_y] logs.ifverbose(log, setup, 'Using reference image:' + reference_image_name) except Exception as e: log.ifverbose(log, setup, 'Reference/Images ! Abort stage5' + str(e)) status = 'KO' report = 'No reference image found!' return status, report, reduction_metadata if not ('SHIFT_X' in reduction_metadata.images_stats[1].keys()) and ( 'SHIFT_Y' in reduction_metadata.images_stats[1].keys()): log.ifverbose(log, setup, 'No xshift! run stage4 ! Abort stage5') status = 'KO' report = 'No alignment data found!' return status, report, reduction_metadata if large_format_image == False: subtract_small_format_image(new_images, reference_image_name, reference_image_directory, reduction_metadata, setup, data_image_directory, kernel_size, max_adu, ref_stats, maxshift, kernel_directory_path, diffim_directory_path, log=log) #append some metric for the kernel, perhaps its scale factor... reduction_metadata.update_reduction_metadata_reduction_status( new_images, stage_number=5, status=1, log=log) logs.close_log(log) status = 'OK' report = 'Completed successfully' return status, report
bb = astropy.coordinates.get_sun(ti) moon.append([aa.ra.value,aa.dec.value]) illumination.append(microlsimulator.moon_illumination(bb,aa).value) Moon = np.array(moon) distance = np.arccos(np.sin(Moon[:,1]*3.14/180)*np.sin(dec*3.14/180)+np.cos(Moon[:,1]*3.14/180)*np.cos(dec*3.14/180)*np.cos((Moon[:,0]-ra)*3.14/180))*180/3.14 return Moon,distance,np.array(illumination) ### Setup paths etc phot_path = '/home/ebachelet/Work/ROMEREA/LIA/ROME1/LSCA_ip/' hdf_files = h5py.File(phot_path+'photometry.hdf5', 'r') reduction_metadata = metadata.MetaData() reduction_metadata.load_all_metadata(metadata_directory=phot_path, metadata_name='pyDANDIA_metadata.fits') target = SkyCoord(ra=267.83589537*units.deg,dec=-30.060817819*units.deg) ### Extract images quality metrics pscale = reduction_metadata.images_stats[1]['PSCALE'] sky = reduction_metadata.images_stats[1]['SKY'] fwhm = reduction_metadata.images_stats[1]['FWHM'] nstars = reduction_metadata.images_stats[1]['NSTARS']