# Determine the full path to the special region file
path = fs.join(input_path, arguments.special)
# Inform the user
log.info("Loading region indicating areas that require special attention from " + path + " ...")
# Load the region and create a mask from it
special_region = Region.from_file(path)
# No special region
else: special_region = None
# -----------------------------------------------------------------
# Create an Extractor instance and configure it according to the command-line arguments
extractor = SourceExtractor.from_arguments(arguments)
# Set the interpolation method (if specified)
if arguments.interpolation is not None: extractor.config.interpolation_method = arguments.interpolation
# Set sigma-clipping
if arguments.disable_sigma_clipping: extractor.config.sigma_clip = False
# Run the extractor
extractor.run(image.frames.primary, galaxy_region, star_region, saturation_region, other_region, galaxy_segments, star_segments, other_segments, animation, special_region)
# -----------------------------------------------------------------
# Save the animation
if animation is not None:
path = fs.join(output_path, "animation.gif")
def Magic(pod, source_dict, band_dict, kwargs_dict):
# Run AstroMagic in try-while statements; if debug mode is disabled, an un-star-subtracted map will be used after 10 crashes
am_crash = 0
am_fail = True
while am_fail:
try:
# Ensure star-subtraction is actually required
if kwargs_dict['starsub']!=True:
return pod
if band_dict['remove_stars']!=True:
return pod
if band_dict['band_name'] in str(source_dict['starsub_bands_exclude']).split(';'):
print '['+pod['id']+'] User explicitly excluded current band from star subtraction for this source.'
return pod
if str(source_dict['starsub_bands_exclude'])=='True':
print '['+pod['id']+'] User explicitly requested no star subtraction for this source.'
return pod
if pod['verbose']: print '['+pod['id']+'] Removing foreground stars and background galaxies with PTS AstroMagic.'
# The paths to the image and output (absolute or relative to the current working directory)
in_fitspath = pod['in_fitspath']
temp_dir_path = pod['temp_dir_path']
output_path = os.path.join(temp_dir_path, 'AstroMagic', band_dict['band_name'])
image_path = in_fitspath
# Setting the log level to "ERROR" disables all output from PTS except error messages (probably want to see those); full options are "DEBUG", "INFO", "WARNING", "ERROR", "SUCCESS"
logging.setup_log(level="ERROR")
# The path to the bad region (as explained in the previous script that I sent you)
bad_region_path = None
# The FWHM of the image (if known)
fwhm = band_dict['beam_arcsec'] * Unit("arcsec")
# Import the image
importer = ImageImporter()
importer.run(image_path, bad_region_path=bad_region_path, fwhm=fwhm, find_error_frame=False)
# Get the imported image
image = importer.image
# Get the mask of bad pixels
bad_mask = image.masks.bad if "bad" in image.masks else None
# If version of cutout alrady processed by AstroMagic is present, use it; else, commence regular processing
if os.path.exists( os.path.join( temp_dir_path, 'AstroMagic', band_dict['band_name'], source_dict['name']+'_'+band_dict['band_name']+'_StarSub.fits') ):
if pod['verbose']: print '['+pod['id']+'] AstroMagic accessing pre-processed data for this map.'
am_output = astropy.io.fits.getdata( os.path.join( temp_dir_path, 'AstroMagic', band_dict['band_name'], source_dict['name']+'_'+band_dict['band_name']+'_StarSub.fits') )
pod['cutout'] = am_output
pod['pre_reg'] = True
return pod
# Check that pre-fetched catalogue files exist; if not, return pod
if os.path.exists(os.path.join(kwargs_dict['temp_dir_path'], 'AstroMagic', 'Stars.cat')) and os.path.exists(os.path.join(kwargs_dict['temp_dir_path'], 'AstroMagic', 'Galaxies.cat')):
pass
else:
return pod
# The path to the directory where all the output will be placed
if os.path.exists(os.path.join(temp_dir_path, 'AstroMagic', band_dict['band_name'])):
shutil.rmtree(os.path.join(temp_dir_path, 'AstroMagic', band_dict['band_name']))
os.mkdir(os.path.join(temp_dir_path, 'AstroMagic', band_dict['band_name']))
else:
os.mkdir(os.path.join(temp_dir_path, 'AstroMagic', band_dict['band_name']))
# Make copies of pre-fetched catalogues, to prevent simultaneous access conflicts
shutil.copy(os.path.join(kwargs_dict['temp_dir_path'], 'AstroMagic', 'Stars.cat'), os.path.join(temp_dir_path, 'AstroMagic', band_dict['band_name'], 'Stars.cat'))
shutil.copy(os.path.join(kwargs_dict['temp_dir_path'], 'AstroMagic', 'Galaxies.cat'), os.path.join(temp_dir_path, 'AstroMagic', band_dict['band_name'], 'Galaxies.cat'))
# Create a CatalogImporter instance, and run it to import catalogues
catalog_importer = CatalogImporter()
catalog_importer.config.stars.use_catalog_file = True
catalog_importer.config.galaxies.use_catalog_file = True
catalog_importer.config.stars.catalog_path = os.path.join(temp_dir_path, 'AstroMagic', band_dict['band_name'], 'Stars.cat')
catalog_importer.config.galaxies.catalog_path = os.path.join(temp_dir_path, 'AstroMagic', band_dict['band_name'], 'Galaxies.cat')
catalog_importer.run(image.frames.primary)
# If currently handling cut-down thumbnail, only use relevant portion of catalogues
# if 'starsub_thumbnail' in pod.keys():
# if pod['starsub_thumbnail']==True:
galaxy_catalog_thumb = ThumbCatalogue(pod, source_dict, band_dict, kwargs_dict, catalog_importer)
catalog_importer.galactic_catalog = galaxy_catalog_thumb
# Create a SourceFinder instance
finder = SourceFinder()
# If you don't want to do the 'find_other_sources' step, comment-out the line below
finder.config.find_other_sources = False # default is True
# Downsample map for faster run time
if int(band_dict['downsample_factor'])>1:
if pod['verbose']: print '['+pod['id']+'] AstroMagic will run on copy of map downsampled by factor of '+str(int(band_dict['downsample_factor']))+', to improve speed.'
finder.config.downsample_factor = int( band_dict['downsample_factor'] )
# Run the source finder
if pod['verbose']: print '['+pod['id']+'] AstroMagic locating online catalogue sources in map.'
special_region = None # Not important except for debugging
ignore_region = None # Not important except when certain areas need to be completely ignored from the extraction procedure
finder.config.build_catalogs = False # For using pre-fetched catalogue files
try:
finder.run(image.frames.primary, catalog_importer.galactic_catalog, catalog_importer.stellar_catalog, special_region, ignore_region, bad_mask)
except RuntimeError as e:
if 'found' in e.message:
if pod['verbose']: print '['+pod['id']+'] AstroMagic found no sources to remove.'
pod['pre_reg'] = True
return pod
else:
pdb.set_trace()
# Save the galaxy region
galaxy_region = finder.galaxy_region
galaxy_region_path = filesystem.join(output_path, "galaxies.reg")
galaxy_region.save(galaxy_region_path)
# Save the star region
star_region = finder.star_region
star_region_path = filesystem.join(output_path, "stars.reg")
if star_region is not None: star_region.save(star_region_path)
# Save the saturation region
saturation_region = finder.saturation_region
saturation_region_path = filesystem.join(output_path, "saturation.reg")
if saturation_region is not None: saturation_region.save(saturation_region_path)
# Save the region of other sources
other_region = finder.other_region
other_region_path = filesystem.join(output_path, "other_sources.reg")
if other_region is not None: other_region.save(other_region_path)
# Get the segmentation maps (galaxies, stars and other sources) from the SourceFinder
galaxy_segments = finder.galaxy_segments
star_segments = finder.star_segments
other_segments = finder.other_segments
# Make sure target galaxy isn't identified as star segment
target_segment = star_segments[ pod['centre_i'], pod['centre_j'] ]
star_segments[ np.where( star_segments==target_segment ) ] = 0.0
# Handle stars that have been conflated with the target galaxy
star_segments = OverlargeStars(pod, star_segments, saturation_region_path, star_region_path, galaxy_region_path, image, source_dict, band_dict, temp_dir_path)
# Region files can be adjusted by the user; if this is done, they have to be reloaded
star_region = Region.from_file(star_region_path.replace('.reg','_revised.reg'))
saturation_region = Region.from_file(saturation_region_path.replace('.reg','_revised.reg'))
"""
# Remove all but target galaxy from galaxy region file
ExcessGalaxies(galaxy_region_path, galaxy_principal)
galaxy_region = Region.from_file(galaxy_region_path.replace('.reg','_revised.reg'))
"""
# Remopve all galaxies from galaxy region file
shutil.copy2(galaxy_region_path, galaxy_region_path.replace('.reg','_revised.reg'))
gal_header = '# Region file format: DS9 version 4.1\n'
gal_file_new = open( galaxy_region_path.replace('.reg','_revised.reg'), 'w')
gal_file_new.write(gal_header)
gal_file_new.close()
galaxy_region = Region.from_file(galaxy_region_path.replace('.reg','_revised.reg'))
# Create a map of the the segmentation maps
segments = Image("segments")
# Add the segmentation map of the galaxies
segments.add_frame(galaxy_segments, "galaxies")
# Add the segmentation map of the saturated stars
if star_segments is not None: segments.add_frame(star_segments, "stars")
# Add the segmentation map of the other sources
if other_segments is not None: segments.add_frame(other_segments, "other_sources")
# Save the FITS file with the segmentation maps
path = filesystem.join(output_path, "segments.fits")
segments.save(path)
# Create an SourceExtractor instance
if pod['verbose']: print '['+pod['id']+'] AstroMagic extracting background sources.'
extractor = SourceExtractor()
# Run the source extractor
extractor.run(image.frames.primary, galaxy_region, star_region, saturation_region, other_region, galaxy_segments, star_segments, other_segments)
# Determine the path to the result
result_path = filesystem.join( temp_dir_path, 'AstroMagic', band_dict['band_name'], source_dict['name']+'_'+band_dict['band_name']+'_StarSub.fits' )
# Save the resulting image as a FITS file
image.frames.primary.save(result_path, header=image.original_header)
# Grab AstroMagic output and return in pod
am_output = astropy.io.fits.getdata( result_path )
pod['cutout'] = am_output
pod['pre_reg'] = True
am_fail = False
return pod
# Handle exceptions
except:
am_crash += 1
if am_crash>=10:
if kwargs_dict['debug']:
print '['+pod['id']+'] AstroMagic failed for '+pod['id']+'! Suggest debugging.'
raise ValueError('AstroMagic failed for '+pod['id']+'! Suggest debugging.')
else:
print '['+pod['id']+'] AstroMagic failed for '+pod['id']+'! Suggest debugging.'
return pod
def Magic(pod, source_dict, band_dict, kwargs_dict):
# Run AstroMagic in try-while statements; if debug mode is disabled, an un-star-subtracted map will be used after 10 crashes
am_crash = 0
am_fail = True
while am_fail:
try:
# Ensure star-subtraction is actually required
if kwargs_dict['starsub'] != True:
return pod
if band_dict['remove_stars'] != True:
return pod
if band_dict['band_name'] in str(
source_dict['starsub_bands_exclude']).split(';'):
print '[' + pod[
'id'] + '] User explicitly excluded current band from star subtraction for this source.'
return pod
if str(source_dict['starsub_bands_exclude']) == 'True':
print '[' + pod[
'id'] + '] User explicitly requested no star subtraction for this source.'
return pod
if pod['verbose']:
print '[' + pod[
'id'] + '] Removing foreground stars and background galaxies with PTS AstroMagic.'
# The paths to the image and output (absolute or relative to the current working directory)
in_fitspath = pod['in_fitspath']
temp_dir_path = pod['temp_dir_path']
output_path = os.path.join(temp_dir_path, 'AstroMagic',
band_dict['band_name'])
image_path = in_fitspath
# Setting the log level to "ERROR" disables all output from PTS except error messages (probably want to see those); full options are "DEBUG", "INFO", "WARNING", "ERROR", "SUCCESS"
logging.setup_log(level="ERROR")
# The path to the bad region (as explained in the previous script that I sent you)
bad_region_path = None
# The FWHM of the image (if known)
fwhm = band_dict['beam_arcsec'] * Unit("arcsec")
# Import the image
importer = ImageImporter()
importer.run(image_path,
bad_region_path=bad_region_path,
fwhm=fwhm,
find_error_frame=False)
# Get the imported image
image = importer.image
# Get the mask of bad pixels
bad_mask = image.masks.bad if "bad" in image.masks else None
# If version of cutout alrady processed by AstroMagic is present, use it; else, commence regular processing
if os.path.exists(
os.path.join(
temp_dir_path, 'AstroMagic', band_dict['band_name'],
source_dict['name'] + '_' + band_dict['band_name'] +
'_StarSub.fits')):
if pod['verbose']:
print '[' + pod[
'id'] + '] AstroMagic accessing pre-processed data for this map.'
am_output = astropy.io.fits.getdata(
os.path.join(
temp_dir_path, 'AstroMagic', band_dict['band_name'],
source_dict['name'] + '_' + band_dict['band_name'] +
'_StarSub.fits'))
pod['cutout'] = am_output
pod['pre_reg'] = True
return pod
# Check that pre-fetched catalogue files exist; if not, return pod
if os.path.exists(
os.path.join(kwargs_dict['temp_dir_path'], 'AstroMagic',
'Stars.cat')) and os.path.exists(
os.path.join(kwargs_dict['temp_dir_path'],
'AstroMagic',
'Galaxies.cat')):
pass
else:
return pod
# The path to the directory where all the output will be placed
if os.path.exists(
os.path.join(temp_dir_path, 'AstroMagic',
band_dict['band_name'])):
shutil.rmtree(
os.path.join(temp_dir_path, 'AstroMagic',
band_dict['band_name']))
os.mkdir(
os.path.join(temp_dir_path, 'AstroMagic',
band_dict['band_name']))
# Make copies of pre-fetched catalogues, to prevent simultaneous access conflicts
shutil.copy(
os.path.join(kwargs_dict['temp_dir_path'], 'AstroMagic',
'Stars.cat'),
os.path.join(temp_dir_path, 'AstroMagic',
band_dict['band_name'], 'Stars.cat'))
shutil.copy(
os.path.join(kwargs_dict['temp_dir_path'], 'AstroMagic',
'Galaxies.cat'),
os.path.join(temp_dir_path, 'AstroMagic',
band_dict['band_name'], 'Galaxies.cat'))
# Create a CatalogImporter instance, and run it to import catalogues
catalog_importer = CatalogImporter()
catalog_importer.config.stars.use_catalog_file = True
catalog_importer.config.galaxies.use_catalog_file = True
catalog_importer.config.stars.catalog_path = os.path.join(
temp_dir_path, 'AstroMagic', band_dict['band_name'],
'Stars.cat')
catalog_importer.config.galaxies.catalog_path = os.path.join(
temp_dir_path, 'AstroMagic', band_dict['band_name'],
'Galaxies.cat')
catalog_importer.run(image.frames.primary)
# If currently handling cut-down thumbnail, only use relevant portion of catalogues
# if 'starsub_thumbnail' in pod.keys():
# if pod['starsub_thumbnail']==True:
galaxy_catalog_thumb = ThumbCatalogue(pod, source_dict, band_dict,
kwargs_dict,
catalog_importer)
catalog_importer.galactic_catalog = galaxy_catalog_thumb
# Create a SourceFinder instance
finder = SourceFinder()
# If you don't want to do the 'find_other_sources' step, comment-out the line below
finder.config.find_other_sources = False # default is True
# Downsample map for faster run time
if int(band_dict['downsample_factor']) > 1:
if pod['verbose']:
print '[' + pod[
'id'] + '] AstroMagic will run on copy of map downsampled by factor of ' + str(
int(band_dict['downsample_factor'])
) + ', to improve speed.'
finder.config.downsample_factor = int(
band_dict['downsample_factor'])
# Run the source finder
if pod['verbose']:
print '[' + pod[
'id'] + '] AstroMagic locating online catalogue sources in map.'
special_region = None # Not important except for debugging
ignore_region = None # Not important except when certain areas need to be completely ignored from the extraction procedure
finder.config.build_catalogs = False # For using pre-fetched catalogue files
try:
finder.run(image.frames.primary,
catalog_importer.galactic_catalog,
catalog_importer.stellar_catalog, special_region,
ignore_region, bad_mask)
except RuntimeError as e:
if 'found' in e.message:
if pod['verbose']:
print '[' + pod[
'id'] + '] AstroMagic found no sources to remove.'
pod['pre_reg'] = True
return pod
else:
pdb.set_trace()
# Save the galaxy region
galaxy_region = finder.galaxy_region
galaxy_region_path = filesystem.join(output_path, "galaxies.reg")
galaxy_region.save(galaxy_region_path)
# Save the star region
star_region = finder.star_region
star_region_path = filesystem.join(output_path, "stars.reg")
if star_region is not None: star_region.save(star_region_path)
# Save the saturation region
saturation_region = finder.saturation_region
saturation_region_path = filesystem.join(output_path,
"saturation.reg")
if saturation_region is not None:
saturation_region.save(saturation_region_path)
# Save the region of other sources
other_region = finder.other_region
other_region_path = filesystem.join(output_path,
"other_sources.reg")
if other_region is not None: other_region.save(other_region_path)
# Get the segmentation maps (galaxies, stars and other sources) from the SourceFinder
galaxy_segments = finder.galaxy_segments
star_segments = finder.star_segments
other_segments = finder.other_segments
# Make sure target galaxy isn't identified as star segment
target_segment = star_segments[int(round(pod['centre_i'])),
int(round(pod['centre_j']))]
star_segments[np.where(star_segments == target_segment)] = 0.0
# Handle stars that have been conflated with the target galaxy
star_segments = OverlargeStars(pod, star_segments,
saturation_region_path,
star_region_path,
galaxy_region_path, image,
source_dict, band_dict,
temp_dir_path)
# Region files can be adjusted by the user; if this is done, they have to be reloaded
star_region = Region.from_file(
star_region_path.replace('.reg', '_revised.reg'))
saturation_region = Region.from_file(
saturation_region_path.replace('.reg', '_revised.reg'))
"""
# Remove all but target galaxy from galaxy region file
ExcessGalaxies(galaxy_region_path, galaxy_principal)
galaxy_region = Region.from_file(galaxy_region_path.replace('.reg','_revised.reg'))
"""
# Remopve all galaxies from galaxy region file
shutil.copy2(galaxy_region_path,
galaxy_region_path.replace('.reg', '_revised.reg'))
gal_header = '# Region file format: DS9 version 4.1\n'
gal_file_new = open(
galaxy_region_path.replace('.reg', '_revised.reg'), 'w')
gal_file_new.write(gal_header)
gal_file_new.close()
galaxy_region = Region.from_file(
galaxy_region_path.replace('.reg', '_revised.reg'))
# Create a map of the the segmentation maps
segments = Image("segments")
# Add the segmentation map of the galaxies
segments.add_frame(galaxy_segments, "galaxies")
# Add the segmentation map of the saturated stars
if star_segments is not None:
segments.add_frame(star_segments, "stars")
# Add the segmentation map of the other sources
if other_segments is not None:
segments.add_frame(other_segments, "other_sources")
# Save the FITS file with the segmentation maps
path = filesystem.join(output_path, "segments.fits")
segments.save(path)
# Create an SourceExtractor instance
if pod['verbose']:
print '[' + pod[
'id'] + '] AstroMagic extracting background sources.'
extractor = SourceExtractor()
# Run the source extractor
extractor.run(image.frames.primary, galaxy_region, star_region,
saturation_region, other_region, galaxy_segments,
star_segments, other_segments)
# Determine the path to the result
result_path = filesystem.join(
temp_dir_path, 'AstroMagic', band_dict['band_name'],
source_dict['name'] + '_' + band_dict['band_name'] +
'_StarSub.fits')
# Save the resulting image as a FITS file
image.frames.primary.save(result_path,
header=image.original_header)
# Grab AstroMagic output and return in pod
am_output = astropy.io.fits.getdata(result_path)
pod['cutout'] = am_output
pod['pre_reg'] = True
am_fail = False
return pod
# Handle exceptions
except:
am_crash += 1
if am_crash >= 10:
if kwargs_dict['debug']:
print '[' + pod['id'] + '] AstroMagic failed for ' + pod[
'id'] + '! Suggest debugging.'
raise ValueError('AstroMagic failed for ' + pod['id'] +
'! Suggest debugging.')
else:
print '[' + pod['id'] + '] AstroMagic failed for ' + pod[
'id'] + '! Suggest debugging.'
return pod
## Add your image frame(s)
# Output path can be specified seperately for each frame
frame = Frame.from_file("testIm-i.fits")
#finder.add_frame(frame="J000002.00+051717.0_1237678777941229794-g.fits", name=name, output_path='output/')
#finder.add_frame(...)
#...
## Start the source finder:
# various kinds of input can be passed here:
# - frames: a FrameList instance
# - dataset: a DataSet instance
# - output_paths: a dictionary of output paths for each frame
# - star_finder_settings: different settings for each frame
# - ignore: ignore images (list of names)
# - ignore_stars: ignore stars for these images (list of names)
# - ignore_other_sources: ignore other sources for these images (list of names)
# - extended_source_catalog: object of ExtendedSourceCatalog class
# - point_source_catalog: object of PointSourceCatalog class
# CATALOGS ARE USED FOR ALL FRAMES -> IF CATALOG IS DIFFERENT FOR EACH IMAGE, ONLY ADD ONE FRAME TO THE FINDER AND RUN THIS SCRIPT FOR THE DIFFERENT IMAGES
finder.run(frames=dict([(name, frame)]), extended_source_catalog=catalog)
# Start source extraction
extractor = SourceExtractor()
extractor.config.input = output_find_name
extractor.config.output = output_extract_name
extractor.config.remove_companions = True
extractor.run(frame=frame, name=name)