Beispiel #1
0
    # Determine the full path to the special region file
    path = fs.join(input_path, arguments.special)

    # Inform the user
    log.info("Loading region indicating areas that require special attention from " + path + " ...")

    # Load the region and create a mask from it
    special_region = Region.from_file(path)

# No special region
else: special_region = None

# -----------------------------------------------------------------

# Create an Extractor instance and configure it according to the command-line arguments
extractor = SourceExtractor.from_arguments(arguments)

# Set the interpolation method (if specified)
if arguments.interpolation is not None: extractor.config.interpolation_method = arguments.interpolation

# Set sigma-clipping
if arguments.disable_sigma_clipping: extractor.config.sigma_clip = False

# Run the extractor
extractor.run(image.frames.primary, galaxy_region, star_region, saturation_region, other_region, galaxy_segments, star_segments, other_segments, animation, special_region)

# -----------------------------------------------------------------

# Save the animation
if animation is not None:
    path = fs.join(output_path, "animation.gif")
def Magic(pod, source_dict, band_dict, kwargs_dict):



    # Run AstroMagic in try-while statements; if debug mode is disabled, an un-star-subtracted map will be used after 10 crashes
    am_crash = 0
    am_fail = True
    while am_fail:
        try:



            # Ensure star-subtraction is actually required
            if kwargs_dict['starsub']!=True:
                return pod
            if band_dict['remove_stars']!=True:
                return pod
            if band_dict['band_name'] in str(source_dict['starsub_bands_exclude']).split(';'):
                print '['+pod['id']+'] User explicitly excluded current band from star subtraction for this source.'
                return pod
            if str(source_dict['starsub_bands_exclude'])=='True':
                print '['+pod['id']+'] User explicitly requested no star subtraction for this source.'
                return pod
            if pod['verbose']: print '['+pod['id']+'] Removing foreground stars and background galaxies with PTS AstroMagic.'



            # The paths to the image and output (absolute or relative to the current working directory)
            in_fitspath = pod['in_fitspath']
            temp_dir_path = pod['temp_dir_path']
            output_path = os.path.join(temp_dir_path, 'AstroMagic', band_dict['band_name'])
            image_path = in_fitspath



            # Setting the log level to "ERROR" disables all output from PTS except error messages (probably want to see those); full options are "DEBUG", "INFO", "WARNING", "ERROR", "SUCCESS"
            logging.setup_log(level="ERROR")



            # The path to the bad region (as explained in the previous script that I sent you)
            bad_region_path = None

            # The FWHM of the image (if known)
            fwhm = band_dict['beam_arcsec'] * Unit("arcsec")

            # Import the image
            importer = ImageImporter()
            importer.run(image_path, bad_region_path=bad_region_path, fwhm=fwhm, find_error_frame=False)

            # Get the imported image
            image = importer.image

            # Get the mask of bad pixels
            bad_mask = image.masks.bad if "bad" in image.masks else None



            # If version of cutout alrady processed by AstroMagic is present, use it; else, commence regular processing
            if os.path.exists( os.path.join( temp_dir_path, 'AstroMagic', band_dict['band_name'], source_dict['name']+'_'+band_dict['band_name']+'_StarSub.fits') ):
                if pod['verbose']: print '['+pod['id']+'] AstroMagic accessing pre-processed data for this map.'
                am_output = astropy.io.fits.getdata( os.path.join( temp_dir_path, 'AstroMagic', band_dict['band_name'], source_dict['name']+'_'+band_dict['band_name']+'_StarSub.fits') )
                pod['cutout'] = am_output
                pod['pre_reg'] = True
                return pod



            # Check that pre-fetched catalogue files exist; if not, return pod
            if os.path.exists(os.path.join(kwargs_dict['temp_dir_path'], 'AstroMagic', 'Stars.cat')) and os.path.exists(os.path.join(kwargs_dict['temp_dir_path'], 'AstroMagic', 'Galaxies.cat')):
                pass
            else:
                return pod

            # The path to the directory where all the output will be placed
            if os.path.exists(os.path.join(temp_dir_path, 'AstroMagic', band_dict['band_name'])):
                shutil.rmtree(os.path.join(temp_dir_path, 'AstroMagic', band_dict['band_name']))
                os.mkdir(os.path.join(temp_dir_path, 'AstroMagic', band_dict['band_name']))
            else:
                os.mkdir(os.path.join(temp_dir_path, 'AstroMagic', band_dict['band_name']))

            # Make copies of pre-fetched catalogues, to prevent simultaneous access conflicts
            shutil.copy(os.path.join(kwargs_dict['temp_dir_path'], 'AstroMagic', 'Stars.cat'), os.path.join(temp_dir_path, 'AstroMagic', band_dict['band_name'], 'Stars.cat'))
            shutil.copy(os.path.join(kwargs_dict['temp_dir_path'], 'AstroMagic', 'Galaxies.cat'), os.path.join(temp_dir_path, 'AstroMagic', band_dict['band_name'], 'Galaxies.cat'))



            # Create a CatalogImporter instance, and run it to import catalogues
            catalog_importer = CatalogImporter()
            catalog_importer.config.stars.use_catalog_file = True
            catalog_importer.config.galaxies.use_catalog_file = True
            catalog_importer.config.stars.catalog_path = os.path.join(temp_dir_path, 'AstroMagic', band_dict['band_name'], 'Stars.cat')
            catalog_importer.config.galaxies.catalog_path = os.path.join(temp_dir_path, 'AstroMagic', band_dict['band_name'], 'Galaxies.cat')
            catalog_importer.run(image.frames.primary)

            # If currently handling cut-down thumbnail, only use relevant portion of catalogues
#            if 'starsub_thumbnail' in pod.keys():
#                if pod['starsub_thumbnail']==True:
            galaxy_catalog_thumb = ThumbCatalogue(pod, source_dict, band_dict, kwargs_dict, catalog_importer)
            catalog_importer.galactic_catalog = galaxy_catalog_thumb



            # Create a SourceFinder instance
            finder = SourceFinder()

            # If you don't want to do the 'find_other_sources' step, comment-out the line below
            finder.config.find_other_sources = False # default is True

            # Downsample map for faster run time
            if int(band_dict['downsample_factor'])>1:
                if pod['verbose']: print '['+pod['id']+'] AstroMagic will run on copy of map downsampled by factor of '+str(int(band_dict['downsample_factor']))+', to improve speed.'
                finder.config.downsample_factor = int( band_dict['downsample_factor'] )

            # Run the source finder
            if pod['verbose']: print '['+pod['id']+'] AstroMagic locating online catalogue sources in map.'
            special_region = None # Not important except for debugging
            ignore_region = None # Not important except when certain areas need to be completely ignored from the extraction procedure
            finder.config.build_catalogs = False # For using pre-fetched catalogue files
            try:
                finder.run(image.frames.primary, catalog_importer.galactic_catalog, catalog_importer.stellar_catalog, special_region, ignore_region, bad_mask)
            except RuntimeError as e:
                if 'found' in e.message:
                    if pod['verbose']: print '['+pod['id']+'] AstroMagic found no sources to remove.'
                    pod['pre_reg'] = True
                    return pod
                else:
                    pdb.set_trace()



            # Save the galaxy region
            galaxy_region = finder.galaxy_region
            galaxy_region_path = filesystem.join(output_path, "galaxies.reg")
            galaxy_region.save(galaxy_region_path)

            # Save the star region
            star_region = finder.star_region
            star_region_path = filesystem.join(output_path, "stars.reg")
            if star_region is not None: star_region.save(star_region_path)

            # Save the saturation region
            saturation_region = finder.saturation_region
            saturation_region_path = filesystem.join(output_path, "saturation.reg")
            if saturation_region is not None: saturation_region.save(saturation_region_path)

            # Save the region of other sources
            other_region = finder.other_region
            other_region_path = filesystem.join(output_path, "other_sources.reg")
            if other_region is not None: other_region.save(other_region_path)



            # Get the segmentation maps (galaxies, stars and other sources) from the SourceFinder
            galaxy_segments = finder.galaxy_segments
            star_segments = finder.star_segments
            other_segments = finder.other_segments

            # Make sure target galaxy isn't identified as star segment
            target_segment = star_segments[ pod['centre_i'], pod['centre_j'] ]
            star_segments[ np.where( star_segments==target_segment ) ] = 0.0




            # Handle stars that have been conflated with the target galaxy
            star_segments = OverlargeStars(pod, star_segments, saturation_region_path, star_region_path, galaxy_region_path, image, source_dict, band_dict, temp_dir_path)

            # Region files can be adjusted by the user; if this is done, they have to be reloaded
            star_region = Region.from_file(star_region_path.replace('.reg','_revised.reg'))
            saturation_region = Region.from_file(saturation_region_path.replace('.reg','_revised.reg'))
            """
            # Remove all but target galaxy from galaxy region file
            ExcessGalaxies(galaxy_region_path, galaxy_principal)
            galaxy_region = Region.from_file(galaxy_region_path.replace('.reg','_revised.reg'))
            """
            # Remopve all galaxies from galaxy region file
            shutil.copy2(galaxy_region_path, galaxy_region_path.replace('.reg','_revised.reg'))
            gal_header = '# Region file format: DS9 version 4.1\n'
            gal_file_new = open( galaxy_region_path.replace('.reg','_revised.reg'), 'w')
            gal_file_new.write(gal_header)
            gal_file_new.close()
            galaxy_region = Region.from_file(galaxy_region_path.replace('.reg','_revised.reg'))



            # Create a map of the the segmentation maps
            segments = Image("segments")

            # Add the segmentation map of the galaxies
            segments.add_frame(galaxy_segments, "galaxies")

            # Add the segmentation map of the saturated stars
            if star_segments is not None: segments.add_frame(star_segments, "stars")

            # Add the segmentation map of the other sources
            if other_segments is not None: segments.add_frame(other_segments, "other_sources")

            # Save the FITS file with the segmentation maps
            path = filesystem.join(output_path, "segments.fits")
            segments.save(path)



            # Create an SourceExtractor instance
            if pod['verbose']: print '['+pod['id']+'] AstroMagic extracting background sources.'
            extractor = SourceExtractor()

            # Run the source extractor
            extractor.run(image.frames.primary, galaxy_region, star_region, saturation_region, other_region, galaxy_segments, star_segments, other_segments)



            # Determine the path to the result
            result_path = filesystem.join( temp_dir_path, 'AstroMagic', band_dict['band_name'], source_dict['name']+'_'+band_dict['band_name']+'_StarSub.fits' )

            # Save the resulting image as a FITS file
            image.frames.primary.save(result_path, header=image.original_header)



            # Grab AstroMagic output and return in pod
            am_output = astropy.io.fits.getdata( result_path )
            pod['cutout'] = am_output
            pod['pre_reg'] = True
            am_fail = False
            return pod



        # Handle exceptions
        except:
            am_crash += 1
            if am_crash>=10:
                if kwargs_dict['debug']:
                    print '['+pod['id']+'] AstroMagic failed for '+pod['id']+'! Suggest debugging.'
                    raise ValueError('AstroMagic failed for '+pod['id']+'! Suggest debugging.')
                else:
                    print '['+pod['id']+'] AstroMagic failed for '+pod['id']+'! Suggest debugging.'
                    return pod
Beispiel #3
0
def Magic(pod, source_dict, band_dict, kwargs_dict):

    # Run AstroMagic in try-while statements; if debug mode is disabled, an un-star-subtracted map will be used after 10 crashes
    am_crash = 0
    am_fail = True
    while am_fail:
        try:

            # Ensure star-subtraction is actually required
            if kwargs_dict['starsub'] != True:
                return pod
            if band_dict['remove_stars'] != True:
                return pod
            if band_dict['band_name'] in str(
                    source_dict['starsub_bands_exclude']).split(';'):
                print '[' + pod[
                    'id'] + '] User explicitly excluded current band from star subtraction for this source.'
                return pod
            if str(source_dict['starsub_bands_exclude']) == 'True':
                print '[' + pod[
                    'id'] + '] User explicitly requested no star subtraction for this source.'
                return pod
            if pod['verbose']:
                print '[' + pod[
                    'id'] + '] Removing foreground stars and background galaxies with PTS AstroMagic.'

            # The paths to the image and output (absolute or relative to the current working directory)
            in_fitspath = pod['in_fitspath']
            temp_dir_path = pod['temp_dir_path']
            output_path = os.path.join(temp_dir_path, 'AstroMagic',
                                       band_dict['band_name'])
            image_path = in_fitspath

            # Setting the log level to "ERROR" disables all output from PTS except error messages (probably want to see those); full options are "DEBUG", "INFO", "WARNING", "ERROR", "SUCCESS"
            logging.setup_log(level="ERROR")

            # The path to the bad region (as explained in the previous script that I sent you)
            bad_region_path = None

            # The FWHM of the image (if known)
            fwhm = band_dict['beam_arcsec'] * Unit("arcsec")

            # Import the image
            importer = ImageImporter()
            importer.run(image_path,
                         bad_region_path=bad_region_path,
                         fwhm=fwhm,
                         find_error_frame=False)

            # Get the imported image
            image = importer.image

            # Get the mask of bad pixels
            bad_mask = image.masks.bad if "bad" in image.masks else None

            # If version of cutout alrady processed by AstroMagic is present, use it; else, commence regular processing
            if os.path.exists(
                    os.path.join(
                        temp_dir_path, 'AstroMagic', band_dict['band_name'],
                        source_dict['name'] + '_' + band_dict['band_name'] +
                        '_StarSub.fits')):
                if pod['verbose']:
                    print '[' + pod[
                        'id'] + '] AstroMagic accessing pre-processed data for this map.'
                am_output = astropy.io.fits.getdata(
                    os.path.join(
                        temp_dir_path, 'AstroMagic', band_dict['band_name'],
                        source_dict['name'] + '_' + band_dict['band_name'] +
                        '_StarSub.fits'))
                pod['cutout'] = am_output
                pod['pre_reg'] = True
                return pod

            # Check that pre-fetched catalogue files exist; if not, return pod
            if os.path.exists(
                    os.path.join(kwargs_dict['temp_dir_path'], 'AstroMagic',
                                 'Stars.cat')) and os.path.exists(
                                     os.path.join(kwargs_dict['temp_dir_path'],
                                                  'AstroMagic',
                                                  'Galaxies.cat')):
                pass
            else:
                return pod

            # The path to the directory where all the output will be placed
            if os.path.exists(
                    os.path.join(temp_dir_path, 'AstroMagic',
                                 band_dict['band_name'])):
                shutil.rmtree(
                    os.path.join(temp_dir_path, 'AstroMagic',
                                 band_dict['band_name']))
            os.mkdir(
                os.path.join(temp_dir_path, 'AstroMagic',
                             band_dict['band_name']))

            # Make copies of pre-fetched catalogues, to prevent simultaneous access conflicts
            shutil.copy(
                os.path.join(kwargs_dict['temp_dir_path'], 'AstroMagic',
                             'Stars.cat'),
                os.path.join(temp_dir_path, 'AstroMagic',
                             band_dict['band_name'], 'Stars.cat'))
            shutil.copy(
                os.path.join(kwargs_dict['temp_dir_path'], 'AstroMagic',
                             'Galaxies.cat'),
                os.path.join(temp_dir_path, 'AstroMagic',
                             band_dict['band_name'], 'Galaxies.cat'))

            # Create a CatalogImporter instance, and run it to import catalogues
            catalog_importer = CatalogImporter()
            catalog_importer.config.stars.use_catalog_file = True
            catalog_importer.config.galaxies.use_catalog_file = True
            catalog_importer.config.stars.catalog_path = os.path.join(
                temp_dir_path, 'AstroMagic', band_dict['band_name'],
                'Stars.cat')
            catalog_importer.config.galaxies.catalog_path = os.path.join(
                temp_dir_path, 'AstroMagic', band_dict['band_name'],
                'Galaxies.cat')
            catalog_importer.run(image.frames.primary)

            # If currently handling cut-down thumbnail, only use relevant portion of catalogues
            #            if 'starsub_thumbnail' in pod.keys():
            #                if pod['starsub_thumbnail']==True:
            galaxy_catalog_thumb = ThumbCatalogue(pod, source_dict, band_dict,
                                                  kwargs_dict,
                                                  catalog_importer)
            catalog_importer.galactic_catalog = galaxy_catalog_thumb

            # Create a SourceFinder instance
            finder = SourceFinder()

            # If you don't want to do the 'find_other_sources' step, comment-out the line below
            finder.config.find_other_sources = False  # default is True

            # Downsample map for faster run time
            if int(band_dict['downsample_factor']) > 1:
                if pod['verbose']:
                    print '[' + pod[
                        'id'] + '] AstroMagic will run on copy of map downsampled by factor of ' + str(
                            int(band_dict['downsample_factor'])
                        ) + ', to improve speed.'
                finder.config.downsample_factor = int(
                    band_dict['downsample_factor'])

            # Run the source finder
            if pod['verbose']:
                print '[' + pod[
                    'id'] + '] AstroMagic locating online catalogue sources in map.'
            special_region = None  # Not important except for debugging
            ignore_region = None  # Not important except when certain areas need to be completely ignored from the extraction procedure
            finder.config.build_catalogs = False  # For using pre-fetched catalogue files
            try:
                finder.run(image.frames.primary,
                           catalog_importer.galactic_catalog,
                           catalog_importer.stellar_catalog, special_region,
                           ignore_region, bad_mask)
            except RuntimeError as e:
                if 'found' in e.message:
                    if pod['verbose']:
                        print '[' + pod[
                            'id'] + '] AstroMagic found no sources to remove.'
                    pod['pre_reg'] = True
                    return pod
                else:
                    pdb.set_trace()

            # Save the galaxy region
            galaxy_region = finder.galaxy_region
            galaxy_region_path = filesystem.join(output_path, "galaxies.reg")
            galaxy_region.save(galaxy_region_path)

            # Save the star region
            star_region = finder.star_region
            star_region_path = filesystem.join(output_path, "stars.reg")
            if star_region is not None: star_region.save(star_region_path)

            # Save the saturation region
            saturation_region = finder.saturation_region
            saturation_region_path = filesystem.join(output_path,
                                                     "saturation.reg")
            if saturation_region is not None:
                saturation_region.save(saturation_region_path)

            # Save the region of other sources
            other_region = finder.other_region
            other_region_path = filesystem.join(output_path,
                                                "other_sources.reg")
            if other_region is not None: other_region.save(other_region_path)

            # Get the segmentation maps (galaxies, stars and other sources) from the SourceFinder
            galaxy_segments = finder.galaxy_segments
            star_segments = finder.star_segments
            other_segments = finder.other_segments

            # Make sure target galaxy isn't identified as star segment
            target_segment = star_segments[int(round(pod['centre_i'])),
                                           int(round(pod['centre_j']))]
            star_segments[np.where(star_segments == target_segment)] = 0.0

            # Handle stars that have been conflated with the target galaxy
            star_segments = OverlargeStars(pod, star_segments,
                                           saturation_region_path,
                                           star_region_path,
                                           galaxy_region_path, image,
                                           source_dict, band_dict,
                                           temp_dir_path)

            # Region files can be adjusted by the user; if this is done, they have to be reloaded
            star_region = Region.from_file(
                star_region_path.replace('.reg', '_revised.reg'))
            saturation_region = Region.from_file(
                saturation_region_path.replace('.reg', '_revised.reg'))
            """
            # Remove all but target galaxy from galaxy region file
            ExcessGalaxies(galaxy_region_path, galaxy_principal)
            galaxy_region = Region.from_file(galaxy_region_path.replace('.reg','_revised.reg'))
            """
            # Remopve all galaxies from galaxy region file
            shutil.copy2(galaxy_region_path,
                         galaxy_region_path.replace('.reg', '_revised.reg'))
            gal_header = '# Region file format: DS9 version 4.1\n'
            gal_file_new = open(
                galaxy_region_path.replace('.reg', '_revised.reg'), 'w')
            gal_file_new.write(gal_header)
            gal_file_new.close()
            galaxy_region = Region.from_file(
                galaxy_region_path.replace('.reg', '_revised.reg'))

            # Create a map of the the segmentation maps
            segments = Image("segments")

            # Add the segmentation map of the galaxies
            segments.add_frame(galaxy_segments, "galaxies")

            # Add the segmentation map of the saturated stars
            if star_segments is not None:
                segments.add_frame(star_segments, "stars")

            # Add the segmentation map of the other sources
            if other_segments is not None:
                segments.add_frame(other_segments, "other_sources")

            # Save the FITS file with the segmentation maps
            path = filesystem.join(output_path, "segments.fits")
            segments.save(path)

            # Create an SourceExtractor instance
            if pod['verbose']:
                print '[' + pod[
                    'id'] + '] AstroMagic extracting background sources.'
            extractor = SourceExtractor()

            # Run the source extractor
            extractor.run(image.frames.primary, galaxy_region, star_region,
                          saturation_region, other_region, galaxy_segments,
                          star_segments, other_segments)

            # Determine the path to the result
            result_path = filesystem.join(
                temp_dir_path, 'AstroMagic', band_dict['band_name'],
                source_dict['name'] + '_' + band_dict['band_name'] +
                '_StarSub.fits')

            # Save the resulting image as a FITS file
            image.frames.primary.save(result_path,
                                      header=image.original_header)

            # Grab AstroMagic output and return in pod
            am_output = astropy.io.fits.getdata(result_path)
            pod['cutout'] = am_output
            pod['pre_reg'] = True
            am_fail = False
            return pod

        # Handle exceptions
        except:
            am_crash += 1
            if am_crash >= 10:
                if kwargs_dict['debug']:
                    print '[' + pod['id'] + '] AstroMagic failed for ' + pod[
                        'id'] + '! Suggest debugging.'
                    raise ValueError('AstroMagic failed for ' + pod['id'] +
                                     '! Suggest debugging.')
                else:
                    print '[' + pod['id'] + '] AstroMagic failed for ' + pod[
                        'id'] + '! Suggest debugging.'
                    return pod
Beispiel #4
0
## Add your image frame(s)
# Output path can be specified seperately for each frame
frame = Frame.from_file("testIm-i.fits")

#finder.add_frame(frame="J000002.00+051717.0_1237678777941229794-g.fits", name=name, output_path='output/')
#finder.add_frame(...)
#...

## Start the source finder:
# various kinds of input can be passed here:
# - frames: a FrameList instance
# - dataset: a DataSet instance
# - output_paths: a dictionary of output paths for each frame
# - star_finder_settings: different settings for each frame
# - ignore: ignore images (list of names)
# - ignore_stars: ignore stars for these images (list of names)
# - ignore_other_sources: ignore other sources for these images (list of names)
# - extended_source_catalog: object of ExtendedSourceCatalog class
# - point_source_catalog: object of PointSourceCatalog class
# CATALOGS ARE USED FOR ALL FRAMES -> IF CATALOG IS DIFFERENT FOR EACH IMAGE, ONLY ADD ONE FRAME TO THE FINDER AND RUN THIS SCRIPT FOR THE DIFFERENT IMAGES
finder.run(frames=dict([(name, frame)]), extended_source_catalog=catalog)

# Start source extraction
extractor = SourceExtractor()
extractor.config.input = output_find_name
extractor.config.output = output_extract_name

extractor.config.remove_companions = True

extractor.run(frame=frame, name=name)