def sextractor_version(): """ Return the SExtractor version as a tuple. Run SExtractor with the --version option as its sole argument, capture the standard output and parse it. The version number of SExtractor is returned as a tuple (major, minor, micro), such as (2, 8, 6). SExtractorNotInstalled is raised if its executable cannot be found in the current environment. """ # For example: "SExtractor version 2.8.6 (2009-04-09)" PATTERN = "^SExtractor version (\d\.\d{1,2}\.\d{1,2}) \(\d{4}-\d{2}-\d{2}\)$" for executable in SEXTRACTOR_COMMANDS: if methods.which(executable): break else: msg = "SExtractor not found in the current environment" raise SExtractorNotInstalled(msg) try: with tempfile.TemporaryFile() as fd: args = [executable, '--version'] subprocess.check_call(args, stdout=fd) fd.seek(0) output = fd.readline() version = re.match(PATTERN, output).group(1) # From, for example, '2.8.6' to (2, 8, 6) return tuple(int(x) for x in version.split('.')) except subprocess.CalledProcessError, e: raise SExtractorError(e.returncode, e.cmd)
def sextractor_version(): """ Return the SExtractor version as a tuple. Run SExtractor with the --version option as its sole argument, capture the standard output and parse it. The version number of SExtractor is returned as a tuple (major, minor, micro), such as (2, 8, 6). SExtractorNotInstalled is raised if its executable cannot be found in the current environment. """ # For example: "SExtractor version 2.8.6 (2009-04-09)" PATTERN = "^SExtractor version (\d\.\d{1,2}\.\d{1,2}) \(\d{4}-\d{2}-\d{2}\)$" for executable in SEXTRACTOR_COMMANDS: if methods.which(executable): break else: msg = "SExtractor not found in the current environment" raise SExtractorNotInstalled(msg) try: with tempfile.TemporaryFile() as fd: args = [executable, '--version'] subprocess.check_call(args, stdout = fd) fd.seek(0) output = fd.readline() version = re.match(PATTERN, output).group(1) # From, for example, '2.8.6' to (2, 8, 6) return tuple(int(x) for x in version.split('.')) except subprocess.CalledProcessError, e: raise SExtractorError(e.returncode, e.cmd)
def test_sextractor_version(self): # We have no other way of knowing the SExtractor version that is # installed on the system than running it ourselves with the --version # option and examine its output: if sextractor_version() returns the # tuple (2, 8, 6), for example, that means that the SExtractor version # number must contain the string '2.8.6'. What we are doing, basically, # is to test the functionality the other way around: we take the tuple # that sextractror_version() outputs, transform it back to a string and # verify that it corresponds to what the --version option prints. try: executable = methods.which(*astromatic.SEXTRACTOR_COMMANDS)[0] except IndexError: msg = "SExtractor not found in the current environment" raise astromatic.SExtractorNotInstalled(msg) with tempfile.TemporaryFile() as fd: args = [executable, '--version'] subprocess.check_call(args, stdout=fd) fd.seek(0) # For example: "SExtractor version 2.5.0 (2006-07-14)" stdout = '\n'.join(fd.readlines()) version = astromatic.sextractor_version() # From, for example, (2, 5, 0) to '2.5.0' version_str = '.'.join(str(x) for x in version) self.assertTrue(version_str in stdout) # SExtractorNotInstalled must be raised if no SExtractor executable is # detected. In order to simulate this, mock os.environ and clear the # 'PATH' environment variable. In this manner, as the list of paths to # directories where executables may be found is empty, SExtractor (and # any other command) will appear as not installed on the system. environment_copy = copy.deepcopy(os.environ) with mock.patch.object(os, 'environ', environment_copy) as mocked: mocked['PATH'] = '' with self.assertRaises(astromatic.SExtractorNotInstalled): astromatic.sextractor_version()
def test_sextractor_version(self): # We have no other way of knowing the SExtractor version that is # installed on the system than running it ourselves with the --version # option and examine its output: if sextractor_version() returns the # tuple (2, 8, 6), for example, that means that the SExtractor version # number must contain the string '2.8.6'. What we are doing, basically, # is to test the functionality the other way around: we take the tuple # that sextractror_version() outputs, transform it back to a string and # verify that it corresponds to what the --version option prints. try: executable = methods.which(*astromatic.SEXTRACTOR_COMMANDS)[0] except IndexError: msg = "SExtractor not found in the current environment" raise astromatic.SExtractorNotInstalled(msg) with tempfile.TemporaryFile() as fd: args = [executable, '--version'] subprocess.check_call(args, stdout = fd) fd.seek(0) # For example: "SExtractor version 2.5.0 (2006-07-14)" stdout = '\n'.join(fd.readlines()) version = astromatic.sextractor_version() # From, for example, (2, 5, 0) to '2.5.0' version_str = '.'.join(str(x) for x in version) self.assertTrue(version_str in stdout) # SExtractorNotInstalled must be raised if no SExtractor executable is # detected. In order to simulate this, mock os.environ and clear the # 'PATH' environment variable. In this manner, as the list of paths to # directories where executables may be found is empty, SExtractor (and # any other command) will appear as not installed on the system. environment_copy = copy.deepcopy(os.environ) with mock.patch.object(os, 'environ', environment_copy) as mocked: mocked['PATH'] = '' with self.assertRaises(astromatic.SExtractorNotInstalled): astromatic.sextractor_version()
def astrometry_net(path, ra = None, dec = None, radius = 1, verbosity = 0, timeout = None, options = None): """ Do astrometry on a FITS image using Astrometry.net. Use a local build of the amazing Astrometry.net software [1] in order to compute the astrometric solution of a FITS image. This software has many, many advantages over the well-respected SCAMP, but the most important one is that it is a blind astrometric calibration service. We do not need to know literally anything about the image, including approximate coordinates, scale and equinox. It just works, giving us a new FITS file containing the WCS header. In order for this function to work, you must have built and installed the Astrometry.net code in your machine [2]. The main high-level command-line user interface, 'solve-field', is expected to be available in your PATH; otherwise, the AstrometryNetNotInstalled exception is raised. Note that you also need to download the appropriate index files, which are considerably heavy. At the time of this writing, the entire set of indexes built from the 2MASS catalog [4] has a total size of ~32 gigabytes. Raises AstrometryNetError if Astrometry.net exits with a non-zero status code, AstrometryNetTimeoutExpired if the 'timeout' limit is exceeded and AstrometryNetUnsolvedField if the CPU time limit, set in the backend.cfg file (by default located in /usr/local/astrometry/etc/) is hit. [1] http://astrometry.net/ [2] http://astrometry.net/doc/build.html [3] http://astrometry.net/doc/readme.html#getting-index-files [4] http://data.astrometry.net/4200/ [5] https://groups.google.com/d/msg/astrometry/ORVkOk0jSZg/PeCMeAJodyAJ Keyword arguments: ra, dec, radius - restrict the Astrometry.net search to those indexes within 'radius' degrees of the field center given by ('ra', 'dec'). Both the right ascension and declination must be given in order for this feature to work. The three arguments must be expressed in degrees. verbosity - the verbosity level. The default value is zero, meaning that the function executes silently. A value of one makes both the standard output and standard error of Astrometry.net visible. Above that, the number of -v flags send to it equals the value of the argument minus one. For example: verbosity = 3 allows us to see stdout and stderr, and calls Astrometry.net with two -v flags. Most of the time, verbosities greater than one are only needed for debugging. timeout - the maximum number of seconds that Astrometry.net spends on the image before giving up and raising AstrometryNetTimeoutExpired. Note that the backend configuration file (astrometry.cfg) puts a limit on the CPU time that is spent on an image: this can reduce that value but not increase it. options - a dictionary, containing additional options to be passed to solve-field. Each option must map to the corresponding argument (for example, {'--downsample' : '2'}), except in case they do not take any, when they must map to None (e.g., {'--invert' : None}). Both options and values should be given as strings, but they will be automatically cast to string just to be safe. """ emsg = "'%s' not found in the current environment" if not methods.which(ASTROMETRY_COMMAND): raise AstrometryNetNotInstalled(emsg % ASTROMETRY_COMMAND) basename = os.path.basename(path) root, ext = os.path.splitext(basename) # Place all output files in this directory kwargs = dict(prefix = root + '_', suffix = '_astrometry.net') output_dir = tempfile.mkdtemp(**kwargs) # Path to the temporary FITS file containing the WCS header kwargs = dict(prefix = '%s_astrometry_' % root, suffix = ext) with tempfile.NamedTemporaryFile(**kwargs) as fd: output_path = fd.name # If the field solved, Astrometry.net creates a <base>.solved output file # that contains (binary) 1. That is: if this file does not exist, we know # that an astrometric solution could not be found. solved_file = os.path.join(output_dir, root + '.solved') # --dir: place all output files in the specified directory. # --no-plots: don't create any plots of the results. # --new-fits: the new FITS file containing the WCS header. # --no-fits2fits: don't sanitize FITS files; assume they're already valid. # --overwrite: overwrite output files if they already exist. args = [ASTROMETRY_COMMAND, path, '--dir', output_dir, '--no-plots', '--new-fits', output_path, #'--no-fits2fits', '--overwrite'] # -3 / --ra <degrees or hh:mm:ss>: only search in indexes within 'radius' # of the field center given by 'ra' and 'dec' # -4 / --dec <degrees or [+-]dd:mm:ss>: only search in indexes within # 'radius' of the field center given by 'ra' and 'dec' # -5 / --radius <degrees>: only search in indexes within 'radius' of the # field center given by ('ra', 'dec') if ra is not None: args += ['--ra', '%f' % ra] if dec is not None: args += ['--dec', '%f' % dec] if radius is not None: args += ['--radius', '%f' % radius] # -v / --verbose: be more chatty -- repeat for even more verboseness. A # value of 'verbosity' equal to zero means that both the standard output # and error of Astrometry.net and redirected to the null device. Above # that, we send 'verbosity' minus one -v flags to Astrometry.net. if verbosity > 1: args.append('-%s' % ('v' * (verbosity - 1))) # If additional options for solve-field have been specified, append them to # the argument list. All options are assumed to take an argument, except if # they are mapped to None. In this manner, {'--downsample' : 2} is appended # to the argument list as ['--downsample', '2'] (note the automatic cast to # string), while {'--invert' : None} appends only '--invert'. if options: for opt, value in options.iteritems(): opt = str(opt) if value is None: args.append(opt) else: args += [opt, str(value)] # Needed when 'verbosity' is 0 null_fd = open(os.devnull, 'w') try: kwargs = dict(timeout = timeout) if not verbosity: kwargs['stdout'] = kwargs['stderr'] = null_fd subprocess.check_call(args, **kwargs) # .solved file must exist and contain a binary one with open(solved_file, 'rb') as fd: if ord(fd.read()) != 1: raise AstrometryNetUnsolvedField(path) return output_path except subprocess.CalledProcessError, e: raise AstrometryNetError(e.returncode, e.cmd)
def sextractor(path, ext=0, options=None, stdout=None, stderr=None): """ Run SExtractor on the image and return the path to the output catalog. This function runs SExtractor on 'path', using the configuration files defined in the module-level variables SEXTRACTOR_CONFIG, SEXTRACTOR_PARAMS, SEXTRACTOR_FILTER and SEXTRACTOR_STARNNW. It returns the path to the output catalog, which is saved to a temporary location and for whose deletion when it is no longer needed the user is responsible. The SExtractorNotInstalled exception is raised if a SExtractor executable cannot be found, and IOError if any of the four SExtractor configuration files does not exist or is not readable. If a SExtractor version earlier than SEXTRACTOR_REQUIRED_VERSION is installed, SExtractorUpgradeRequired is raised; this is necessary because the syntax that allows us to select on which extension sources are detected was not added until version 2.8.6. Any errors thrown by SExtractor are propagated as SExtractorError exceptions. Lastly, TypeEror is raised if (a) 'ext' is not an integer or (b) 'options' is not a dictionary or any of its keys or values is not a string. Keyword arguments: ext - for multi-extension FITS images, the index of the extension on which SExtractor will be run. It defaults to zero, meaning that sources are detected on the first extension of the FITS image. If a nonexistent extension is specified, the execution of SExtractor fails and the SExtractorError exception is raised. options - a dictionary mapping each SExtractor parameter to its value, and that will override their definition in the configuration files or any default value. In this manner, it is possible to execute SExtractor with different parameters without having to modify the configuration files. For example, {'CLEAN' : 'N', 'CLEAN_PARAM' : '1.1'}, would make SExtractor run with the parameters 'CLEAN' set to 'N' and 'CLEAN_PARAM' set to 1.1, regardless of what the configuration files say. All the keys and values in this dictionary must be strings. stdout - standard output file handle. If None, no redirection will occur. stderr - standard error file handle. If None, no redirection will occur. """ # It is easier to ask forgiveness than permission, yes, but checking the # type here helps avoid some subtle errors. If, say, 'ext' is assigned a # value of 3.8, we do not want it to be silently casted (and truncated) # to three; it is much better (and safer) to have TypeError raised and # let the user know that an invalid, non-integer index was given. if not isinstance(ext, (int, long)): raise TypeError("'ext' must be an integer") for executable in SEXTRACTOR_COMMANDS: if methods.which(executable): break else: msg = "SExtractor not found in the current environment" raise SExtractorNotInstalled(msg) if sextractor_version() < SEXTRACTOR_REQUIRED_VERSION: # From, for example, (2, 8, 6) to '2.8.6' version_str = '.'.join(str(x) for x in SEXTRACTOR_REQUIRED_VERSION) msg = "SExtractor version %s or newer is needed" % version_str raise SExtractorUpgradeRequired(msg) # If the loop did not break (and thus SExtractorNotInstalled was not # raised), 'executable' contains the first command that was found root, _ = os.path.splitext(os.path.basename(path)) catalog_fd, catalog_path = \ tempfile.mkstemp(prefix = '%s_' % root, suffix = '.cat') os.close(catalog_fd) # Raise IOError if any of the configuration files is nonexistent or not # readable. We cannot trust that SExtractor will fail when this happens as # it may not abort the execution, but instead just issue a warning and use # the internal defaults. As of version 2.8.6, only -PARAMETERS_NAME and # -FILTER_NAME, if unreadable, cause the execution of SExtractor to fail. for config_file in (SEXTRACTOR_CONFIG, SEXTRACTOR_PARAMS, SEXTRACTOR_FILTER, SEXTRACTOR_STARNNW): if not os.path.exists(config_file): msg = "configuration file %s not found" raise IOError(msg % config_file) if not os.access(config_file, os.R_OK): msg = "configuration file %s cannot be read" raise IOError(msg % config_file) args = [ executable, path + '[%d]' % ext, '-c', SEXTRACTOR_CONFIG, '-PARAMETERS_NAME', SEXTRACTOR_PARAMS, '-FILTER_NAME', SEXTRACTOR_FILTER, '-STARNNW_NAME', SEXTRACTOR_STARNNW, '-CATALOG_NAME', catalog_path ] if options: try: for key, value in options.iteritems(): args += ['-%s' % key, value] except AttributeError: msg = "'options' must be a dictionary" raise TypeError(msg) try: subprocess.check_call(args, stdout=stdout, stderr=stderr) return catalog_path except subprocess.CalledProcessError, e: try: os.unlink(catalog_path) except (IOError, OSError): pass raise SExtractorError(e.returncode, e.cmd)
def astrometry_net(path, ra = None, dec = None, radius = 1, verbosity = 0, timeout = None): """ Do astrometry on a FITS image using Astrometry.net. Use a local build of the amazing Astrometry.net software [1] in order to compute the astrometric solution of a FITS image. This software has many, many advantages over the well-respected SCAMP, but the most important one is that it is a blind astrometric calibration service. We do not need to know literally anything about the image, including approximate coordinates, scale and equinox. It just works, giving us a new FITS file containing the WCS header. In order for this function to work, you must have built and installed the Astrometry.net code in your machine [2]. The main high-level command-line user interface, 'solve-field', is expected to be available in your PATH; otherwise, the AstrometryNetNotInstalled exception is raised. Note that you also need to download the appropriate index files, which are considerably heavy. At the time of this writing, the entire set of indexes built from the 2MASS catalog [4] has a total size of ~32 gigabytes. Raises AstrometryNetError if Astrometry.net exits with a non-zero status code, AstrometryNetTimeoutExpired if the 'timeout' limit is exceeded and AstrometryNetUnsolvedField if the CPU time limit, set in the backend.cfg file (by default located in /usr/local/astrometry/etc/) is hit. [1] http://astrometry.net/ [2] http://astrometry.net/doc/build.html [3] http://astrometry.net/doc/readme.html#getting-index-files [4] http://data.astrometry.net/4200/ [5] https://groups.google.com/d/msg/astrometry/ORVkOk0jSZg/PeCMeAJodyAJ Keyword arguments: ra, dec, radius - restrict the Astrometry.net search to those indexes within 'radius' degrees of the field center given by ('ra', 'dec'). Both the right ascension and declination must be given in order for this feature to work. The three arguments must be expressed in degrees. verbosity - the verbosity level. The default value is zero, meaning that the function executes silently. A value of one makes both the standard output and standard error of Astrometry.net visible. Above that, the number of -v flags send to it equals the value of the argument minus one. For example: verbosity = 3 allows us to see stdout and stderr, and calls Astrometry.net with two -v flags. Most of the time, verbosities greater than one are only needed for debugging. timeout - the maximum number of seconds that Astrometry.net spends on the image before giving up and raising AstrometryNetTimeoutExpired. Note that the backend configuration file (astrometry.cfg) puts a limit on the CPU time that is spent on an image: this can reduce that value but not increase it. """ emsg = "'%s' not found in the current environment" if not methods.which(ASTROMETRY_COMMAND): raise AstrometryNetNotInstalled(emsg % ASTROMETRY_COMMAND) img = fitsimage.FITSImage(path) tempfile_prefix = '%s_' % img.basename_woe # Place all output files in this directory kwargs = dict(prefix = tempfile_prefix, suffix = '_astrometry.net') output_dir = tempfile.mkdtemp(**kwargs) # Path to the temporary FITS file containing the WCS header root, ext = os.path.splitext(img.basename) kwargs = dict(prefix = '%s_astrometry_' % root, suffix = ext) with tempfile.NamedTemporaryFile(**kwargs) as fd: output_path = fd.name # If the field solved, Astrometry.net creates a <base>.solved output file # that contains (binary) 1. That is: if this file does not exist, we know # that an astrometric solution could not be found. solved_file = os.path.join(output_dir, root + '.solved') # --dir: place all output files in the specified directory. # --no-plots: don't create any plots of the results. # --new-fits: the new FITS file containing the WCS header. # --no-fits2fits: don't sanitize FITS files; assume they're already valid. # --overwrite: overwrite output files if they already exist. args = [ASTROMETRY_COMMAND, path, '--dir', output_dir, '--no-plots', '--new-fits', output_path, '--no-fits2fits', '--overwrite'] # -3 / --ra <degrees or hh:mm:ss>: only search in indexes within 'radius' # of the field center given by 'ra' and 'dec' # -4 / --dec <degrees or [+-]dd:mm:ss>: only search in indexes within # 'radius' of the field center given by 'ra' and 'dec' # -5 / --radius <degrees>: only search in indexes within 'radius' of the # field center given by ('ra', 'dec') if ra is not None: args += ['--ra', '%f' % ra] if dec is not None: args += ['--dec', '%f' % dec] if radius is not None: args += ['--radius', '%f' % radius] # -v / --verbose: be more chatty -- repeat for even more verboseness. A # value of 'verbosity' equal to zero means that both the standard output # and error of Astrometry.net and redirected to the null device. Above # that, we send 'verbosity' minus one -v flags to Astrometry.net. if verbosity > 1: args.append('-%s' % ('v' * (verbosity - 1))) # Needed when 'verbosity' is 0 null_fd = open(os.devnull, 'w') try: kwargs = dict(timeout = timeout) if not verbosity: kwargs['stdout'] = kwargs['stderr'] = null_fd subprocess.check_call(args, **kwargs) # .solved file must exist and contain a binary one with open(solved_file, 'rb') as fd: if ord(fd.read()) != 1: raise AstrometryNetUnsolvedField(path) return output_path except subprocess.CalledProcessError, e: raise AstrometryNetError(e.returncode, e.cmd)
def sextractor(path, ext = 0, options = None, stdout = None, stderr = None): """ Run SExtractor on the image and return the path to the output catalog. This function runs SExtractor on 'path', using the configuration files defined in the module-level variables SEXTRACTOR_CONFIG, SEXTRACTOR_PARAMS, SEXTRACTOR_FILTER and SEXTRACTOR_STARNNW. It returns the path to the output catalog, which is saved to a temporary location and for whose deletion when it is no longer needed the user is responsible. The SExtractorNotInstalled exception is raised if a SExtractor executable cannot be found, and IOError if any of the four SExtractor configuration files does not exist or is not readable. If a SExtractor version earlier than SEXTRACTOR_REQUIRED_VERSION is installed, SExtractorUpgradeRequired is raised; this is necessary because the syntax that allows us to select on which extension sources are detected was not added until version 2.8.6. Any errors thrown by SExtractor are propagated as SExtractorError exceptions. Lastly, TypeEror is raised if (a) 'ext' is not an integer or (b) 'options' is not a dictionary or any of its keys or values is not a string. Keyword arguments: ext - for multi-extension FITS images, the index of the extension on which SExtractor will be run. It defaults to zero, meaning that sources are detected on the first extension of the FITS image. If a nonexistent extension is specified, the execution of SExtractor fails and the SExtractorError exception is raised. options - a dictionary mapping each SExtractor parameter to its value, and that will override their definition in the configuration files or any default value. In this manner, it is possible to execute SExtractor with different parameters without having to modify the configuration files. For example, {'CLEAN' : 'N', 'CLEAN_PARAM' : '1.1'}, would make SExtractor run with the parameters 'CLEAN' set to 'N' and 'CLEAN_PARAM' set to 1.1, regardless of what the configuration files say. All the keys and values in this dictionary must be strings. stdout - standard output file handle. If None, no redirection will occur. stderr - standard error file handle. If None, no redirection will occur. """ # It is easier to ask forgiveness than permission, yes, but checking the # type here helps avoid some subtle errors. If, say, 'ext' is assigned a # value of 3.8, we do not want it to be silently casted (and truncated) # to three; it is much better (and safer) to have TypeError raised and # let the user know that an invalid, non-integer index was given. if not isinstance(ext, (int, long)): raise TypeError("'ext' must be an integer") for executable in SEXTRACTOR_COMMANDS: if methods.which(executable): break else: msg = "SExtractor not found in the current environment" raise SExtractorNotInstalled(msg) if sextractor_version() < SEXTRACTOR_REQUIRED_VERSION: # From, for example, (2, 8, 6) to '2.8.6' version_str = '.'.join(str(x) for x in SEXTRACTOR_REQUIRED_VERSION) msg = "SExtractor version %s or newer is needed" % version_str raise SExtractorUpgradeRequired(msg) # If the loop did not break (and thus SExtractorNotInstalled was not # raised), 'executable' contains the first command that was found root, _ = os.path.splitext(os.path.basename(path)) catalog_fd, catalog_path = \ tempfile.mkstemp(prefix = '%s_' % root, suffix = '.cat') os.close(catalog_fd) # Raise IOError if any of the configuration files is nonexistent or not # readable. We cannot trust that SExtractor will fail when this happens as # it may not abort the execution, but instead just issue a warning and use # the internal defaults. As of version 2.8.6, only -PARAMETERS_NAME and # -FILTER_NAME, if unreadable, cause the execution of SExtractor to fail. for config_file in (SEXTRACTOR_CONFIG, SEXTRACTOR_PARAMS, SEXTRACTOR_FILTER, SEXTRACTOR_STARNNW): if not os.path.exists(config_file): msg = "configuration file %s not found" raise IOError(msg % config_file) if not os.access(config_file, os.R_OK): msg = "configuration file %s cannot be read" raise IOError(msg % config_file) args = [executable, path + '[%d]' % ext, '-c', SEXTRACTOR_CONFIG, '-PARAMETERS_NAME', SEXTRACTOR_PARAMS, '-FILTER_NAME', SEXTRACTOR_FILTER, '-STARNNW_NAME', SEXTRACTOR_STARNNW, '-CATALOG_NAME', catalog_path] if options: try: for key, value in options.iteritems(): args += ['-%s' % key, value] except AttributeError: msg = "'options' must be a dictionary" raise TypeError(msg) try: subprocess.check_call(args, stdout = stdout, stderr = stderr) return catalog_path except subprocess.CalledProcessError, e: try: os.unlink(catalog_path) except (IOError, OSError): pass raise SExtractorError(e.returncode, e.cmd)