def checkFiles(input):
"""
Checks that input files are in the correct format.
Converts geis and waiver fits files to multiextension fits.
"""
from stsci.tools.check_files import geis2mef, waiver2mef, checkFiles
logger.info("\n\tChecking files %s" % input)
removed_files = []
newfiles = []
if not isinstance(input, list):
input = [input]
for file in input:
try:
imgfits, imgtype = fileutil.isFits(file)
except IOError:
logger.warning(
"\n\tFile %s could not be found, removing it from the input list.\n"
% file)
removed_files.append(file)
continue
# Check for existence of waiver FITS input, and quit if found.
# Or should we print a warning and continue but not use that file
if imgfits:
if imgtype == 'waiver':
newfilename = waiver2mef(file, convert_dq=True)
if newfilename == None:
logger.warning(
"\n\tRemoving file %s from input list - could not convert waiver to mef"
% file)
removed_files.append(file)
else:
newfiles.append(newfilename)
else:
newfiles.append(file)
# If a GEIS image is provided as input, create a new MEF file with
# a name generated using 'buildFITSName()'
# Convert the corresponding data quality file if present
if not imgfits:
newfilename = geis2mef(file, convert_dq=True)
if newfilename == None:
logger.warning(
"\n\tRemoving file %s from input list - could not convert geis to mef"
% file)
removed_files.append(file)
else:
newfiles.append(newfilename)
if removed_files:
logger.warning(
'\n\tThe following files will be removed from the list of files to be processed %s'
% removed_files)
newfiles = checkFiles(newfiles)[0]
logger.info(
"\n\tThese files passed the input check and will be processed: %s" %
newfiles)
return newfiles
def buildFileListOrig(input, output=None, ivmlist=None,
wcskey=None, updatewcs=True, **workinplace):
"""
Builds a file list which has undergone various instrument-specific
checks for input to MultiDrizzle, including splitting STIS associations.
Compared to buildFileList, this version returns the list of the
original file names as specified by the user (e.g., before GEIS->MEF, or
WAIVER FITS->MEF conversion).
"""
# NOTE: original file name is required in order to correctly associate
# user catalog files (e.g., user masks to be used with 'skymatch') with
# corresponding imageObjects.
filelist, output, ivmlist, oldasndict = processFilenames(input,output)
# verify that all input images specified can be updated as needed
filelist = util.verifyFilePermissions(filelist)
if filelist is None or len(filelist) == 0:
return None, None, None, None, None
manageInputCopies(filelist,**workinplace)
# to keep track of the original file names we do the following trick:
# pack filelist with the ivmlist using zip and later unpack the zipped list.
#
# NOTE: this required a small modification of the checkStisFiles function
# in stsci.tools.check_files to be able to handle ivmlists that are tuples.
if ivmlist is None:
ivmlist = len(filelist)*[None]
else:
assert(len(filelist) == len(ivmlist)) #TODO: remove after debugging
ivmlist = list(zip(ivmlist,filelist))
# Check format of FITS files - convert Waiver/GEIS to MEF if necessary
filelist, ivmlist = check_files.checkFITSFormat(filelist, ivmlist)
# check for non-polynomial distortion correction
if not updatewcs:
# with updatewcs turned on, any problems will get resolved
# so we do not need to be concerned about the state of the DGEOFILEs
filelist = checkDGEOFile(filelist)
# run all WCS updating
updated_input = _process_input_wcs(filelist, wcskey, updatewcs)
newfilelist, ivmlist = check_files.checkFiles(updated_input, ivmlist)
if updatewcs:
uw.updatewcs(','.join(set(newfilelist) - set(filelist)))
if len(ivmlist) > 0:
ivmlist, filelist = list(zip(*ivmlist))
else:
filelist = [] # insure that both filelist and ivmlist are defined as empty lists
return newfilelist, ivmlist, output, oldasndict, filelist
def buildFileListOrig(input, output=None, ivmlist=None,
wcskey=None, updatewcs=True, **workinplace):
"""
Builds a file list which has undergone various instrument-specific
checks for input to MultiDrizzle, including splitting STIS associations.
Compared to buildFileList, this version returns the list of the
original file names as specified by the user (e.g., before GEIS->MEF, or
WAIVER FITS->MEF conversion).
"""
# NOTE: original file name is required in order to correctly associate
# user catalog files (e.g., user masks to be used with 'skymatch') with
# corresponding imageObjects.
filelist, output, ivmlist, oldasndict = processFilenames(input,output)
# verify that all input images specified can be updated as needed
filelist = util.verifyFilePermissions(filelist)
if filelist is None or len(filelist) == 0:
return None, None, None, None, None
manageInputCopies(filelist,**workinplace)
# to keep track of the original file names we do the following trick:
# pack filelist with the ivmlist using zip and later unpack the zipped list.
#
# NOTE: this required a small modification of the checkStisFiles function
# in stsci.tools.check_files to be able to handle ivmlists that are tuples.
if ivmlist is None:
ivmlist = len(filelist)*[None]
else:
assert(len(filelist) == len(ivmlist)) #TODO: remove after debugging
ivmlist = list(zip(ivmlist,filelist))
# Check format of FITS files - convert Waiver/GEIS to MEF if necessary
filelist, ivmlist = check_files.checkFITSFormat(filelist, ivmlist)
# check for non-polynomial distortion correction
if not updatewcs:
# with updatewcs turned on, any problems will get resolved
# so we do not need to be concerned about the state of the DGEOFILEs
filelist = checkDGEOFile(filelist)
# run all WCS updating
updated_input = _process_input_wcs(filelist, wcskey, updatewcs)
newfilelist, ivmlist = check_files.checkFiles(updated_input, ivmlist)
if updatewcs:
uw.updatewcs(','.join(set(newfilelist) - set(filelist)))
if len(ivmlist) > 0:
ivmlist, filelist = list(zip(*ivmlist))
else:
filelist = [] # insure that both filelist and ivmlist are defined as empty lists
return newfilelist, ivmlist, output, oldasndict, filelist
def checkFiles(input):
"""
Checks that input files are in the correct format.
Converts geis and waiver fits files to multiextension fits.
"""
from stsci.tools.check_files import geis2mef, waiver2mef, checkFiles
logger.info("\n\tChecking files %s" % input)
removed_files = []
newfiles = []
if not isinstance(input, list):
input = [input]
for file in input:
try:
imgfits, imgtype = fileutil.isFits(file)
except IOError:
logger.warning("File {0} could not be found, removing it from the"
"input list.".format(file))
removed_files.append(file)
continue
# Check for existence of waiver FITS input, and quit if found.
# Or should we print a warning and continue but not use that file
if imgfits:
if imgtype == 'waiver':
newfilename = waiver2mef(file, convert_dq=True)
if newfilename is None:
logger.warning("Could not convert waiver to mef."
"Removing file {0} from input list".format(file))
removed_files.append(file)
else:
newfiles.append(newfilename)
else:
newfiles.append(file)
# If a GEIS image is provided as input, create a new MEF file with
# a name generated using 'buildFITSName()'
# Convert the corresponding data quality file if present
if not imgfits:
newfilename = geis2mef(file, convert_dq=True)
if newfilename is None:
logger.warning("Could not convert file {0} from geis to mef, removing it from input list".format(file))
removed_files.append(file)
else:
newfiles.append(newfilename)
if removed_files:
logger.warning('\n\tThe following files will be removed from the list of files',
'to be processed %s' % removed_files)
newfiles = checkFiles(newfiles)[0]
logger.info("\n\tThese files passed the input check and will be processed: %s" % newfiles)
return newfiles
def updatewcs(input,
vacorr=True,
tddcorr=True,
npolcorr=True,
d2imcorr=True,
checkfiles=True,
verbose=False):
"""
Updates HST science files with the best available calibration information.
This allows users to retrieve from the archive self contained science files
which do not require additional reference files.
Basic WCS keywords are updated in the process and new keywords (following WCS
Paper IV and the SIP convention) as well as new extensions are added to the science files.
Example
-------
>>>from stwcs import updatewcs
>>>updatewcs.updatewcs(filename)
Dependencies
------------
`stsci.tools`
`astropy.io.fits`
`astropy.wcs`
Parameters
----------
input: a python list of file names or a string (wild card characters allowed)
input files may be in fits, geis or waiver fits format
vacorr: boolean
If True, vecocity aberration correction will be applied
tddcorr: boolean
If True, time dependent distortion correction will be applied
npolcorr: boolean
If True, a Lookup table distortion will be applied
d2imcorr: boolean
If True, detector to image correction will be applied
checkfiles: boolean
If True, the format of the input files will be checked,
geis and waiver fits files will be converted to MEF format.
Default value is True for standalone mode.
"""
if verbose == False:
logger.setLevel(100)
else:
formatter = logging.Formatter(
"%(asctime)s - %(name)s - %(levelname)s - %(message)s")
log_filename = 'stwcs.log'
fh = logging.FileHandler(log_filename, mode='w')
fh.setLevel(logging.DEBUG)
fh.setFormatter(formatter)
logger.addHandler(fh)
logger.setLevel(verbose)
args = "vacorr=%s, tddcorr=%s, npolcorr=%s, d2imcorr=%s, checkfiles=%s, \
" % (str(vacorr), str(tddcorr), str(npolcorr), str(d2imcorr),
str(checkfiles))
logger.info('\n\tStarting UPDATEWCS: %s', time.asctime())
files = parseinput.parseinput(input)[0]
logger.info("\n\tInput files: %s, " % [i for i in files])
logger.info("\n\tInput arguments: %s" % args)
if checkfiles:
files = checkFiles(files)
if not files:
print('No valid input, quitting ...\n')
return
for f in files:
acorr = apply_corrections.setCorrections(f, vacorr=vacorr, \
tddcorr=tddcorr,npolcorr=npolcorr, d2imcorr=d2imcorr)
if 'MakeWCS' in acorr and newIDCTAB(f):
logger.warning(
"\n\tNew IDCTAB file detected. All current WCSs will be deleted"
)
cleanWCS(f)
makecorr(f, acorr)
return files
def updatewcs(input, vacorr=True, tddcorr=True, npolcorr=True, d2imcorr=True,
checkfiles=True, verbose=False, use_db=True):
"""
Updates HST science files with the best available calibration information.
This allows users to retrieve from the archive self contained science files
which do not require additional reference files.
Basic WCS keywords are updated in the process and new keywords (following WCS
Paper IV and the SIP convention) as well as new extensions are added to the science files.
Examples
--------
>>> from stwcs import updatewcs
>>> updatewcs.updatewcs(filename)
Dependencies
`stsci.tools`
`astropy.io.fits`
`astropy.wcs`
`requests`
`lxml`
Parameters
----------
input : a python list of file names or a string (wild card
characters allowed) input files may be in fits, geis or
waiver fits format
vacorr : boolean
If True, vecocity aberration correction will be applied
tddcorr : boolean
If True, time dependent distortion correction will be applied
npolcorr : boolean
If True, a Lookup table distortion will be applied
d2imcorr : boolean
If True, detector to image correction will be applied
checkfiles : boolean
If True, the format of the input files will be checked,
geis and waiver fits files will be converted to MEF format.
Default value is True for standalone mode.
use_db : boolean
If True, attempt to add astrometric solutions from the
MAST astrometry database.
Default value is True.
"""
if not verbose:
logger.setLevel(100)
write_db_log = False
else:
fmt = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
formatter = logging.Formatter(fmt)
log_filename = 'stwcs.log'
fh = logging.FileHandler(log_filename, mode='w')
fh.setLevel(logging.DEBUG)
fh.setFormatter(formatter)
logger.addHandler(fh)
logger.setLevel(verbose)
write_db_log = True
args = "vacorr=%s, tddcorr=%s, npolcorr=%s, d2imcorr=%s, checkfiles=%s, \
" % (str(vacorr), str(tddcorr), str(npolcorr),
str(d2imcorr), str(checkfiles))
logger.info('\n\tStarting UPDATEWCS: %s', time.asctime())
toclose = True
if isinstance(input, fits.HDUList):
input = [input]
if isinstance(input, list) and isinstance(input[0], fits.HDUList):
files = input
file_names = [inp.filename() for inp in files]
toclose = False
else:
file_names = parseinput.parseinput(input)[0]
files = []
for item in file_names:
files.append(fits.open(item, mode='update'))
logger.info("\n\tInput files: {}".format(file_names))
logger.info("\n\tInput arguments: %s" % args)
if checkfiles:
files = checkFiles(files)
file_names = [inp.filename() for inp in files]
if not files:
print('No valid input, quitting ...\n')
return
if use_db:
# Establish any available connection to
# an accessible astrometry web-service
astrometry = astrometry_utils.AstrometryDB(write_log=write_db_log)
for f in files:
acorr = apply_corrections.setCorrections(f, vacorr=vacorr, tddcorr=tddcorr,
npolcorr=npolcorr, d2imcorr=d2imcorr)
if 'MakeWCS' in acorr and newIDCTAB(f):
logger.warning("\n\tNew IDCTAB file detected. All current WCSs will be deleted")
cleanWCS(f)
makecorr(f, acorr)
if use_db:
# Add any new astrometry solutions available from
# an accessible astrometry web-service
astrometry.updateObs(f)
if toclose:
f.close()
return file_names
def updatewcs(input, vacorr=True, tddcorr=True, npolcorr=True, d2imcorr=True,
checkfiles=True, verbose=False, use_db=True):
"""
Updates HST science files with the best available calibration information.
This allows users to retrieve from the archive self contained science files
which do not require additional reference files.
Basic WCS keywords are updated in the process and new keywords (following WCS
Paper IV and the SIP convention) as well as new extensions are added to the science files.
Examples
--------
>>> from stwcs import updatewcs
>>> updatewcs.updatewcs(filename)
Dependencies
`stsci.tools`
`astropy.io.fits`
`astropy.wcs`
`requests`
`lxml`
Parameters
----------
input: a python list of file names or a string (wild card
characters allowed) input files may be in fits, geis or
waiver fits format
vacorr: boolean
If True, vecocity aberration correction will be applied
tddcorr: boolean
If True, time dependent distortion correction will be applied
npolcorr: boolean
If True, a Lookup table distortion will be applied
d2imcorr: boolean
If True, detector to image correction will be applied
checkfiles: boolean
If True, the format of the input files will be checked,
geis and waiver fits files will be converted to MEF format.
Default value is True for standalone mode.
use_db: boolean
If True, attempt to add astrometric solutions from the
MAST astrometry database.
Default value is True.
"""
if not verbose:
logger.setLevel(100)
else:
fmt = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
formatter = logging.Formatter(fmt)
log_filename = 'stwcs.log'
fh = logging.FileHandler(log_filename, mode='w')
fh.setLevel(logging.DEBUG)
fh.setFormatter(formatter)
logger.addHandler(fh)
logger.setLevel(verbose)
args = "vacorr=%s, tddcorr=%s, npolcorr=%s, d2imcorr=%s, checkfiles=%s, \
" % (str(vacorr), str(tddcorr), str(npolcorr),
str(d2imcorr), str(checkfiles))
logger.info('\n\tStarting UPDATEWCS: %s', time.asctime())
files = parseinput.parseinput(input)[0]
logger.info("\n\tInput files: %s, " % [i for i in files])
logger.info("\n\tInput arguments: %s" % args)
if checkfiles:
files = checkFiles(files)
if not files:
print('No valid input, quitting ...\n')
return
if use_db:
# Establish any available connection to
# an accessible astrometry web-service
astrometry = astrometry_utils.AstrometryDB()
for f in files:
acorr = apply_corrections.setCorrections(f, vacorr=vacorr, tddcorr=tddcorr,
npolcorr=npolcorr, d2imcorr=d2imcorr)
if 'MakeWCS' in acorr and newIDCTAB(f):
logger.warning("\n\tNew IDCTAB file detected. All current WCSs will be deleted")
cleanWCS(f)
makecorr(f, acorr)
if use_db:
# Add any new astrometry solutions available from
# an accessible astrometry web-service
astrometry.updateObs(f)
return files