def fromhdulist(cls, hdulist, compress=False):
"""
Creates a new FitsHDU from a given HDUList object.
Parameters
----------
hdulist : HDUList
A valid Headerlet object.
compress : bool (optional)
Gzip compress the FITS file
"""
fileobj = bs = BytesIO()
if compress:
if hasattr(hdulist, '_file'):
name = fileobj_name(hdulist._file)
else:
name = None
fileobj = gzip.GzipFile(name, mode='wb', fileobj=bs)
hdulist.writeto(fileobj)
if compress:
fileobj.close()
bs.seek(0)
cards = [
('XTENSION', cls._extension, 'FITS extension'),
('BITPIX', 8, 'array data type'),
('NAXIS', 1, 'number of array dimensions'),
('NAXIS1', len(bs.getvalue()), 'Axis length'),
('PCOUNT', 0, 'number of parameters'),
('GCOUNT', 1, 'number of groups'),
]
# Add the XINDn keywords proposed by Perry, though nothing is done with
# these at the moment
if len(hdulist) > 1:
for idx, hdu in enumerate(hdulist[1:]):
cards.append(('XIND' + str(idx + 1), hdu._hdrLoc,
'byte offset of extension %d' % (idx + 1)))
cards.append(('COMPRESS', compress, 'Uses gzip compression'))
header = Header(cards)
# TODO: This wrapping of the fileobj should probably be handled by
# cls.fromstring, though cls.fromstring itself has a strange
# implementation that I probably need to fix. For example, it
# shouldn't care about fileobjs. There should be a _BaseHDU.fromfile
# for that (there is _BaseHDU.readfrom which plays that role, but its
# semantics are also a little unclear...)
return cls.fromstring(header, fileobj=_File(bs))
def append(filename, data, header=None, checksum=False, verify=True, **kwargs):
"""
Append the header/data to FITS file if filename exists, create if not.
If only `data` is supplied, a minimal header is created.
Parameters
----------
filename : file path, file object, or file like object
File to write to. If opened, must be opened for update (rb+)
unless it is a new file, then it must be opened for append
(ab+). A file or `GzipFile` object opened for update will be
closed after return.
data : array, table, or group data object
the new data used for appending
header : `Header` object (optional)
The header associated with `data`. If `None`, an appropriate
header will be created for the data object supplied.
checksum : bool (optional)
When `True` adds both ``DATASUM`` and ``CHECKSUM`` cards to
the header of the HDU when written to the file.
verify: bool (optional)
When `True`, the existing FITS file will be read in to verify
it for correctness before appending. When `False`, content is
simply appended to the end of the file. Setting *verify* to
`False` can be much faster.
kwargs
Any additional keyword arguments to be passed to `pyfits.open`.
"""
name, closed, noexist_or_empty = _stat_filename_or_fileobj(filename)
if noexist_or_empty:
#
# The input file or file like object either doesn't exits or is
# empty. Use the writeto convenience function to write the
# output to the empty object.
#
writeto(filename, data, header, checksum=checksum, **kwargs)
else:
hdu = _makehdu(data, header)
if isinstance(hdu, PrimaryHDU):
hdu = ImageHDU(data, header)
if verify or not closed:
f = fitsopen(filename, mode='append')
f.append(hdu)
# Set a flag in the HDU so that only this HDU gets a checksum when
# writing the file.
hdu._output_checksum = checksum
f.close(closed=closed)
else:
f = _File(filename, mode='append')
hdu._output_checksum = checksum
hdu._writeto(f)
f.close()
def _readfrom(cls, fileobj=None, data=None, mode='readonly',
memmap=False, save_backup=False, **kwargs):
"""
Provides the implementations from HDUList.fromfile and
HDUList.fromstring, both of which wrap this method, as their
implementations are largely the same.
"""
if fileobj is not None:
# instantiate a FITS file object (ffo)
ffo = _File(fileobj, mode=mode, memmap=memmap)
hdulist = cls(file=ffo)
else:
hdulist = cls()
# This method is currently only called from HDUList.fromstring and
# HDUList.fromfile. If fileobj is None then this must be the
# fromstring case; the data type of `data` will be checked in the
# _BaseHDU.fromstring call.
hdulist._save_backup = save_backup
saved_compression_enabled = compressed.COMPRESSION_ENABLED
try:
if ('disable_image_compression' in kwargs and
kwargs['disable_image_compression']):
compressed.COMPRESSION_ENABLED = False
if mode == 'ostream':
# Output stream--not interested in reading/parsing the
# HDUs--just writing to the output file
return hdulist
# read all HDUs
while True:
try:
if fileobj is not None:
try:
hdu = _BaseHDU.readfrom(ffo, **kwargs)
except EOFError:
break
except IOError, err:
if ffo.writeonly:
break
else:
raise
else:
if not data:
break
hdu = _BaseHDU.fromstring(data)
data = data[hdu._datLoc + hdu._datSpan:]
hdulist.append(hdu)
hdu._new = False
if 'checksum' in kwargs:
hdu._output_checksum = kwargs['checksum']
# check in the case there is extra space after the last HDU or
# corrupted HDU
except (VerifyError, ValueError), err:
warnings.warn(
'Error validating header for HDU #%d (note: PyFITS '
'uses zero-based indexing).\n%s\n'
'There may be extra bytes after the last HDU or the '
'file is corrupted.' %
(len(hdulist), indent(str(err))), VerifyWarning)
break
def _flush_resize(self):
"""
Implements flushing changes in update mode when parts of one or more HDU
need to be resized.
"""
old_name = self.__file.name
old_memmap = self.__file.memmap
name = _tmp_name(old_name)
if not self.__file.file_like:
old_mode = os.stat(old_name).st_mode
# The underlying file is an acutal file object. The HDUList is
# resized, so we need to write it to a tmp file, delete the
# original file, and rename the tmp file to the original file.
if self.__file.compression == 'gzip':
new_file = gzip.GzipFile(name, mode='ab+')
else:
new_file = name
hdulist = self.fromfile(new_file, mode='append')
for hdu in self:
hdu._writeto(hdulist.__file, inplace=True, copy=True)
if sys.platform.startswith('win'):
# Collect a list of open mmaps to the data; this well be used
# later. See below.
mmaps = [(idx, _get_array_mmap(hdu.data), hdu.data)
for idx, hdu in enumerate(self) if hdu._data_loaded]
hdulist.__file.close()
self.__file.close()
if sys.platform.startswith('win'):
# Close all open mmaps to the data. This is only necessary on
# Windows, which will not allow a file to be renamed or deleted
# until all handles to that file have been closed.
for idx, mmap, arr in mmaps:
if mmap is not None:
mmap.close()
os.remove(self.__file.name)
# reopen the renamed new file with "update" mode
os.rename(name, old_name)
os.chmod(old_name, old_mode)
if isinstance(new_file, gzip.GzipFile):
old_file = gzip.GzipFile(old_name, mode='rb+')
else:
old_file = old_name
ffo = _File(old_file, mode='update', memmap=old_memmap)
self.__file = ffo
for hdu in self:
# Need to update the _file attribute and close any open mmaps
# on each HDU
if (hdu._data_loaded and
_get_array_mmap(hdu.data) is not None):
del hdu.data
hdu._file = ffo
if sys.platform.startswith('win'):
# On Windows, all the original data mmaps were closed above.
# However, it's possible that the user still has references to
# the old data which would no longer work (possibly even cause
# a segfault if they try to access it). This replaces the
# buffers used by the original arrays with the buffers of mmap
# arrays created from the new file. This seems to work, but
# it's a flaming hack and carries no guarantees that it won't
# lead to odd behavior in practice. Better to just not keep
# references to data from files that had to be resized upon
# flushing (on Windows--again, this is no problem on Linux).
for idx, mmap, arr in mmaps:
if mmap is not None:
arr.data = self[idx].data.data
del mmaps # Just to be sure
else:
# The underlying file is not a file object, it is a file like
# object. We can't write out to a file, we must update the file
# like object in place. To do this, we write out to a temporary
# file, then delete the contents in our file like object, then
# write the contents of the temporary file to the now empty file
# like object.
self.writeto(name)
hdulist = self.fromfile(name)
ffo = self.__file
ffo.truncate(0)
ffo.seek(0)
for hdu in hdulist:
hdu._writeto(ffo, inplace=True, copy=True)
# Close the temporary file and delete it.
hdulist.close()
os.remove(hdulist.__file.name)
# reset the resize attributes after updating
self._resize = False
self._truncate = False
for hdu in self:
hdu._header._modified = False
hdu._new = False
hdu._file = ffo
def __init__(self, name, header):
"""
Construct a `StreamingHDU` object given a file name and a header.
Parameters
----------
name : file path, file object, or file like object
The file to which the header and data will be streamed.
If opened, the file object must be opened for append
(ab+).
header : `Header` instance
The header object associated with the data to be written
to the file.
Notes
-----
The file will be opened and the header appended to the end of
the file. If the file does not already exist, it will be
created, and if the header represents a Primary header, it
will be written to the beginning of the file. If the file
does not exist and the provided header is not a Primary
header, a default Primary HDU will be inserted at the
beginning of the file and the provided header will be added as
the first extension. If the file does already exist, but the
provided header represents a Primary header, the header will
be modified to an image extension header and appended to the
end of the file.
"""
if isinstance(name, gzip.GzipFile):
raise TypeError('StreamingHDU not supported for GzipFile objects.')
self._header = header.copy()
# handle a file object instead of a file name
filename = fileobj_name(name) or ''
#
# Check if the file already exists. If it does not, check to see
# if we were provided with a Primary Header. If not we will need
# to prepend a default PrimaryHDU to the file before writing the
# given header.
#
newfile = False
if filename:
if not os.path.exists(filename) or os.path.getsize(filename) == 0:
newfile = True
elif (hasattr(name, 'len') and name.len == 0):
newfile = True
if newfile:
if 'SIMPLE' not in self._header:
hdulist = HDUList([PrimaryHDU()])
hdulist.writeto(name, 'exception')
else:
#
# This will not be the first extension in the file so we
# must change the Primary header provided into an image
# extension header.
#
if 'SIMPLE' in self._header:
self._header.set('XTENSION', 'IMAGE', 'Image extension',
after='SIMPLE')
del self._header['SIMPLE']
if 'PCOUNT' not in self._header:
dim = self._header['NAXIS']
if dim == 0:
dim = ''
else:
dim = str(dim)
self._header.set('PCOUNT', 0, 'number of parameters',
after='NAXIS' + dim)
if 'GCOUNT' not in self._header:
self._header.set('GCOUNT', 1, 'number of groups',
after='PCOUNT')
self._ffo = _File(name, 'append')
# TODO : Fix this once the HDU writing API is cleaned up
tmp_hdu = _BaseHDU()
# Passing self._header as an argument to _BaseHDU() will cause its
# values to be modified in undesired ways...need to have a better way
# of doing this
tmp_hdu._header = self._header
self._header_offset = tmp_hdu._writeheader(self._ffo)[0]
self._data_offset = self._ffo.tell()
self._size = self.size
if self._size != 0:
self.writecomplete = False
else:
self.writecomplete = True
