def _makeTableHDU(data):
# Syntax used with PyFITS 0.4.2:
#_format = "s24,s14,s6,f32,f32,f32"
#recData = pyfits.record.record(data, _format)
# Do this when the Table classes are updated
#hdu = pyfits.Table(recData, name = "ASN")
col1 = pyfits.Column(name='MEMNAME',
format='24a',
array=chararray.array(data['name']))
col2 = pyfits.Column(name='MEMTYPE',
format='14a',
array=chararray.array(data['mtype']))
col3 = pyfits.Column(name='MEMPRSNT',
format='1b',
array=numpy.array(data['mprsnt']).astype(numpy.uint8))
col4 = pyfits.Column(name='XOFFSET',
format='1r',
unit='arcsec',
array=numpy.array(data['xsh']))
col5 = pyfits.Column(name='YOFFSET',
format='1r',
unit='arcsec',
array=numpy.array(data['ysh']))
col6 = pyfits.Column(name='ROTATION',
format='1r',
unit='degrees',
array=numpy.array(data['rot']))
hdu = pyfits.new_table([col1, col2, col3, col4, col5, col6],
nrows=len(data['name']))
return hdu
def _convert_to_valid_data_type(self, array, format):
# Convert the format to a type we understand
if isinstance(array, Delayed):
return array
elif array is None:
return array
else:
if 'A' in format and 'P' not in format:
if array.dtype.char in 'SU':
fsize = int(_convert_format(format)[1:])
return chararray.array(array, itemsize=fsize)
else:
numpy_format = _convert_format(format)
return _convert_array(array, np.dtype(numpy_format))
elif 'L' in format:
# boolean needs to be scaled back to storage values ('T', 'F')
if array.dtype == np.dtype('bool'):
return np.where(array == False, ord('F'), ord('T'))
else:
return np.where(array == 0, ord('F'), ord('T'))
elif 'X' not in format and 'P' not in format:
(repeat, fmt, option) = _parse_tformat(format)
# Preserve byte order of the original array for now; see #77
numpy_format = array.dtype.byteorder + _convert_format(fmt)
return _convert_array(array, np.dtype(numpy_format))
elif 'X' in format:
return _convert_array(array, np.dtype('uint8'))
else:
return array
def _makep(input, desp_output, format, nrows=None):
"""
Construct the P format column array, both the data descriptors and
the data. It returns the output "data" array of data type `dtype`.
The descriptor location will have a zero offset for all columns
after this call. The final offset will be calculated when the file
is written.
Parameters
----------
input
input object array
desp_output
output "descriptor" array of data type ``Int32``--must be nrows wide in
its first dimension
format
the _FormatP object reperesenting the format of the variable array
nrows : int, optional
number of rows to create in the column; defaults to the number of rows
in the input array
"""
_offset = 0
if not nrows:
nrows = len(input)
n = min(len(input), nrows)
data_output = _VLF([None] * nrows, dtype=format.dtype)
if format.dtype == 'a':
_nbytes = 1
else:
_nbytes = np.array([], dtype=np.typeDict[format.dtype]).itemsize
for idx in range(nrows):
if idx < len(input):
rowval = input[idx]
else:
if format.dtype == 'a':
rowval = ' ' * data_output.max
else:
rowval = [0] * data_output.max
if format.dtype == 'a':
data_output[idx] = chararray.array(encode_ascii(rowval),
itemsize=1)
else:
data_output[idx] = np.array(rowval, dtype=format.dtype)
desp_output[idx, 0] = len(data_output[idx])
desp_output[idx, 1] = _offset
_offset += len(data_output[idx]) * _nbytes
return data_output
def _makep(input, desp_output, format, nrows=None):
"""
Construct the P format column array, both the data descriptors and
the data. It returns the output "data" array of data type `dtype`.
The descriptor location will have a zero offset for all columns
after this call. The final offset will be calculated when the file
is written.
Parameters
----------
input
input object array
desp_output
output "descriptor" array of data type ``Int32``--must be nrows wide in
its first dimension
format
the _FormatP object reperesenting the format of the variable array
nrows : int, optional
number of rows to create in the column; defaults to the number of rows
in the input array
"""
_offset = 0
if not nrows:
nrows = len(input)
n = min(len(input), nrows)
data_output = _VLF([None] * nrows, dtype=format.dtype)
if format.dtype == 'a':
_nbytes = 1
else:
_nbytes = np.array([], dtype=format.dtype).itemsize
for idx in range(nrows):
if idx < len(input):
rowval = input[idx]
else:
if format.dtype == 'a':
rowval = ' ' * data_output.max
else:
rowval = [0] * data_output.max
if format.dtype == 'a':
data_output[idx] = chararray.array(encode_ascii(rowval),
itemsize=1)
else:
data_output[idx] = np.array(rowval, dtype=format.dtype)
desp_output[idx, 0] = len(data_output[idx])
desp_output[idx, 1] = _offset
_offset += len(data_output[idx]) * _nbytes
return data_output
def newalloc(self,name,like,type=None):
'''new uninitialized field based on characteristics of like field'''
if isinstance(like,basestring):
like = self._data[like]
try:
value = like.new(type=type)
except AttributeError:
# assume char-array
value = nastr.array(' ',itemsize=like.itemsize(),shape=like.shape)
self[name] = value
self._clone_field_hook(src=like,dst=name)
return value
def newalloc(self,name,like,type=None):
'''new uninitialized field based on characteristics of like field'''
if isinstance(like,basestring):
like = self._data[like]
try:
value = like.new(type=type)
except AttributeError:
# assume char-array
value = nastr.array(' ',itemsize=like.itemsize(),shape=like.shape)
self[name] = value
self._clone_field_hook(src=like,dst=name)
return value
def get_process_control_status(self, channel=None):
key = 'PCS'
cmd = self._build_command(key, channel)
r = self.ask(cmd)
r = self._parse_response(r)
if channel is None:
if r is None:
# from numpy import random,char
r = random.randint(0, 2, 6)
r = ','.join(char.array(r))
r = r.split(',')
return r
def __setitem__(self, key, value):
"""
To make sure the new item has consistent data type to avoid
misalignment.
"""
if isinstance(value, np.ndarray) and value.dtype == self.dtype:
pass
elif isinstance(value, chararray.chararray) and value.itemsize == 1:
pass
elif self.element_dtype == 'a':
value = chararray.array(value, itemsize=1)
else:
value = np.array(value, dtype=self.element_dtype)
np.ndarray.__setitem__(self, key, value)
self.max = max(self.max, len(value))
def __setitem__(self, key, value):
"""
To make sure the new item has consistent data type to avoid
misalignment.
"""
if isinstance(value, np.ndarray) and value.dtype == self.dtype:
pass
elif isinstance(value, chararray.chararray) and value.itemsize == 1:
pass
elif self.element_dtype == 'a':
value = chararray.array(value, itemsize=1)
else:
value = np.array(value, dtype=self.element_dtype)
np.ndarray.__setitem__(self, key, value)
self.max = max(self.max, len(value))
def get_process_control_status(self, channel=None):
key = 'PCS'
cmd = self._build_command(key, channel)
r = self.ask(cmd)
r = self._parse_response(r)
if channel is None:
if r is None:
# from numpy import random,char
r = random.randint(0, 2, 6)
r = ','.join(char.array(r))
r = r.split(',')
return r
def convert_data(self):
info('converting hv text files to hdf5 ...')
self.PBar.start(len(glob(join(self.DataDir, '*', '*.log'))))
f = h5py.File(join(self.DataDir, 'data.hdf5'), 'w')
for d in glob(join(self.DataDir, '*_*')):
arrays = []
for file_name in glob(join(d, '*.log')):
if getsize(file_name) == 0:
remove_file(file_name)
log_date = self.get_log_date(file_name)
data = genfromtxt(file_name, usecols=arange(3), dtype=[('timestamps', object), ('voltages', 'f2'), ('currents', 'f4')])
data = data[where(invert(isnan(data['voltages'])))[0]] # remove text entries
date_times = array(log_date.strftime('%Y-%m-%d ') + char.array(data['timestamps'].astype('U'))).astype(datetime64).astype(datetime)
data['timestamps'] = (array([time_stamp(dt, log_date.utcoffset().seconds) for dt in date_times]).astype('u4'))
data = data.astype([('timestamps', 'u4'), ('voltages', 'f2'), ('currents', 'f4')])
arrays.append(data)
self.PBar.update()
if len(arrays):
f.create_dataset(basename(d), data=concatenate(arrays))
def convert_data(self):
info('converting hv text files to hdf5 ...')
self.PBar.start(len(glob(join(self.DataDir, '*', '*.log'))))
f = h5py.File(join(self.DataDir, 'data.hdf5'), 'w')
for d in glob(join(self.DataDir, '*_*')):
arrays = []
for file_name in sorted(glob(join(d, '*.log'))):
if getsize(file_name) == 0:
remove_file(file_name)
self.PBar.update()
continue
log_date = self.get_log_date(file_name)
data = genfromtxt(file_name, usecols=arange(3), dtype=[('timestamps', 'U10'), ('voltages', 'f2'), ('currents', 'f4')], invalid_raise=False)
data = data[invert(isnan(data['voltages']))] # remove text entries
data['timestamps'] = array(log_date.strftime('%Y-%m-%d ') + char.array(data['timestamps'])).astype(datetime64) - datetime64('1970-01-01T00:00:00') - log_date.utcoffset().seconds
data = data.astype([('timestamps', 'u4'), ('voltages', 'f2'), ('currents', 'f4')])
arrays.append(data)
self.PBar.update()
if len(arrays):
f.create_dataset(basename(d), data=concatenate(arrays))
def __new__(cls, input, dtype="a"):
"""
Parameters
----------
input
a sequence of variable-sized elements.
"""
if dtype == "a":
try:
# this handles ['abc'] and [['a','b','c']]
# equally, beautiful!
input = map(lambda x: chararray.array(x, itemsize=1), input)
except:
raise ValueError("Inconsistent input data array: %s" % input)
a = np.array(input, dtype=np.object)
self = np.ndarray.__new__(cls, shape=(len(input)), buffer=a, dtype=np.object)
self.max = 0
self.element_dtype = dtype
return self
def _convert_to_valid_data_type(self, array, format):
# Convert the format to a type we understand
if isinstance(array, Delayed):
return array
elif array is None:
return array
else:
if 'A' in format and 'P' not in format:
if array.dtype.char in 'SU':
fsize = int(_convert_format(format)[1:])
return chararray.array(array, itemsize=fsize)
else:
numpy_format = _convert_format(format)
return _convert_array(array, np.dtype(numpy_format))
elif 'X' not in format and 'P' not in format:
(repeat, fmt, option) = _parse_tformat(format)
# Preserve byte order of the original array for now; see #77
numpy_format = array.dtype.byteorder + _convert_format(fmt)
return _convert_array(array, np.dtype(numpy_format))
elif 'X' in format:
return _convert_array(array, np.dtype('uint8'))
else:
return array
def _convert_to_valid_data_type(self, array):
# Convert the format to a type we understand
if isinstance(array, Delayed):
return array
elif array is None:
return array
else:
format = self.format
dims = self._dims
if 'A' in format and 'P' not in format:
if array.dtype.char in 'SU':
if dims:
# The 'last' dimension (first in the order given
# in the TDIMn keyword itself) is the number of
# characters in each string
fsize = dims[-1]
else:
fsize = int(_convert_format(format)[1:])
return chararray.array(array, itemsize=fsize)
else:
numpy_format = _convert_format(format)
return _convert_array(array, np.dtype(numpy_format))
elif 'L' in format:
# boolean needs to be scaled back to storage values ('T', 'F')
if array.dtype == np.dtype('bool'):
return np.where(array == False, ord('F'), ord('T'))
else:
return np.where(array == 0, ord('F'), ord('T'))
elif 'X' not in format and 'P' not in format:
(repeat, fmt, option) = _parse_tformat(format)
# Preserve byte order of the original array for now; see #77
numpy_format = array.dtype.byteorder + _convert_format(fmt)
return _convert_array(array, np.dtype(numpy_format))
elif 'X' in format:
return _convert_array(array, np.dtype('uint8'))
else:
return array
def _convert_to_valid_data_type(self, array):
# Convert the format to a type we understand
if isinstance(array, Delayed):
return array
elif array is None:
return array
else:
format = self.format
dims = self._dims
if 'A' in format and 'P' not in format:
if array.dtype.char in 'SU':
if dims:
# The 'last' dimension (first in the order given
# in the TDIMn keyword itself) is the number of
# characters in each string
fsize = dims[-1]
else:
fsize = int(_convert_format(format)[1:])
return chararray.array(array, itemsize=fsize)
else:
numpy_format = _convert_format(format)
return _convert_array(array, np.dtype(numpy_format))
elif 'L' in format:
# boolean needs to be scaled back to storage values ('T', 'F')
if array.dtype == np.dtype('bool'):
return np.where(array == False, ord('F'), ord('T'))
else:
return np.where(array == 0, ord('F'), ord('T'))
elif 'X' not in format and 'P' not in format:
(repeat, fmt, option) = _parse_tformat(format)
# Preserve byte order of the original array for now; see #77
numpy_format = array.dtype.byteorder + _convert_format(fmt)
return _convert_array(array, np.dtype(numpy_format))
elif 'X' in format:
return _convert_array(array, np.dtype('uint8'))
else:
return array
def __new__(cls, input, dtype='a'):
"""
Parameters
----------
input
a sequence of variable-sized elements.
"""
if dtype == 'a':
try:
# this handles ['abc'] and [['a','b','c']]
# equally, beautiful!
input = map(lambda x: chararray.array(x, itemsize=1), input)
except:
raise ValueError('Inconsistent input data array: %s' % input)
a = np.array(input, dtype=np.object)
self = np.ndarray.__new__(cls,
shape=(len(input)),
buffer=a,
dtype=np.object)
self.max = 0
self.element_dtype = dtype
return self
def __init__(self, name=None, format=None, unit=None, null=None,
bscale=None, bzero=None, disp=None, start=None,
dim=None, array=None):
"""
Construct a `Column` by specifying attributes. All attributes
except `format` can be optional.
Parameters
----------
name : str, optional
column name, corresponding to ``TTYPE`` keyword
format : str, optional
column format, corresponding to ``TFORM`` keyword
unit : str, optional
column unit, corresponding to ``TUNIT`` keyword
null : str, optional
null value, corresponding to ``TNULL`` keyword
bscale : int-like, optional
bscale value, corresponding to ``TSCAL`` keyword
bzero : int-like, optional
bzero value, corresponding to ``TZERO`` keyword
disp : str, optional
display format, corresponding to ``TDISP`` keyword
start : int, optional
column starting position (ASCII table only), corresponding
to ``TBCOL`` keyword
dim : str, optional
column dimension corresponding to ``TDIM`` keyword
"""
# any of the input argument (except array) can be a Card or just
# a number/string
for attr in KEYWORD_ATTRIBUTES:
value = locals()[attr] # get the argument's value
if isinstance(value, Card):
setattr(self, attr, value.value)
else:
setattr(self, attr, value)
# if the column data is not ndarray, make it to be one, i.e.
# input arrays can be just list or tuple, not required to be ndarray
if format is not None:
# check format
if not isinstance(format, _ColumnFormat):
try:
# legit FITS format?
format = _ColumnFormat(format)
recformat = format.recformat
except ValueError:
try:
# legit recarray format?
recformat = format
format = _ColumnFormat.from_recformat(format)
except ValueError:
raise ValueError('Illegal format `%s`.' % format)
self.format = format
# Zero-length formats are legal in the FITS format, but since they
# are not supported by numpy we mark columns that use them as
# "phantom" columns, that are not considered when reading the data
# as a record array.
if self.format[0] == '0' or \
(self.format[-1] == '0' and self.format[-2].isalpha()):
self._phantom = True
else:
self._phantom = False
# does not include Object array because there is no guarantee
# the elements in the object array are consistent.
if not isinstance(array,
(np.ndarray, chararray.chararray, Delayed)):
try: # try to convert to a ndarray first
if array is not None:
array = np.array(array)
except:
try: # then try to conver it to a strings array
itemsize = int(recformat[1:])
array = chararray.array(array, itemsize=itemsize)
except ValueError:
# then try variable length array
if isinstance(recformat, _FormatP):
array = _VLF(array, dtype=recformat.dtype)
else:
raise ValueError('Data is inconsistent with the '
'format `%s`.' % format)
else:
raise ValueError('Must specify format to construct Column.')
# scale the array back to storage values if there is bscale/bzero
if isinstance(array, np.ndarray):
# make a copy if scaled, so as not to corrupt the original array
if bzero not in ['', None, 0] or bscale not in ['', None, 1]:
if bzero not in ['', None, 0]:
array = array - bzero
if bscale not in ['', None, 1]:
array = array / bscale
array = self._convert_to_valid_data_type(array, self.format)
self.array = array
def __init__(self,
name=None,
format=None,
unit=None,
null=None,
bscale=None,
bzero=None,
disp=None,
start=None,
dim=None,
array=None):
"""
Construct a `Column` by specifying attributes. All attributes
except `format` can be optional.
Parameters
----------
name : str, optional
column name, corresponding to ``TTYPE`` keyword
format : str, optional
column format, corresponding to ``TFORM`` keyword
unit : str, optional
column unit, corresponding to ``TUNIT`` keyword
null : str, optional
null value, corresponding to ``TNULL`` keyword
bscale : int-like, optional
bscale value, corresponding to ``TSCAL`` keyword
bzero : int-like, optional
bzero value, corresponding to ``TZERO`` keyword
disp : str, optional
display format, corresponding to ``TDISP`` keyword
start : int, optional
column starting position (ASCII table only), corresponding
to ``TBCOL`` keyword
dim : str, optional
column dimension corresponding to ``TDIM`` keyword
"""
if format is None:
raise ValueError('Must specify format to construct Column.')
# any of the input argument (except array) can be a Card or just
# a number/string
for attr in KEYWORD_ATTRIBUTES:
value = locals()[attr] # get the argument's value
if isinstance(value, Card):
setattr(self, attr, value.value)
else:
setattr(self, attr, value)
# if the column data is not ndarray, make it to be one, i.e.
# input arrays can be just list or tuple, not required to be ndarray
# check format
if not isinstance(format, _ColumnFormat):
try:
# legit FITS format?
format = _ColumnFormat(format)
recformat = format.recformat
except ValueError:
try:
# legit recarray format?
recformat = format
format = _ColumnFormat.from_recformat(format)
except ValueError:
raise ValueError('Illegal format `%s`.' % format)
self.format = format
# Zero-length formats are legal in the FITS format, but since they
# are not supported by numpy we mark columns that use them as
# "phantom" columns, that are not considered when reading the data
# as a record array.
if self.format[0] == '0' or \
(self.format[-1] == '0' and self.format[-2].isalpha()):
self._phantom = True
else:
self._phantom = False
if isinstance(dim, basestring):
self._dims = _parse_tdim(dim)
elif isinstance(dim, tuple):
self._dims = dim
elif not dim:
self._dims = tuple()
else:
raise TypeError(
"`dim` argument must be a string containing a valid value "
"for the TDIMn header keyword associated with this column, "
"or a tuple containing the C-order dimensions for the column")
if self._dims:
repeat = _parse_tformat(format)[0]
if reduce(operator.mul, self._dims) > repeat:
raise ValueError(
"The repeat count of the column format %r for column %r "
"is fewer than the number of elements per the TDIM "
"argument %r." % (name, format, dim))
# does not include Object array because there is no guarantee
# the elements in the object array are consistent.
if not isinstance(array, (np.ndarray, chararray.chararray, Delayed)):
try: # try to convert to a ndarray first
if array is not None:
array = np.array(array)
except:
try: # then try to convert it to a strings array
itemsize = int(recformat[1:])
array = chararray.array(array, itemsize=itemsize)
except ValueError:
# then try variable length array
if isinstance(recformat, _FormatP):
array = _VLF(array, dtype=recformat.dtype)
else:
raise ValueError('Data is inconsistent with the '
'format `%s`.' % format)
# scale the array back to storage values if there is bscale/bzero
if isinstance(array, np.ndarray):
# make a copy if scaled, so as not to corrupt the original array
if bzero not in ['', None, 0] or bscale not in ['', None, 1]:
if bzero not in ['', None, 0]:
array = array - bzero
if bscale not in ['', None, 1]:
array = array / bscale
array = self._convert_to_valid_data_type(array)
self.array = array