def testFormatsStructure(self):
"""Check the structure of the formats list.
"""
common.verbosePrint('\nTesting formats structure')
for f in nriterators.flattenFormats(self.formats, check=True):
self.failIfEqual(f, None)
def testFormatsStructure(self):
"""Check the structure of the formats list.
"""
common.verbosePrint('\nTesting formats structure')
for f in nriterators.flattenFormats(self.formats, check=True):
self.failIfEqual(f, None)
def _checkFormats(formats):
"""
Check the format of the `formats` list.
For an explanation of argument meanings see the `array()` function.
"""
# Formats description must be a list or a tuple
if not (isinstance(formats, list) or isinstance(formats, tuple)):
raise TypeError("""``formats`` argument must be a list or a tuple""")
# Formats elements must be strings or sequences
for item in nriterators.flattenFormats(formats, check=True):
if item is None:
raise TypeError("""elements of the ``formats`` list must """ """be strings or sequences""")
def _checkFormats(formats):
"""
Check the format of the `formats` list.
For an explanation of argument meanings see the `array()` function.
"""
# Formats description must be a list or a tuple
if not (isinstance(formats, list) or isinstance(formats, tuple)):
raise TypeError("""``formats`` argument must be a list or a tuple""")
# Formats elements must be strings or sequences
for item in nriterators.flattenFormats(formats, check=True):
if item is None:
raise TypeError("""elements of the ``formats`` list must """
"""be strings or sequences""")
def fromarrays(arrayList,
formats=None,
names=None,
shape=0,
byteorder=sys.byteorder,
aligned=0,
descr=None):
"""
Create a new instance of a `NestedRecArray` from `arrayList` arrays.
This function can be used to build a new array of nested records
from a list of arrays, one for each field. The new array is
returned as a result.
The function works much like ``numarray.records.fromarrays()``, but
`arrayList` may also contain nested fields, i.e. sequences of other
arrays (nested or not). All non-nested arrays appearing in
`arrayList` must have the same length.
The rest of arguments work as explained in `array()`.
Example
=======
Let us build the sample array used in the examples of `array()`. In
the old way:
>>> nra = fromarrays(
... [[1, 2], [(0.5, 1.0), (0, 0)], [['a1', 'a2'], [1j, 1+.1j]]],
... names=['id', 'pos', ('info', ['name', 'value'])],
... formats=['Int64', '(2,)Float32', ['a2', 'Complex64']])
In the new way:
>>> nra = fromarrays(
... [[1, 2], [(0.5, 1.0), (0, 0)], [['a1', 'a2'], [1j, 1+.1j]]],
... descr=[('id', 'Int64'), ('pos', '(2,)Float32'),
... ('info', [('name', 'a2'), ('value', 'Complex64')])])
Note how `formats` and `descr` mimic the structure of the whole
`arrayList`.
"""
_checkArrayList(arrayList)
# Check if a buffer structure is given. It must be given in order to
# disambiguate possible ambiguities
_isThereStructure(formats, descr, arrayList)
# Check if buffer structure is specified using descr OR formats (and,
# optionally, names)
_onlyOneSyntax(descr, formats, names)
# This is to keep compatibility with numarray.records.array function
if isinstance(formats, str):
formats = formats.split(',')
if isinstance(names, str):
names = names.split(',')
# Check the descr format
# Check for '/' in descr
if descr is None:
descr = makeDescr(formats, names)
_checkDescr(descr)
_checkFieldsInDescr(descr)
# Check the formats format
if formats is None:
formats = makeFormats(descr)
_checkFormats(formats)
# Check the names format
if names is None:
names = makeNames(descr)
_checkNames(names)
# Flatten the structure descriptors
flatFormats = [item for item in nriterators.flattenFormats(formats)]
flatNames = [item for item in nriterators.flattenNames(names)]
# Create a regular recarray from the arrays list
flatArrayList = []
nriterators.flattenArraysList(arrayList, descr, flatArrayList)
ra = numarray.records.fromarrays(flatArrayList,
formats=flatFormats,
names=flatNames,
shape=shape,
byteorder=byteorder,
aligned=aligned)
# Create the nested recarray
nra = NestedRecArray(ra, descr)
return nra
def array(buffer=None,
formats=None,
shape=0,
names=None,
byteorder=sys.byteorder,
aligned=0,
descr=None):
"""
Create a new instance of a `NestedRecArray`.
This function can be used to build a new array of nested records.
The new array is returned as a result.
The function works much like ``numarray.records.array()``, with some
differences:
1. In addition to flat buffers and regular sequences of non-nested
elements, the `buffer` argument can take regular sequences where
each element has a structure nested to an arbitrary depth. Of
course, all elements in a non-flat buffer must have the same
format.
2. The `formats` argument only supports sequences of strings and
other sequences. Each string defines the shape and type of a
non-nested field. Each sequence contains the formats of the
sub-fields of a nested field.
The structure of this argument must match that of the elements in
`buffer`. This argument may have a recursive structure.
3. The `names` argument only supports lists of strings and 2-tuples.
Each string defines the name of a non-nested field. Each 2-tuple
contains the name of a nested field and a list describing the
names of its sub-fields.
The structure of this argument must match that of the elements in
`buffer`. This argument may have a recursive structure.
The `descr` argument is a new-style description of the structure of
the `buffer`. It is intended to replace the `formats` and `names`
arguments, so they can not be used at the same time [#descr]_.
The `descr` argument is a list of 2-tuples, each of them describing
a field. The first value in a tuple is the *name* of the field,
while the second one is a description of its *structure*. If the
second value is a string, it defines the format (shape and type) of
a non-nested field. Else, it is a list of 2-tuples describing the
sub-fields of a nested field.
If `descr` is ``None`` (or omitted), the whole structure of the
array is tried to be inferred from that of the `buffer`, and
automatic names (``c1``, ``c2`` etc. on each nested field) are
assigned to all fields.
The `descr` argument may have a recursive structure.
Please note that names used in `names` or `descr` should *not*
contain the string ``'/'``, since it is used as the field/sub-field
separator by `NestedRecArray.asRecArray()`. If the separator is
found in a name, a ``ValueError`` is raised.
.. [#descr] The syntax of `descr` is based on that of the
``__array_descr__`` attribute in the proposed standard
`N-dimensional array interface`__.
__ http://numeric.scipy.org/array_interface.html
When to use `descr` or `formats`
================================
Since `descr` requires both the name and structure of fields to
always be specified, the `formats` argument comes more handy when
one does not want to explicitly specify names. However it is not
allowed to use the `names` argument without the `formats` one. This
is due to the fact that automatic inferrence of the `buffer`
structure is not implemented. When fully specifying names and
structure, the `descr` argument is preferred over `formats` and
`names` for the sake of code legibility and conciseness.
Examples
========
The following examples will help to clarify the words above. In
them, an array of two elements is created. Each element has three
fields: a 64-bit integer (``id``), a bi-dimensional 32-bit floating
point (``pos``) and a nested field (``info``); the nested field has
two sub-fields: a two-character string (``name``) and a 64-bit
complex (``value``).
Example 1
---------
In this example the array is created by specifying both its contents
and its structure, so the structure of the used arguments must be
coherent.
This is how the array would be created in the old-style way,
i.e. using the `formats` and `names` arguments:
>>> nra = array(
... [(1, (0.5, 1.0), ('a1', 1j)), (2, (0, 0), ('a2', 1+.1j))],
... names=['id', 'pos', ('info', ['name', 'value'])],
... formats=['Int64', '(2,)Float32', ['a2', 'Complex64']])
And this is how the array would be created in the new-style way,
i.e. using the `descr` argument:
>>> nra = array(
... [(1, (0.5, 1.0), ('a1', 1j)), (2, (0, 0), ('a2', 1+.1j))],
... descr=[('id', 'Int64'), ('pos', '(2,)Float32'),
... ('info', [('name', 'a2'), ('value', 'Complex64')])])
Note how `formats` and `descr` mimic the structure of each element
in `buffer`.
Example 2
---------
Now the array is created from a flat string representing the data in
memory. Names will be automatically assigned. For that to work,
the resulting array shape and record format must be fully specified.
>>> datastring = binary_representation_of_data
>>> nra = array(
... datastring, shape=2,
... formats=['Int64', '(2,)Float32', ['a2', 'Complex64']])
Byte ordering and alignment is assumed to be that of the host
machine, since it has not been explicitly stated via the `byteorder`
and `aligned` arguments.
"""
# Check if buffer structure is specified using descr OR formats (and,
# optionally, names)
_onlyOneSyntax(descr, formats, names)
# Create or check the descr format
if descr is None:
if formats is not None:
descr = makeDescr(formats, names)
# `buffer` can still be some object which describes its own structure,
# so `descr`/`formats` are not yet required.
else:
_checkDescr(descr)
_checkFieldsInDescr(descr)
# After this, if it exists a formats, it will always exists a descr
# This is to keep compatibility with numarray.records.array function
if isinstance(formats, str):
formats = formats.split(',')
if isinstance(names, str):
names = names.split(',')
# First, check for easily convertible objects (NRA, NA and NumPy
# objects)
# F. Alted 2006-01-20
if isinstance(buffer, NestedRecArray):
buffer = buffer.copy() # Always return a copy of the data
# Return as soon as possible is not descr, formats or names specified
if (descr is None and formats is None and names is None):
return buffer
# Check that descriptions are consistent
if descr is not None:
fmts = [
item for item in nriterators.flattenFormats(makeFormats(descr))
]
# Check just the formats, not the names
if _matchFormats(fmts, buffer._formats):
raise ValueError, \
"""buffer structure doesn't match that provided by the format
specification."""
# New description is compatible. Assign it to the NRA
buffer.descr = descr
# Also the names (they may have been changed in new description)
if names is None:
names = makeNames(descr)
# Assignements in both the NRA and flatArray are necessary
buffer._names = [i for i in nriterators.flattenNames(names)]
buffer._flatArray._names = [
i for i in nriterators.flattenNames(names)
]
return buffer
if isinstance(buffer, numarray.records.RecArray):
buffer = buffer.copy() # Always return a copy of the data
# Return as soon as possible is not descr, formats or names specified
if (descr is None and formats is None and names is None):
descr = makeDescr(buffer._formats, buffer._names)
# Return the nested recarray
return NestedRecArray(buffer, descr)
# Check that descriptions are consistent
if formats is not None and _matchFormats(formats, buffer._formats):
raise ValueError, \
"""buffer structure doesn't match that provided by the format
specification."""
if names is not None:
buffer._names = names
# Also, the names may have been changed in new description
elif descr is not None:
buffer._names = makeNames(descr)
if descr is None:
descr = makeDescr(buffer._formats, buffer._names)
# Refresh the cache of fields (just in case the names has changed)
buffer._fields = buffer._get_fields()
# Return the nested recarray
return NestedRecArray(buffer, descr)
# Check for numpy ndarrays, recarrays, records or scalar records
if numpy_imported and isinstance(buffer, (numpy.ndarray, numpy.void)):
buffer = buffer.copy() # Always return a copy of the data
# Try to convert into a nestedrecarray
try:
nra = fromnumpy(buffer)
except Exception, exc: #XXX
raise ValueError, \
"""buffer parameter of type numpy cannot be converted into a NestedRecArray
object. The error was: <%s>""" % (exc,)
# Check that descriptions are consistent
if descr is not None:
fmt1 = [
i for i in nriterators.flattenFormats(makeFormats(nra.descr))
]
fmt2 = [i for i in nriterators.flattenFormats(makeFormats(descr))]
if _matchFormats(fmt1, fmt2):
raise ValueError, \
"""buffer structure doesn't match that provided by the format specification."""
return nra
# Check if a buffer structure is given. It must be given in order to
# disambiguate possible ambiguities
_isThereStructure(formats, descr, buffer)
# Check the formats format
if formats is None:
formats = makeFormats(descr)
_checkFormats(formats)
# Check the names format
if names is None:
names = makeNames(descr)
_checkNames(names)
# Flatten the structure descriptors
flatFormats = [item for item in nriterators.flattenFormats(formats)]
flatNames = [item for item in nriterators.flattenNames(names)]
# Check the buffer structure (order matters!)
if (isinstance(buffer, types.ListType)
or isinstance(buffer, types.TupleType)):
if (isinstance(buffer[0], numarray.NumArray)
or isinstance(buffer[0], numarray.strings.CharArray)):
return fromarrays(buffer,
formats=formats,
shape=shape,
names=names,
byteorder=byteorder,
aligned=aligned)
elif buffer:
_checkBufferStructure(descr, buffer)
def fromarrays(arrayList, formats=None, names=None, shape=0, byteorder=sys.byteorder, aligned=0, descr=None):
"""
Create a new instance of a `NestedRecArray` from `arrayList` arrays.
This function can be used to build a new array of nested records
from a list of arrays, one for each field. The new array is
returned as a result.
The function works much like ``numarray.records.fromarrays()``, but
`arrayList` may also contain nested fields, i.e. sequences of other
arrays (nested or not). All non-nested arrays appearing in
`arrayList` must have the same length.
The rest of arguments work as explained in `array()`.
Example
=======
Let us build the sample array used in the examples of `array()`. In
the old way:
>>> nra = fromarrays(
... [[1, 2], [(0.5, 1.0), (0, 0)], [['a1', 'a2'], [1j, 1+.1j]]],
... names=['id', 'pos', ('info', ['name', 'value'])],
... formats=['Int64', '(2,)Float32', ['a2', 'Complex64']])
In the new way:
>>> nra = fromarrays(
... [[1, 2], [(0.5, 1.0), (0, 0)], [['a1', 'a2'], [1j, 1+.1j]]],
... descr=[('id', 'Int64'), ('pos', '(2,)Float32'),
... ('info', [('name', 'a2'), ('value', 'Complex64')])])
Note how `formats` and `descr` mimic the structure of the whole
`arrayList`.
"""
_checkArrayList(arrayList)
# Check if a buffer structure is given. It must be given in order to
# disambiguate possible ambiguities
_isThereStructure(formats, descr, arrayList)
# Check if buffer structure is specified using descr OR formats (and,
# optionally, names)
_onlyOneSyntax(descr, formats, names)
# This is to keep compatibility with numarray.records.array function
if isinstance(formats, str):
formats = formats.split(",")
if isinstance(names, str):
names = names.split(",")
# Check the descr format
# Check for '/' in descr
if descr is None:
descr = makeDescr(formats, names)
_checkDescr(descr)
_checkFieldsInDescr(descr)
# Check the formats format
if formats is None:
formats = makeFormats(descr)
_checkFormats(formats)
# Check the names format
if names is None:
names = makeNames(descr)
_checkNames(names)
# Flatten the structure descriptors
flatFormats = [item for item in nriterators.flattenFormats(formats)]
flatNames = [item for item in nriterators.flattenNames(names)]
# Create a regular recarray from the arrays list
flatArrayList = []
nriterators.flattenArraysList(arrayList, descr, flatArrayList)
ra = numarray.records.fromarrays(
flatArrayList, formats=flatFormats, names=flatNames, shape=shape, byteorder=byteorder, aligned=aligned
)
# Create the nested recarray
nra = NestedRecArray(ra, descr)
return nra
def array(buffer=None, formats=None, shape=0, names=None, byteorder=sys.byteorder, aligned=0, descr=None):
"""
Create a new instance of a `NestedRecArray`.
This function can be used to build a new array of nested records.
The new array is returned as a result.
The function works much like ``numarray.records.array()``, with some
differences:
1. In addition to flat buffers and regular sequences of non-nested
elements, the `buffer` argument can take regular sequences where
each element has a structure nested to an arbitrary depth. Of
course, all elements in a non-flat buffer must have the same
format.
2. The `formats` argument only supports sequences of strings and
other sequences. Each string defines the shape and type of a
non-nested field. Each sequence contains the formats of the
sub-fields of a nested field.
The structure of this argument must match that of the elements in
`buffer`. This argument may have a recursive structure.
3. The `names` argument only supports lists of strings and 2-tuples.
Each string defines the name of a non-nested field. Each 2-tuple
contains the name of a nested field and a list describing the
names of its sub-fields.
The structure of this argument must match that of the elements in
`buffer`. This argument may have a recursive structure.
The `descr` argument is a new-style description of the structure of
the `buffer`. It is intended to replace the `formats` and `names`
arguments, so they can not be used at the same time [#descr]_.
The `descr` argument is a list of 2-tuples, each of them describing
a field. The first value in a tuple is the *name* of the field,
while the second one is a description of its *structure*. If the
second value is a string, it defines the format (shape and type) of
a non-nested field. Else, it is a list of 2-tuples describing the
sub-fields of a nested field.
If `descr` is ``None`` (or omitted), the whole structure of the
array is tried to be inferred from that of the `buffer`, and
automatic names (``c1``, ``c2`` etc. on each nested field) are
assigned to all fields.
The `descr` argument may have a recursive structure.
Please note that names used in `names` or `descr` should *not*
contain the string ``'/'``, since it is used as the field/sub-field
separator by `NestedRecArray.asRecArray()`. If the separator is
found in a name, a ``ValueError`` is raised.
.. [#descr] The syntax of `descr` is based on that of the
``__array_descr__`` attribute in the proposed standard
`N-dimensional array interface`__.
__ http://numeric.scipy.org/array_interface.html
When to use `descr` or `formats`
================================
Since `descr` requires both the name and structure of fields to
always be specified, the `formats` argument comes more handy when
one does not want to explicitly specify names. However it is not
allowed to use the `names` argument without the `formats` one. This
is due to the fact that automatic inferrence of the `buffer`
structure is not implemented. When fully specifying names and
structure, the `descr` argument is preferred over `formats` and
`names` for the sake of code legibility and conciseness.
Examples
========
The following examples will help to clarify the words above. In
them, an array of two elements is created. Each element has three
fields: a 64-bit integer (``id``), a bi-dimensional 32-bit floating
point (``pos``) and a nested field (``info``); the nested field has
two sub-fields: a two-character string (``name``) and a 64-bit
complex (``value``).
Example 1
---------
In this example the array is created by specifying both its contents
and its structure, so the structure of the used arguments must be
coherent.
This is how the array would be created in the old-style way,
i.e. using the `formats` and `names` arguments:
>>> nra = array(
... [(1, (0.5, 1.0), ('a1', 1j)), (2, (0, 0), ('a2', 1+.1j))],
... names=['id', 'pos', ('info', ['name', 'value'])],
... formats=['Int64', '(2,)Float32', ['a2', 'Complex64']])
And this is how the array would be created in the new-style way,
i.e. using the `descr` argument:
>>> nra = array(
... [(1, (0.5, 1.0), ('a1', 1j)), (2, (0, 0), ('a2', 1+.1j))],
... descr=[('id', 'Int64'), ('pos', '(2,)Float32'),
... ('info', [('name', 'a2'), ('value', 'Complex64')])])
Note how `formats` and `descr` mimic the structure of each element
in `buffer`.
Example 2
---------
Now the array is created from a flat string representing the data in
memory. Names will be automatically assigned. For that to work,
the resulting array shape and record format must be fully specified.
>>> datastring = binary_representation_of_data
>>> nra = array(
... datastring, shape=2,
... formats=['Int64', '(2,)Float32', ['a2', 'Complex64']])
Byte ordering and alignment is assumed to be that of the host
machine, since it has not been explicitly stated via the `byteorder`
and `aligned` arguments.
"""
# Check if buffer structure is specified using descr OR formats (and,
# optionally, names)
_onlyOneSyntax(descr, formats, names)
# Create or check the descr format
if descr is None:
if formats is not None:
descr = makeDescr(formats, names)
# `buffer` can still be some object which describes its own structure,
# so `descr`/`formats` are not yet required.
else:
_checkDescr(descr)
_checkFieldsInDescr(descr)
# After this, if it exists a formats, it will always exists a descr
# This is to keep compatibility with numarray.records.array function
if isinstance(formats, str):
formats = formats.split(",")
if isinstance(names, str):
names = names.split(",")
# First, check for easily convertible objects (NRA, NA and NumPy
# objects)
# F. Alted 2006-01-20
if isinstance(buffer, NestedRecArray):
buffer = buffer.copy() # Always return a copy of the data
# Return as soon as possible is not descr, formats or names specified
if descr is None and formats is None and names is None:
return buffer
# Check that descriptions are consistent
if descr is not None:
fmts = [item for item in nriterators.flattenFormats(makeFormats(descr))]
# Check just the formats, not the names
if _matchFormats(fmts, buffer._formats):
raise ValueError("buffer structure doesn't match that " "provided by the format specification.")
# New description is compatible. Assign it to the NRA
buffer.descr = descr
# Also the names (they may have been changed in new description)
if names is None:
names = makeNames(descr)
# Assignements in both the NRA and flatArray are necessary
buffer._names = [i for i in nriterators.flattenNames(names)]
buffer._flatArray._names = [i for i in nriterators.flattenNames(names)]
return buffer
if isinstance(buffer, numarray.records.RecArray):
buffer = buffer.copy() # Always return a copy of the data
# Return as soon as possible is not descr, formats or names specified
if descr is None and formats is None and names is None:
descr = makeDescr(buffer._formats, buffer._names)
# Return the nested recarray
return NestedRecArray(buffer, descr)
# Check that descriptions are consistent
if formats is not None and _matchFormats(formats, buffer._formats):
raise ValueError("buffer structure doesn't match that provided " "by the format specification.")
if names is not None:
buffer._names = names
# Also, the names may have been changed in new description
elif descr is not None:
buffer._names = makeNames(descr)
if descr is None:
descr = makeDescr(buffer._formats, buffer._names)
# Refresh the cache of fields (just in case the names has changed)
buffer._fields = buffer._get_fields()
# Return the nested recarray
return NestedRecArray(buffer, descr)
# Check for numpy ndarrays, recarrays, records or scalar records
if numpy_imported and isinstance(buffer, (numpy.ndarray, numpy.void)):
buffer = buffer.copy() # Always return a copy of the data
# Try to convert into a nestedrecarray
try:
nra = fromnumpy(buffer)
except Exception, exc: # XXX
raise ValueError(
"buffer parameter of type numpy cannot be "
"converted into a NestedRecArray object. "
"The error was: <%s>" % (exc,)
)
# Check that descriptions are consistent
if descr is not None:
fmt1 = [i for i in nriterators.flattenFormats(makeFormats(nra.descr))]
fmt2 = [i for i in nriterators.flattenFormats(makeFormats(descr))]
if _matchFormats(fmt1, fmt2):
raise ValueError("buffer structure doesn't match that " "provided by the format specification.")
return nra
# Check if a buffer structure is given. It must be given in order to
# disambiguate possible ambiguities
_isThereStructure(formats, descr, buffer)
# Check the formats format
if formats is None:
formats = makeFormats(descr)
_checkFormats(formats)
# Check the names format
if names is None:
names = makeNames(descr)
_checkNames(names)
# Flatten the structure descriptors
flatFormats = [item for item in nriterators.flattenFormats(formats)]
flatNames = [item for item in nriterators.flattenNames(names)]
# Check the buffer structure (order matters!)
if isinstance(buffer, list) or isinstance(buffer, tuple):
if isinstance(buffer[0], numarray.NumArray) or isinstance(buffer[0], numarray.strings.CharArray):
return fromarrays(buffer, formats=formats, shape=shape, names=names, byteorder=byteorder, aligned=aligned)
elif buffer:
_checkBufferStructure(descr, buffer)
# Flatten the buffer (if any)
if buffer is None:
flatBuffer = None
else:
# Buffer is a list of sequences. Every sublist represents a row
# of the array
