def numpyB64(str64,
dtype=None,
shape_len_compression=None,
compression=None):
decoded_bytearray = b64decode_as_bytearray(str64, validate=True)
if isinstance(shape_len_compression, str):
compression = shape_len_compression
shape_len = None
else:
shape_len = shape_len_compression
if compression:
if compression == "blosc":
decoded_bytearray = blosc.decompress(decoded_bytearray,
as_bytearray=True)
else:
raise Exception(f"unknow {compression} compression")
if dtype in ("bool", bool):
numpy_uint8_containing_8bits = frombuffer(
decoded_bytearray, uint8) # pas de copie -> read only
numpy_uint8_containing_8bits = unpackbits(
numpy_uint8_containing_8bits
) # copie dans un numpy array de uint8 mutable
if shape_len is None:
shape_len = len(numpy_uint8_containing_8bits)
return ndarray(shape_len, dtype,
numpy_uint8_containing_8bits) # pas de recopie
else:
if isinstance(dtype, list):
dtype = [(str(champName), champType)
for champName, champType in dtype]
if shape_len is None:
array = frombuffer(decoded_bytearray, dtype) # pas de recopie
else:
array = ndarray(shape_len, dtype,
decoded_bytearray) # pas de recopie
if (
nb_bits == 32 and serialize_parameters.
numpyB64_convert_int64_to_int32_and_align_in_Python_32Bit
): # pour pouvoir deserialiser les classifiers en python 32 bit ?
if array.dtype in (int64, "int64"):
return array.astype(int32)
elif isinstance(dtype, list):
newTypes = []
for champ in dtype:
champName, champType = champ
if champName:
champType = numpy_dtype(champType)
if champType in (int64, "int64"):
newTypes.append((champName, int32))
else:
newTypes.append((champName, champType))
newDtype = numpy_dtype(newTypes, align=True)
newN = ndarray(len(array), newDtype)
for champName, champType in newTypes:
if champName:
newN[champName][:] = array[champName]
return newN
return array
def read_dzt_data(fileobject,
dtype,
samples_per_scan=None,
channels=None,
**kwargs):
"""
Parameters
----------
fileobject : fileobject
Object with `.read` method returning bytes.
dtype : int or str, optional
Number format for the data.
For ints assume:
8 = 'uint8'
16 : 'uint16'
32 : 'int32'
64 : 'int64'
samples_per_scan : int, optional
channels : int, optional
skip_initial : int, optional
Skip first `skip_initial` bytes.
Default is 0 bytes.
Returns
-------
data : list of ndarrays
Each channel in Fortran format (column oriented).
In case of failing to reshape, returns one numpy array in a list
error_message : str
"""
skip_initial = kwargs.get("skip_initial", None)
dtype_dict = {8: "uint8", 16: "uint16", 32: "int32", 64: "int64"}
if not isinstance(dtype, str):
dtype = dtype_dict[dtype]
dtype = numpy_dtype(dtype)
if skip_initial is not None:
fileobject.read(skip_initial)
# count = -1 :: read to end of the file
data_array = fromfile(fileobject, count=-1, dtype=dtype)
if (samples_per_scan is None) or (data_array.size % samples_per_scan != 0):
if samples_per_scan is None:
samples_per_scan = "None"
err_msg = "error in samples_per_scan : divmod({}, {}) = div:{}, mod:{}"
dsize = data_array.size
sps = samples_per_scan
div, mod = divmod(data_array.size, samples_per_scan)
return [data_array], err_msg.format(dsize, sps, div, mod)
N = samples_per_scan
D = data_array.size // samples_per_scan
data_array = data_array.reshape(N, D, order="F")
if not channels:
err_msg = "channel count is: {}"
return [data_array], err_msg.format(channels)
data = []
for channel in range(channels):
# Slice array with start:end:step
# start = (0, 1, ...),
# end = None
# step=number of channels
data.append(data_array[:, channel::channels])
return data, None
def _create_data(
self,
ncvar,
construct=None,
unpacked_dtype=False,
uncompress_override=None,
parent_ncvar=None,
):
"""TODO.
.. versionadded:: 3.0.0
:Parameters:
ncvar: `str`
The name of the netCDF variable that contains the data.
construct: optional
unpacked_dtype: `False` or `numpy.dtype`, optional
uncompress_override: `bool`, optional
:Returns:
`Data`
"""
g = self.read_vars
is_cfa_variable = (
g["cfa"]
and construct.get_property("cf_role", None) == "cfa_variable"
)
if not is_cfa_variable:
# --------------------------------------------------------
# Create data for a normal netCDF variable
# --------------------------------------------------------
return super()._create_data(
ncvar=ncvar,
construct=construct,
unpacked_dtype=unpacked_dtype,
uncompress_override=uncompress_override,
parent_ncvar=parent_ncvar,
)
# ------------------------------------------------------------
# Still here? Then create data for a CFA netCDF variable
# ------------------------------------------------------------
# print (' Creating data from CFA variable', repr(ncvar),
# repr(construct))
try:
cfa_data = json.loads(construct.get_property("cfa_array"))
except ValueError as error:
raise ValueError(
"Error during JSON-decoding of netCDF attribute 'cfa_array': "
"{}".format(error)
)
variable = g["variables"][ncvar]
cfa_data["file"] = g["filename"]
cfa_data["Units"] = construct.Units
cfa_data["fill_value"] = construct.fill_value()
cfa_data["_pmshape"] = cfa_data.pop("pmshape", ())
cfa_data["_pmaxes"] = cfa_data.pop("pmdimensions", ())
base = cfa_data.get("base", None)
if base is not None:
cfa_data["base"] = pathjoin(dirname(g["filename"]), base)
ncdimensions = construct.get_property("cfa_dimensions", "").split()
dtype = variable.dtype
if dtype is str:
# netCDF string types have a dtype of `str`, which needs
# to be reset as a numpy.dtype, but we don't know what
# without reading the data, so set it to None for now.
dtype = None
# UNICODE???? TODO
if self._is_char(ncvar) and dtype.kind in "SU" and ncdimensions:
strlen = g["nc"].dimensions[ncdimensions[-1]].size
if strlen > 1:
ncdimensions.pop()
dtype = numpy_dtype("S{0}".format(strlen))
# --- End: if
cfa_data["dtype"] = dtype
cfa_data["_axes"] = ncdimensions
cfa_data["shape"] = [
g["nc"].dimensions[ncdim].size for ncdim in ncdimensions
]
for attrs in cfa_data["Partitions"]:
# FORMAT
sformat = attrs.get("subarray", {}).pop("format", "netCDF")
if sformat is not None:
attrs["format"] = sformat
# DTYPE
dtype = attrs.get("subarray", {}).pop("dtype", None)
if dtype not in (None, "char"):
attrs["subarray"]["dtype"] = numpy_dtype(dtype)
# UNITS and CALENDAR
units = attrs.pop("punits", None)
calendar = attrs.pop("pcalendar", None)
if units is not None or calendar is not None:
attrs["Units"] = Units(units, calendar)
# AXES
pdimensions = attrs.pop("pdimensions", None)
if pdimensions is not None:
attrs["axes"] = pdimensions
# REVERSE
reverse = attrs.pop("reverse", None)
if reverse is not None:
attrs["reverse"] = reverse
# LOCATION: Change to python indexing (i.e. range does not
# include the final index)
for r in attrs["location"]:
r[1] += 1
# PART: Change to python indexing (i.e. slice range does
# not include the final index)
part = attrs.get("part", None)
if part:
p = []
for x in ast_literal_eval(part):
if isinstance(x, list):
if x[2] > 0:
p.append(slice(x[0], x[1] + 1, x[2]))
elif x[1] == 0:
p.append(slice(x[0], None, x[2]))
else:
p.append(slice(x[0], x[1] - 1, x[2]))
else:
p.append(list(x))
# --- End: for
attrs["part"] = p
# --- End: for
construct.del_property("cf_role")
construct.del_property("cfa_array")
construct.del_property("cfa_dimensions", None)
out = self._create_Data(loadd=cfa_data)
return out