def revmap(integerarray):
data = np.where(integerarray.array > 2**31,
(integerarray.array -
2**31).astype('uint32').view('float32'),
(~integerarray.array).astype('uint32').view('float32'))
# TODO Check why np.nan values are mapped to the value below
data[data == 5.8774704e-39] = np.nan
return FloatArray.from_numpy(data)
def subset(obj, size):
if isinstance(obj, np.ndarray):
result, _ = randomsubset(obj, size)
elif isinstance(obj, xr.DataArray):
_, slices = randomsubset(obj.values, size)
slices = {x: y for x, y in zip(obj.dims, slices)}
result = obj.isel(**slices)
elif isinstance(obj, FloatArray):
array, _ = randomsubset(obj.array, size)
result = FloatArray(array)
else:
raise TypeError("Can't understand type.")
return result
def revmap(integerarray):
if not isinstance(integerarray, IntegerArray):
err_type = "Expected IntegerArray, got {}".format(
type(integerarray))
raise TypeError(err_type)
if integerarray.array.dtype in (np.int32, np.uint32):
length, d = 32, np.float32
elif integerarray.array.dtype in (
np.int64,
np.uint64,
int,
):
length, d = 64, np.float64
else:
err_msg = 'Expected 32 or 64 bits, got {}'.format(
integerarray.array.dtype)
raise TypeError(err_msg)
data = _v_rev_lindstrom(integerarray.array, length).astype(d)
return FloatArray.from_numpy(data)
def revmap(integerarray):
if not isinstance(integerarray, IntegerArray):
err_type = "Expected IntegerArray, got {}".format(
type(integerarray))
raise TypeError(err_type)
if integerarray.array.dtype in (np.int32, np.uint32):
i, o, d = ('>l', '>f', np.float32)
elif integerarray.array.dtype in (
np.int64,
np.uint64,
int,
):
i, o, d = ('>Q', '>d', np.float64)
else:
err_msg = 'Expected 32 or 64 bits, got {}'.format(
integerarray.array.dtype)
raise TypeError(err_msg)
data = _vraw(integerarray.array, i, o).astype(d)
return FloatArray.from_numpy(data)
# Sequencer
from cframe.modifier.sequencer.linear import Linear
# Predictor
from cframe.modifier.predictor.lastvalue import LastValue
from cframe.modifier.predictor.stride import Stride
# Subtractor
from cframe.modifier.subtractor.xor import XOR
from cframe.modifier.subtractor.floatingpoint import FPD
# Encoder
from cframe.modifier.encoder.raw import RawEncoder
DATA = [
FloatArray.from_data('pre', 'tas'),
# FloatArray.from_data('pre', 'ua'),
]
MAP = [
Lindstrom,
RawBinary,
Raw,
Ordered,
]
SEQ = [
Linear,
]
PRE = [
def revmap(integerarray):
data = integerarray.array.view('float32').copy()
return FloatArray.from_numpy(data)
def test_floatarray_with_other_object(value):
"""Comparison with arbitary types."""
with pytest.raises(TypeError) as err:
_ = FloatArray(ARR) == value
assert "Comparison failed" in str(err)
def test_mapping_and_reverse_work(mapper):
f = FloatArray.from_data('pre', 'tas')
iarr = mapper.map(f)
fnew = mapper.revmap(iarr)
assert np.array_equal(fnew.array, f.array)
def test_equal_floatarray():
"""Comparison with itself."""
assert FloatArray(ARR) == FloatArray(ARR)
def test_floatarray_with_npndarray():
"""Comparison with related values."""
assert FloatArray(ARR) == ARR
def test_value_error(value):
"""Wrong input types."""
with pytest.raises(TypeError) as err:
FloatArray(value)
assert "Expected np.nd" in str(err)
def test_raise_error(form):
"""Wrong np.dtypes of array."""
with pytest.raises(TypeError) as err:
FloatArray(ARR.astype(form))
assert "Expected np.nd" in str(err)
def test_importtype():
""""Correct initialisation."""
assert isinstance(FloatArray(ARR), FloatArray)
# Feeder
from cframe.toolbox.feeder import SeqFeeder
from cframe.toolbox.workflow import Workflow
from cframe.objects.arrays.floatarray import FloatArray
import sys
from cframe.toolbox.subsetting import Subset
if len(sys.argv) < 2:
raise Exception('Forgot the variable?')
val = int(sys.argv[2]) if len(sys.argv) > 2 else 0
if sys.argv[1] == 'ua':
a = FloatArray.from_data('pre', 'ua')
a = FloatArray.from_numpy(a.array[0,12,:,:])
if val != 0:
a = FloatArray.from_numpy(Subset.subset(a.array, val))
elif sys.argv[1] in ('tas', 'pr'):
a = FloatArray.from_data('pre', sys.argv[1])
if val != 0:
a = FloatArray.from_numpy(Subset.subset(a.array, val))
else:
raise Exception('Can not understand variable')
original = a.array.nbytes
print(a.array.shape)
value = int(arguments['N'])
sub = Subset.subset(data, value)
print(sub.shape)
elif arguments['shannon']:
from cframe.toolbox.shannon.entropy import shannon_binary
entropy = shannon_binary(data.flatten(), 32)
print(entropy)
elif arguments['parallel']:
from cframe.toolbox.parallel import ParallelProcessWorkflow
from cframe.toolbox.feeder import SeqFeeder
if arguments['--climate']:
# climate compression
for var in ['ua', 'tas']:
print("Compression of {}".format(var))
a = FloatArray.from_data('pre', var)
if var == 'ua':
a = FloatArray.from_numpy(a.array[0, 12, :, :])
if arguments['--subset'] != 'None':
value = int(arguments['--subset'])
from cframe.toolbox.subsetting import Subset
a = Subset.subset(a, value)
workflows = [
Workflow(m, Linear, p, sb, RawEncoder)
for m in [Ordered, Raw]
for p in [LastValue, Akumuli, TwoStride, StrideConfidence]
for sb in [XOR, FPD]
]
par = ParallelProcessWorkflow()
compressions = par.compress(workflows, a, 0, SeqFeeder)
import operator