def test_fromShape_categories():
d = {
'col1': [1, 2, 3, 4.0, 10],
'col2':
pd.Categorical([3, 4, 5, 6, 0]),
'col3':
pd.Categorical(['q', '2', 'c', '4', 'x'], ordered=True),
'cold':
pd.Series([
'05-09-1988', '22-12-1994', '21-11-1995', '22-06-1994',
'12-12-2012'
],
dtype='datetime64[ns]')
}
f = Frame(d)
f = f.setIndex(['col2', 'col3', 'col1'])
g = Frame.fromShape(f.shape)
s = Shape()
# fromShape does preserve index
s.colNames = ['cold']
s.colTypes = [Types.Datetime]
s.index = ['col3', 'col1', 'col2']
s.indexTypes = [
IndexType(Types.Ordinal),
IndexType(Types.Numeric),
IndexType(Types.Nominal)
]
assert g.shape == s == f.shape
def test_typing():
a = 4
b = pd.DataFrame()
d: int64 = 12
c = Frame()
assert isinstance(type(a), type) and isinstance(type(b), type) and issubclass(type(c), Frame) and \
isinstance(type(c), type) and isinstance(type(d), type)
def test_str_to_Timestamp_validation():
d = {
'col1': ['3', '0', '5', '6', '0'],
'col2': [3, 4, 5.1, 6, 0],
'col3': ['123', '2', '0.43', '4', '2021 January'],
'cold': ['05091988', '22121994', '21111995', '22061994', '12122012']
}
f = Frame(d)
op = ToTimestamp()
op.addInputShape(f.shape, 0)
assert op.getOutputShape() is None
with pytest.raises(exp.OptionValidationError):
op.setOptions(attributes={}, errors='raise')
assert not op.hasOptions() and op.getOutputShape() is None
with pytest.raises(exp.OptionValidationError):
op.setOptions(attributes={0: {'format': '%s'}}, errors='')
assert not op.hasOptions() and op.getOutputShape() is None
op.setOptions(attributes={0: {'format': '%d'}}, errors='coerce')
assert op.hasOptions()
assert op.getOutputShape().colTypes == [
Types.Datetime, Types.Numeric, Types.String, Types.String
]
def test_toString():
d = {
'col1': ['3', '0', '5', '6', '0'],
'col2': [3, 4, 5.1, 6, 0],
'col3': ['123', '2', '0.43', '4', '2021 January'],
'cold': ['05091988', '22121994', '21111995', '22061994', '12122012']
}
f = Frame(d)
op = ToString()
assert op.getOutputShape() is None
op.addInputShape(f.shape, 0)
assert op.getOutputShape() is None
opts = {'attributes': {1: None}}
assert op.getOptions() == {'attributes': dict()}
op.setOptions(**opts)
assert op.getOutputShape().colTypes == [
Types.String, Types.String, Types.String, Types.String
]
assert op.getOptions() == opts
assert isDictDeepCopy(op.getOptions(), opts)
g = op.execute(f)
assert op.getOutputShape() == g.shape
def test_SetInput():
d = {
'col1': [1, 2, 3, 4.0, 10],
'col2': [3, 4, 5, 6, 0],
'col3': ['q', '2', 'c', '4', 'x'],
'date':
['05-09-1988', '22-12-1994', '21-11-1995', '22-06-1994', '12-12-2012']
}
f = Frame(d)
work = WorkbenchModelMock()
# Set dataframe
work.setDataframeByName('var', f)
op = SetInput(work)
assert op.getOutputShape() is None
assert op.getOptions() == {'inputF': None}
op.setOptions(inputF='var')
op.addInputShape(Shape(), pos=0) # this does nothing
assert op.getOptions() == {'inputF': 'var'}
assert op.getOutputShape() == f.shape
g = op.execute()
assert g == f
# g should be a copy
f = f.rename({'col1': 'ewew'})
assert g != f
def test_rename():
d = {
'col1': [1, 2, 3, 4, 10],
'col2': [3, 4, 5, 6, 0],
'col3': ['q', '2', 'c', '4', 'x']
}
f = Frame(d)
op = RenameColumns()
op.addInputShape(f.shape, pos=0)
assert op.getOutputShape() is None
op.setOptions(names={0: 'col4', 2: 'col1'})
assert op.getOptions() == [{0: 'col4', 2: 'col1'}]
os = f.shape.clone()
os.colNames = ['col4', 'col2', 'col1']
assert op.getOutputShape() == os
g = op.execute(f)
gd = {
'col4': [1, 2, 3, 4, 10],
'col2': [3, 4, 5, 6, 0],
'col1': ['q', '2', 'c', '4', 'x']
}
assert g.to_dict() == gd
def test_unsetOptions_toNumeric():
d = {
'col1': pd.Categorical([1, 2, 3, 4, 10]),
'col2': [3, 4, 5, 6, 0],
'col3': ['q', '2', 'c', '4', 'x']
}
f = Frame(d)
op = ToNumeric()
op.addInputShape(f.shape, pos=0)
assert op.getOptions() == {
'attributes': {},
'errors': 'raise'
} and not op.hasOptions()
op.setOptions(attributes={0: dict()}, errors='raise')
assert op.getOptions() == {'attributes': {0: None}, 'errors': 'raise'}
assert op._shapes[0] == f.shape
op.unsetOptions()
assert op.getOptions() == {'attributes': {}, 'errors': 'raise'}
assert op._shapes[0] == f.shape
op.removeInputShape(0)
assert op.getOptions() == {'attributes': {}, 'errors': 'raise'}
assert op._shapes == [None]
op.setOptions(attributes={1: dict()}, errors='coerce')
assert op.getOptions() == {'attributes': {1: None}, 'errors': 'coerce'}
assert op._shapes == [None]
op.addInputShape(f.shape, pos=0)
assert op.getOptions() == {'attributes': {1: None}, 'errors': 'coerce'}
assert op._shapes[0] == f.shape
def test_nominal_to_ordinal_cat():
d = {
'col1': pd.Categorical(["5", "0", "5", "U", "0 ww"], ordered=False),
'col2': [3, 4, 5.1, 6, 0]
}
f = Frame(d)
op = ToCategorical()
op.addInputShape(f.shape, pos=0)
op.setOptions(attributes={0: {'cat': 'U 0 1 5', 'ordered': True}})
# Predict output shape
os = f.shape.columnsDict
os['col1'] = Types.Ordinal
assert op.getOutputShape().columnsDict == os
g = op.execute(f)
gd = {
'col1': ['5', '0', '5', 'U', None],
'col2': [3.0, 4.0, 5.1, 6.0, 0.0]
}
assert nan_to_None(g.to_dict()) == gd
assert g.shape.columnsDict == os
assert list(
g.getRawFrame()['col1'].dtype.categories) == ['U', '0', '1', '5']
assert g.getRawFrame()['col1'].dtype.ordered is True
def test_addInputShape_exc():
d = {'col1': [1, 2, 3, 4.0, 10], 'col2': [3, 4, 5, 6, 0], 'col3': ['q', '2', 'c', '4', 'x'],
'date': ['05-09-1988', '22-12-1994', '21-11-1995', '22-06-1994', '12-12-2012']}
f = Frame(d)
op = DummyOp()
with pytest.raises(ValueError):
op.addInputShape(f.shape, pos=-1)
def computeDiff(self) -> None:
frame1 = self.columnsL.model().frameModel().frame.getRawFrame()
frame2 = self.columnsR.model().frameModel().frame.getRawFrame()
changedMask = frame1 != frame2
diffRows = changedMask.any(1)
diffColumns = changedMask.any(0)
frame = frame1.loc[diffRows, diffColumns]
self.tableWidget.model().sourceModel().setFrame(Frame(frame))
def test_shape_index():
d = {
'col1': [1, 2, 3, 4.0, 10],
'col2': [3, 4, 5, 6, 0],
'col3': ['q', '2', 'c', '4', 'x']
}
f = Frame(d)
f = Frame(f.getRawFrame().set_index('col3'))
# Desired shape obj
s = Shape()
s.index = ['col3']
s.indexTypes = [IndexType(Types.String)]
s.colNames = ['col1', 'col2']
s.colTypes = [Types.Numeric, Types.Numeric]
assert f.shape == s
assert f.nRows == 5
def test_unsetOptions_toCategory():
d = {
'col1': pd.Categorical([1, 2, 3, 4, 10]),
'col2': [3, 4, 5, 6, 0],
'col3': ['q', '2', 'c', '4', 'x']
}
f = Frame(d)
op = ToCategorical()
op.addInputShape(f.shape, pos=0)
op.setOptions(attributes={0: {'cat': ' " 2 e + " 1 ', 'ordered': True}})
assert op.getOptions() == {
'attributes': {
0: {
'cat': '"2 e +" 1',
'ordered': True
}
}
}
assert op._ToCategorical__attributes == {0: (['2 e +', '1'], True)}
assert op._shapes == [f.shape]
op.unsetOptions()
assert op.getOptions() == {'attributes': dict()}
assert op._ToCategorical__attributes == dict()
assert op._shapes == [f.shape]
op.removeInputShape(0)
assert op.getOptions() == {'attributes': dict()}
assert op._ToCategorical__attributes == dict()
assert op._shapes == [None]
op.setOptions(attributes={1: dict()})
assert op.getOptions() == {
'attributes': {
1: {
'cat': '',
'ordered': False
}
}
}
assert op._ToCategorical__attributes == {1: (None, None)}
assert op._shapes == [None]
assert op.getOutputShape() is None
op.addInputShape(f.shape, pos=0)
assert op.getOptions() == {
'attributes': {
1: {
'cat': '',
'ordered': False
}
}
}
assert op._ToCategorical__attributes == {1: (None, None)}
assert op._shapes == [f.shape]
def test_rename_bis():
d = {
'col1': [1, 2, 3, 4, 10],
'col2': [3, 4, 5, 6, 0],
'col3': ['q', '2', 'c', '4', 'x']
}
f = Frame(d)
g = f.rename(['cola', '21eeds', 'ij_'])
assert g.colnames == ['cola', '21eeds', 'ij_'
] and f.colnames == ['col1', 'col2', 'col3']
def test_rename():
d = {
'col1': [1, 2, 3, 4, 10],
'col2': [3, 4, 5, 6, 0],
'col3': ['q', '2', 'c', '4', 'x']
}
f = Frame(d)
g = f.rename({'col2': 'new'})
assert g.colnames == ['col1', 'new', 'col3'
] and f.colnames == ['col1', 'col2', 'col3']
def test_rename_excep():
d = {
'col1': [1, 2, 3, 4, 10],
'col2': [3, 4, 5, 6, 0],
'col3': ['q', '2', 'c', '4', 'x']
}
f = Frame(d)
names = f.colnames
names.append('1')
with pytest.raises(ValueError):
f.rename(names)
def __init__(self,
parent: QWidget = None,
frame: Union[Frame, Shape] = Frame()):
super().__init__(parent)
if isinstance(frame, Frame):
self.__frame: Frame = frame
self.__shape: Shape = self.__frame.shape
elif isinstance(frame, Shape): # it's a Shape
self.__frame: Frame = Frame()
self.__shape: Shape = frame
else:
self.__frame: Frame = Frame()
self.__shape: Shape = Shape()
# Dictionary { attributeIndex: value }
self._statistics: Dict[int, Dict[str, object]] = dict()
self._histogram: Dict[int, Dict[Any, int]] = dict()
# Dataframe name
self.name: str = ''
# Set of alive workers by identifier (attribute number, type, operation)
self._runningWorkers: Set[Tuple] = set()
self._dataAccessMutex = QMutex()
def test_cat_toNumeric():
d = {
'col1': pd.Categorical(['3', '0', '5', '6', '0']),
'col2': [3, 4, 5, 6, 0],
'col3': ['123', '2', '0.43', '4', '90']
}
# 'cold': pd.Series(['05-09-1988', '22-12-1994', '21-11-1995', '22-06-1994', '12-12-2012'],
# dtype='datetime64[ns]')}
f = Frame(d)
op = ToNumeric()
op.addInputShape(f.shape, pos=0)
assert op.getOutputShape() is None
assert op.getOptions() == {'attributes': {}, 'errors': 'raise'}
op.setOptions(attributes={0: None}, errors='coerce')
assert op.getOptions() == {'attributes': {0: None}, 'errors': 'coerce'}
# Predict output shape
os = f.shape.clone()
os.colTypes[0] = Types.Numeric
assert op.getOutputShape() == os
# Removing options/input_shape causes None to be returned
op.removeInputShape(0)
assert op.getOutputShape() is None
op.addInputShape(f.shape, pos=0)
op.unsetOptions()
assert op.getOutputShape() is None
op.setOptions(attributes={0: dict()}, errors='raise')
assert op.getOutputShape() == os # Re-adding everything
g = op.execute(f)
gd = {
'col1': [3.0, 0.0, 5.0, 6.0, 0.0],
'col2': [3, 4, 5, 6, 0],
'col3': ['123', '2', '0.43', '4', '90']
}
assert g.to_dict() == gd
assert g.shape == os
# Coerce is the same
op.setOptions(attributes={0: dict()}, errors='coerce')
assert op.getOutputShape() == os
g = op.execute(f)
assert g.to_dict() == gd
assert g.shape == os
def test_fromShape_datetime():
d = {
'col1': [1, 2, 3, 4.0, 10],
'col2': [3, 4, 5, 6, 0],
'col3': ['q', '2', 'c', '4', 'x'],
'cold':
pd.Series([
'05-09-1988', '22-12-1994', '21-11-1995', '22-06-1994',
'12-12-2012'
],
dtype='datetime64[ns]')
}
f = Frame(d)
f = Frame(f.getRawFrame().set_index(['col3', 'cold']))
g = Frame.fromShape(f.shape)
s = Shape()
# fromShape does preserve index
s.colNames = ['col1', 'col2']
s.colTypes = [Types.Numeric, Types.Numeric]
s.index = ['col3', 'cold']
s.indexTypes = [IndexType(Types.String), IndexType(Types.Datetime)]
assert g.shape == s == f.shape
def test_shape():
d = {
'col1': [1, 2, 3, 4, 10],
'col2': [3, 4, 5, 6, 0],
'col3': ['q', '2', 'c', '4', 'x']
}
f = Frame(d)
s = Shape()
s.index = ['Unnamed']
s.indexTypes = [IndexType(Types.Numeric)]
s.colNames = ['col1', 'col2', 'col3']
s.colTypes = [Types.Numeric, Types.Numeric, Types.String]
assert f.shape == s
assert f.nRows == 5
def test_unsetOptions():
d = {
'col1': [1, 2, 3, 4, 10],
'col2': [3, 4, 5, 6, 0],
'col3': ['q', '2', 'c', '4', 'x']
}
f = Frame(d)
op = RenameColumns()
op.addInputShape(f.shape, pos=0)
op.setOptions(names={0: 'col4', 2: 'col1'})
assert op.getOptions() == [{0: 'col4', 2: 'col1'}]
op.unsetOptions()
assert op.getOptions() == [{}]
def test_str_to_Timestamp_raise():
d = {
'col1': pd.Categorical([3, 0, 5, 6, 0]),
'col2': [3, 4, 5.1, 6, 0],
'col3': ['123', '2', '0.43', '4', '90'],
'cold': ['05091988', '22121994', '21111995', '22061994', '12122012']
}
f = Frame(d)
op = ToTimestamp()
op.addInputShape(f.shape, 0)
assert op.getOutputShape() is None
op.setOptions(attributes={3: {'format': '%d%m%Y'}}, errors='raise')
sd = f.shape.clone()
sd.colTypes[3] = Types.Datetime
assert op.getOutputShape() == sd
g = op.execute(f)
dateCol = g.getRawFrame()['cold']
assert op.getOutputShape() == g.shape == sd
def test_str_toNumeric():
d = {
'col1': pd.Categorical([3, 0, 5, 6, 0]),
'col2': [3, 4, 5, 6, 0],
'col3': ['123', '2', '0.43', '4', '90']
}
# 'cold': pd.Series(['05-09-1988', '22-12-1994', '21-11-1995', '22-06-1994', '12-12-2012'],
# dtype='datetime64[ns]')}
f = Frame(d)
op = ToNumeric()
op.addInputShape(f.shape, pos=0)
op.setOptions(attributes={0: dict(), 2: dict()}, errors='raise')
# Predict output shape
os = f.shape.columnsDict
os['col1'] = Types.Numeric
os['col3'] = Types.Numeric
assert op.getOutputShape().columnsDict == os
# Removing options/input_shape causes None to be returned
op.removeInputShape(0)
assert op.getOutputShape() is None
op.addInputShape(f.shape, pos=0)
op.unsetOptions()
assert op.getOutputShape() is None
op.setOptions(attributes={0: dict(), 2: dict()}, errors='coerce')
assert op.getOutputShape().columnsDict == os # Re-adding everything
g = op.execute(f)
gd = {
'col1': [3.0, 0.0, 5.0, 6.00, 0.0],
'col2': [3., 4., 5., 6., 0.0],
'col3': [123.0, 2.0, 0.43, 4.0, 90.0]
}
assert roundValues(g.to_dict(), 3) == gd
assert g.shape.columnsDict == os
assert g.shape.indexDict == f.shape.indexDict
def test_str_toCategory():
d = {
'col1': pd.Categorical(["3", "0", "5", "6", "0"]),
'col2': ["3", "4", "5.1", "6", None],
'col3': ['123', '2', '0.43', 'nan', '90']
}
# 'cold': pd.Series(['05-09-1988', '22-12-1994', '21-11-1995', '22-06-1994', '12-12-2012'],
# dtype='datetime64[ns]')}
f = Frame(d)
op = ToCategorical()
op.addInputShape(f.shape, pos=0)
op.setOptions(attributes={1: {'cat': '4 3 0', 'ordered': True}, 2: dict()})
# Predict output shape
os = f.shape.columnsDict
os['col3'] = Types.Nominal
os['col2'] = Types.Ordinal
assert op.getOutputShape().columnsDict == os
# Removing options/input_shape causes None to be returned
op.removeInputShape(0)
assert op.getOutputShape() is None
op.addInputShape(f.shape, pos=0)
op.unsetOptions()
assert op.getOutputShape() is None
op.setOptions(attributes={1: {'cat': '4 3 0', 'ordered': True}, 2: dict()})
assert op.getOutputShape().columnsDict == os # Re-adding everything
g = op.execute(f)
gd = {
'col1': ["3", "0", "5", "6", "0"],
'col2': ['3', '4', None, None, None],
'col3': ['123', '2', '0.43', 'nan', '90']
}
assert nan_to_None(g.to_dict()) == gd
assert g.shape.columnsDict == os
def test_num_toCategory():
d = {
'col1': pd.Categorical([3, 0, 5, 6, 0]),
'col2': [3, 4, 5.1, 6, 0],
'col4': [1, 2, 3, 4, 5], # this will become a float
'col3': ['123', '2', '0.43', '4', '90']
}
f = Frame(d)
op = ToCategorical()
op.addInputShape(f.shape, pos=0)
op.setOptions(attributes={1: dict(), 2: dict()})
# Predict output shape
os = f.shape.columnsDict
os['col2'] = Types.Nominal
os['col4'] = Types.Nominal
assert op.getOutputShape().columnsDict == os
# Removing options/input_shape causes None to be returned
op.removeInputShape(0)
assert op.getOutputShape() is None
op.addInputShape(f.shape, pos=0)
op.unsetOptions()
assert op.getOutputShape() is None
op.setOptions(attributes={1: dict(), 2: dict()})
assert op.getOutputShape().columnsDict == os # Re-adding everything
g = op.execute(f)
gd = {
'col1': [3, 0, 5, 6, 0],
'col2': ["3.0", "4.0", "5.1", "6.0", "0.0"],
'col3': ['123', '2', '0.43', '4', '90'],
'col4': ["1.0", "2.0", "3.0", "4.0", "5.0"]
}
assert g.to_dict() == gd
assert g.shape.columnsDict == os
def test_add_removeInputShape():
d = {'col1': [1, 2, 3, 4.0, 10], 'col2': [3, 4, 5, 6, 0], 'col3': ['q', '2', 'c', '4', 'x'],
'date': ['05-09-1988', '22-12-1994', '21-11-1995', '22-06-1994', '12-12-2012']}
f = Frame(d)
class MyOp(DummyOp):
def maxInputNumber(self) -> int:
return 2
mop = MyOp()
gs = f.shape
gs.colNames[0] = 'hola'
mop.addInputShape(f.shape, pos=1)
mop.addInputShape(gs, pos=0)
assert mop._shapes == [gs, f.shape]
assert mop._shapes != [f.shape, gs]
# Test if exception is thrown
with pytest.raises(IndexError):
mop.addInputShape(gs, pos=2)
# Test if exception is thrown in DummyOp
with pytest.raises(IndexError):
DummyOp().addInputShape(gs, pos=1)
# Adding / removing input shape should preserve the list length
ss = mop._shapes.copy()
mop.removeInputShape(pos=0)
assert mop._shapes == [None, ss[1]]
mop.addInputShape(gs, pos=0)
assert mop._shapes == [gs, f.shape]
def test_cat_toCategory():
d = {
'col1': pd.Categorical(["5", "0", "5", "U", "0"]),
'col2': [3, 4, 5.1, 6, 0],
'col4': [1, 2, 3, 4, 5], # this will become a float
'col3': ['123', '2', '0.43', '4', '90']
}
f = Frame(d)
op = ToCategorical()
op.addInputShape(f.shape, pos=0)
op.setOptions(attributes={0: {'cat': '5 0'}})
# Predict output shape
os = f.shape.columnsDict
assert op.getOutputShape().columnsDict == os
# Removing options/input_shape causes None to be returned
op.removeInputShape(0)
assert op.getOutputShape() is None
op.addInputShape(f.shape, pos=0)
op.unsetOptions()
assert op.getOutputShape() is None
op.setOptions(attributes={0: {'cat': '5 0'}})
assert op.getOutputShape().columnsDict == os # Re-adding everything
g = op.execute(f)
gd = {
'col1': ["5", "0", "5", None, "0"],
'col2': [3.0, 4.0, 5.1, 6.0, 0.0],
'col3': ['123', '2', '0.43', '4', '90'],
'col4': [1.0, 2.0, 3.0, 4.0, 5.0]
}
assert nan_to_None(g.to_dict()) == gd
assert g.shape.columnsDict == os
def run(self):
header = pd.read_csv(self.__path,
sep=self.__sep,
index_col=False,
nrows=0)
self.resultReady.emit(Frame(header))
def test_str_to_Timestamp_coerce():
d = {
'col1': ['3', '0', '5', '6', '0'],
'col2': [3, 4, 5.1, 6, 0],
'col3': ['123', '2', '0.43', '4', '2021 January'],
'cold': ['05091988', '22121994', '21111995', '22061994', '12122012']
}
f = Frame(d)
op = ToTimestamp()
assert op.getOutputShape() is None
op.addInputShape(f.shape, 0)
assert op.getOutputShape() is None
assert op.getOptions() == {'attributes': {}, 'errors': 'raise'}
op.setOptions(attributes={
0: dict(),
2: {
'format': '%Y %B'
},
3: {
'format': '%d%m%Y'
}
},
errors='raise')
assert op.getOutputShape().colTypes == [
Types.Datetime, Types.Numeric, Types.Datetime, Types.Datetime
]
assert op.getOptions() == {
'attributes': {
0: {
'format': ''
},
2: {
'format': '%Y %B'
},
3: {
'format': '%d%m%Y'
}
},
'errors': 'raise'
}
# Raise exception since pandas cannot convert all values to datetime
with pytest.raises(Exception):
op.execute(f)
op.setOptions(attributes={
0: dict(),
2: {
'format': '%Y %B'
},
3: {
'format': '%d%m%Y'
}
},
errors='coerce')
g = op.execute(f)
pg = g.getRawFrame()
assert op.getOutputShape() == g.shape