def test_fromShape_categories(): d = { 'col1': [1, 2, 3, 4.0, 10], 'col2': pd.Categorical([3, 4, 5, 6, 0]), 'col3': pd.Categorical(['q', '2', 'c', '4', 'x'], ordered=True), 'cold': pd.Series([ '05-09-1988', '22-12-1994', '21-11-1995', '22-06-1994', '12-12-2012' ], dtype='datetime64[ns]') } f = Frame(d) f = f.setIndex(['col2', 'col3', 'col1']) g = Frame.fromShape(f.shape) s = Shape() # fromShape does preserve index s.colNames = ['cold'] s.colTypes = [Types.Datetime] s.index = ['col3', 'col1', 'col2'] s.indexTypes = [ IndexType(Types.Ordinal), IndexType(Types.Numeric), IndexType(Types.Nominal) ] assert g.shape == s == f.shape
def test_typing(): a = 4 b = pd.DataFrame() d: int64 = 12 c = Frame() assert isinstance(type(a), type) and isinstance(type(b), type) and issubclass(type(c), Frame) and \ isinstance(type(c), type) and isinstance(type(d), type)
def test_str_to_Timestamp_validation(): d = { 'col1': ['3', '0', '5', '6', '0'], 'col2': [3, 4, 5.1, 6, 0], 'col3': ['123', '2', '0.43', '4', '2021 January'], 'cold': ['05091988', '22121994', '21111995', '22061994', '12122012'] } f = Frame(d) op = ToTimestamp() op.addInputShape(f.shape, 0) assert op.getOutputShape() is None with pytest.raises(exp.OptionValidationError): op.setOptions(attributes={}, errors='raise') assert not op.hasOptions() and op.getOutputShape() is None with pytest.raises(exp.OptionValidationError): op.setOptions(attributes={0: {'format': '%s'}}, errors='') assert not op.hasOptions() and op.getOutputShape() is None op.setOptions(attributes={0: {'format': '%d'}}, errors='coerce') assert op.hasOptions() assert op.getOutputShape().colTypes == [ Types.Datetime, Types.Numeric, Types.String, Types.String ]
def test_toString(): d = { 'col1': ['3', '0', '5', '6', '0'], 'col2': [3, 4, 5.1, 6, 0], 'col3': ['123', '2', '0.43', '4', '2021 January'], 'cold': ['05091988', '22121994', '21111995', '22061994', '12122012'] } f = Frame(d) op = ToString() assert op.getOutputShape() is None op.addInputShape(f.shape, 0) assert op.getOutputShape() is None opts = {'attributes': {1: None}} assert op.getOptions() == {'attributes': dict()} op.setOptions(**opts) assert op.getOutputShape().colTypes == [ Types.String, Types.String, Types.String, Types.String ] assert op.getOptions() == opts assert isDictDeepCopy(op.getOptions(), opts) g = op.execute(f) assert op.getOutputShape() == g.shape
def test_SetInput(): d = { 'col1': [1, 2, 3, 4.0, 10], 'col2': [3, 4, 5, 6, 0], 'col3': ['q', '2', 'c', '4', 'x'], 'date': ['05-09-1988', '22-12-1994', '21-11-1995', '22-06-1994', '12-12-2012'] } f = Frame(d) work = WorkbenchModelMock() # Set dataframe work.setDataframeByName('var', f) op = SetInput(work) assert op.getOutputShape() is None assert op.getOptions() == {'inputF': None} op.setOptions(inputF='var') op.addInputShape(Shape(), pos=0) # this does nothing assert op.getOptions() == {'inputF': 'var'} assert op.getOutputShape() == f.shape g = op.execute() assert g == f # g should be a copy f = f.rename({'col1': 'ewew'}) assert g != f
def test_rename(): d = { 'col1': [1, 2, 3, 4, 10], 'col2': [3, 4, 5, 6, 0], 'col3': ['q', '2', 'c', '4', 'x'] } f = Frame(d) op = RenameColumns() op.addInputShape(f.shape, pos=0) assert op.getOutputShape() is None op.setOptions(names={0: 'col4', 2: 'col1'}) assert op.getOptions() == [{0: 'col4', 2: 'col1'}] os = f.shape.clone() os.colNames = ['col4', 'col2', 'col1'] assert op.getOutputShape() == os g = op.execute(f) gd = { 'col4': [1, 2, 3, 4, 10], 'col2': [3, 4, 5, 6, 0], 'col1': ['q', '2', 'c', '4', 'x'] } assert g.to_dict() == gd
def test_unsetOptions_toNumeric(): d = { 'col1': pd.Categorical([1, 2, 3, 4, 10]), 'col2': [3, 4, 5, 6, 0], 'col3': ['q', '2', 'c', '4', 'x'] } f = Frame(d) op = ToNumeric() op.addInputShape(f.shape, pos=0) assert op.getOptions() == { 'attributes': {}, 'errors': 'raise' } and not op.hasOptions() op.setOptions(attributes={0: dict()}, errors='raise') assert op.getOptions() == {'attributes': {0: None}, 'errors': 'raise'} assert op._shapes[0] == f.shape op.unsetOptions() assert op.getOptions() == {'attributes': {}, 'errors': 'raise'} assert op._shapes[0] == f.shape op.removeInputShape(0) assert op.getOptions() == {'attributes': {}, 'errors': 'raise'} assert op._shapes == [None] op.setOptions(attributes={1: dict()}, errors='coerce') assert op.getOptions() == {'attributes': {1: None}, 'errors': 'coerce'} assert op._shapes == [None] op.addInputShape(f.shape, pos=0) assert op.getOptions() == {'attributes': {1: None}, 'errors': 'coerce'} assert op._shapes[0] == f.shape
def test_nominal_to_ordinal_cat(): d = { 'col1': pd.Categorical(["5", "0", "5", "U", "0 ww"], ordered=False), 'col2': [3, 4, 5.1, 6, 0] } f = Frame(d) op = ToCategorical() op.addInputShape(f.shape, pos=0) op.setOptions(attributes={0: {'cat': 'U 0 1 5', 'ordered': True}}) # Predict output shape os = f.shape.columnsDict os['col1'] = Types.Ordinal assert op.getOutputShape().columnsDict == os g = op.execute(f) gd = { 'col1': ['5', '0', '5', 'U', None], 'col2': [3.0, 4.0, 5.1, 6.0, 0.0] } assert nan_to_None(g.to_dict()) == gd assert g.shape.columnsDict == os assert list( g.getRawFrame()['col1'].dtype.categories) == ['U', '0', '1', '5'] assert g.getRawFrame()['col1'].dtype.ordered is True
def test_addInputShape_exc(): d = {'col1': [1, 2, 3, 4.0, 10], 'col2': [3, 4, 5, 6, 0], 'col3': ['q', '2', 'c', '4', 'x'], 'date': ['05-09-1988', '22-12-1994', '21-11-1995', '22-06-1994', '12-12-2012']} f = Frame(d) op = DummyOp() with pytest.raises(ValueError): op.addInputShape(f.shape, pos=-1)
def computeDiff(self) -> None: frame1 = self.columnsL.model().frameModel().frame.getRawFrame() frame2 = self.columnsR.model().frameModel().frame.getRawFrame() changedMask = frame1 != frame2 diffRows = changedMask.any(1) diffColumns = changedMask.any(0) frame = frame1.loc[diffRows, diffColumns] self.tableWidget.model().sourceModel().setFrame(Frame(frame))
def test_shape_index(): d = { 'col1': [1, 2, 3, 4.0, 10], 'col2': [3, 4, 5, 6, 0], 'col3': ['q', '2', 'c', '4', 'x'] } f = Frame(d) f = Frame(f.getRawFrame().set_index('col3')) # Desired shape obj s = Shape() s.index = ['col3'] s.indexTypes = [IndexType(Types.String)] s.colNames = ['col1', 'col2'] s.colTypes = [Types.Numeric, Types.Numeric] assert f.shape == s assert f.nRows == 5
def test_unsetOptions_toCategory(): d = { 'col1': pd.Categorical([1, 2, 3, 4, 10]), 'col2': [3, 4, 5, 6, 0], 'col3': ['q', '2', 'c', '4', 'x'] } f = Frame(d) op = ToCategorical() op.addInputShape(f.shape, pos=0) op.setOptions(attributes={0: {'cat': ' " 2 e + " 1 ', 'ordered': True}}) assert op.getOptions() == { 'attributes': { 0: { 'cat': '"2 e +" 1', 'ordered': True } } } assert op._ToCategorical__attributes == {0: (['2 e +', '1'], True)} assert op._shapes == [f.shape] op.unsetOptions() assert op.getOptions() == {'attributes': dict()} assert op._ToCategorical__attributes == dict() assert op._shapes == [f.shape] op.removeInputShape(0) assert op.getOptions() == {'attributes': dict()} assert op._ToCategorical__attributes == dict() assert op._shapes == [None] op.setOptions(attributes={1: dict()}) assert op.getOptions() == { 'attributes': { 1: { 'cat': '', 'ordered': False } } } assert op._ToCategorical__attributes == {1: (None, None)} assert op._shapes == [None] assert op.getOutputShape() is None op.addInputShape(f.shape, pos=0) assert op.getOptions() == { 'attributes': { 1: { 'cat': '', 'ordered': False } } } assert op._ToCategorical__attributes == {1: (None, None)} assert op._shapes == [f.shape]
def test_rename_bis(): d = { 'col1': [1, 2, 3, 4, 10], 'col2': [3, 4, 5, 6, 0], 'col3': ['q', '2', 'c', '4', 'x'] } f = Frame(d) g = f.rename(['cola', '21eeds', 'ij_']) assert g.colnames == ['cola', '21eeds', 'ij_' ] and f.colnames == ['col1', 'col2', 'col3']
def test_rename(): d = { 'col1': [1, 2, 3, 4, 10], 'col2': [3, 4, 5, 6, 0], 'col3': ['q', '2', 'c', '4', 'x'] } f = Frame(d) g = f.rename({'col2': 'new'}) assert g.colnames == ['col1', 'new', 'col3' ] and f.colnames == ['col1', 'col2', 'col3']
def test_rename_excep(): d = { 'col1': [1, 2, 3, 4, 10], 'col2': [3, 4, 5, 6, 0], 'col3': ['q', '2', 'c', '4', 'x'] } f = Frame(d) names = f.colnames names.append('1') with pytest.raises(ValueError): f.rename(names)
def __init__(self, parent: QWidget = None, frame: Union[Frame, Shape] = Frame()): super().__init__(parent) if isinstance(frame, Frame): self.__frame: Frame = frame self.__shape: Shape = self.__frame.shape elif isinstance(frame, Shape): # it's a Shape self.__frame: Frame = Frame() self.__shape: Shape = frame else: self.__frame: Frame = Frame() self.__shape: Shape = Shape() # Dictionary { attributeIndex: value } self._statistics: Dict[int, Dict[str, object]] = dict() self._histogram: Dict[int, Dict[Any, int]] = dict() # Dataframe name self.name: str = '' # Set of alive workers by identifier (attribute number, type, operation) self._runningWorkers: Set[Tuple] = set() self._dataAccessMutex = QMutex()
def test_cat_toNumeric(): d = { 'col1': pd.Categorical(['3', '0', '5', '6', '0']), 'col2': [3, 4, 5, 6, 0], 'col3': ['123', '2', '0.43', '4', '90'] } # 'cold': pd.Series(['05-09-1988', '22-12-1994', '21-11-1995', '22-06-1994', '12-12-2012'], # dtype='datetime64[ns]')} f = Frame(d) op = ToNumeric() op.addInputShape(f.shape, pos=0) assert op.getOutputShape() is None assert op.getOptions() == {'attributes': {}, 'errors': 'raise'} op.setOptions(attributes={0: None}, errors='coerce') assert op.getOptions() == {'attributes': {0: None}, 'errors': 'coerce'} # Predict output shape os = f.shape.clone() os.colTypes[0] = Types.Numeric assert op.getOutputShape() == os # Removing options/input_shape causes None to be returned op.removeInputShape(0) assert op.getOutputShape() is None op.addInputShape(f.shape, pos=0) op.unsetOptions() assert op.getOutputShape() is None op.setOptions(attributes={0: dict()}, errors='raise') assert op.getOutputShape() == os # Re-adding everything g = op.execute(f) gd = { 'col1': [3.0, 0.0, 5.0, 6.0, 0.0], 'col2': [3, 4, 5, 6, 0], 'col3': ['123', '2', '0.43', '4', '90'] } assert g.to_dict() == gd assert g.shape == os # Coerce is the same op.setOptions(attributes={0: dict()}, errors='coerce') assert op.getOutputShape() == os g = op.execute(f) assert g.to_dict() == gd assert g.shape == os
def test_fromShape_datetime(): d = { 'col1': [1, 2, 3, 4.0, 10], 'col2': [3, 4, 5, 6, 0], 'col3': ['q', '2', 'c', '4', 'x'], 'cold': pd.Series([ '05-09-1988', '22-12-1994', '21-11-1995', '22-06-1994', '12-12-2012' ], dtype='datetime64[ns]') } f = Frame(d) f = Frame(f.getRawFrame().set_index(['col3', 'cold'])) g = Frame.fromShape(f.shape) s = Shape() # fromShape does preserve index s.colNames = ['col1', 'col2'] s.colTypes = [Types.Numeric, Types.Numeric] s.index = ['col3', 'cold'] s.indexTypes = [IndexType(Types.String), IndexType(Types.Datetime)] assert g.shape == s == f.shape
def test_shape(): d = { 'col1': [1, 2, 3, 4, 10], 'col2': [3, 4, 5, 6, 0], 'col3': ['q', '2', 'c', '4', 'x'] } f = Frame(d) s = Shape() s.index = ['Unnamed'] s.indexTypes = [IndexType(Types.Numeric)] s.colNames = ['col1', 'col2', 'col3'] s.colTypes = [Types.Numeric, Types.Numeric, Types.String] assert f.shape == s assert f.nRows == 5
def test_unsetOptions(): d = { 'col1': [1, 2, 3, 4, 10], 'col2': [3, 4, 5, 6, 0], 'col3': ['q', '2', 'c', '4', 'x'] } f = Frame(d) op = RenameColumns() op.addInputShape(f.shape, pos=0) op.setOptions(names={0: 'col4', 2: 'col1'}) assert op.getOptions() == [{0: 'col4', 2: 'col1'}] op.unsetOptions() assert op.getOptions() == [{}]
def test_str_to_Timestamp_raise(): d = { 'col1': pd.Categorical([3, 0, 5, 6, 0]), 'col2': [3, 4, 5.1, 6, 0], 'col3': ['123', '2', '0.43', '4', '90'], 'cold': ['05091988', '22121994', '21111995', '22061994', '12122012'] } f = Frame(d) op = ToTimestamp() op.addInputShape(f.shape, 0) assert op.getOutputShape() is None op.setOptions(attributes={3: {'format': '%d%m%Y'}}, errors='raise') sd = f.shape.clone() sd.colTypes[3] = Types.Datetime assert op.getOutputShape() == sd g = op.execute(f) dateCol = g.getRawFrame()['cold'] assert op.getOutputShape() == g.shape == sd
def test_str_toNumeric(): d = { 'col1': pd.Categorical([3, 0, 5, 6, 0]), 'col2': [3, 4, 5, 6, 0], 'col3': ['123', '2', '0.43', '4', '90'] } # 'cold': pd.Series(['05-09-1988', '22-12-1994', '21-11-1995', '22-06-1994', '12-12-2012'], # dtype='datetime64[ns]')} f = Frame(d) op = ToNumeric() op.addInputShape(f.shape, pos=0) op.setOptions(attributes={0: dict(), 2: dict()}, errors='raise') # Predict output shape os = f.shape.columnsDict os['col1'] = Types.Numeric os['col3'] = Types.Numeric assert op.getOutputShape().columnsDict == os # Removing options/input_shape causes None to be returned op.removeInputShape(0) assert op.getOutputShape() is None op.addInputShape(f.shape, pos=0) op.unsetOptions() assert op.getOutputShape() is None op.setOptions(attributes={0: dict(), 2: dict()}, errors='coerce') assert op.getOutputShape().columnsDict == os # Re-adding everything g = op.execute(f) gd = { 'col1': [3.0, 0.0, 5.0, 6.00, 0.0], 'col2': [3., 4., 5., 6., 0.0], 'col3': [123.0, 2.0, 0.43, 4.0, 90.0] } assert roundValues(g.to_dict(), 3) == gd assert g.shape.columnsDict == os assert g.shape.indexDict == f.shape.indexDict
def test_str_toCategory(): d = { 'col1': pd.Categorical(["3", "0", "5", "6", "0"]), 'col2': ["3", "4", "5.1", "6", None], 'col3': ['123', '2', '0.43', 'nan', '90'] } # 'cold': pd.Series(['05-09-1988', '22-12-1994', '21-11-1995', '22-06-1994', '12-12-2012'], # dtype='datetime64[ns]')} f = Frame(d) op = ToCategorical() op.addInputShape(f.shape, pos=0) op.setOptions(attributes={1: {'cat': '4 3 0', 'ordered': True}, 2: dict()}) # Predict output shape os = f.shape.columnsDict os['col3'] = Types.Nominal os['col2'] = Types.Ordinal assert op.getOutputShape().columnsDict == os # Removing options/input_shape causes None to be returned op.removeInputShape(0) assert op.getOutputShape() is None op.addInputShape(f.shape, pos=0) op.unsetOptions() assert op.getOutputShape() is None op.setOptions(attributes={1: {'cat': '4 3 0', 'ordered': True}, 2: dict()}) assert op.getOutputShape().columnsDict == os # Re-adding everything g = op.execute(f) gd = { 'col1': ["3", "0", "5", "6", "0"], 'col2': ['3', '4', None, None, None], 'col3': ['123', '2', '0.43', 'nan', '90'] } assert nan_to_None(g.to_dict()) == gd assert g.shape.columnsDict == os
def test_num_toCategory(): d = { 'col1': pd.Categorical([3, 0, 5, 6, 0]), 'col2': [3, 4, 5.1, 6, 0], 'col4': [1, 2, 3, 4, 5], # this will become a float 'col3': ['123', '2', '0.43', '4', '90'] } f = Frame(d) op = ToCategorical() op.addInputShape(f.shape, pos=0) op.setOptions(attributes={1: dict(), 2: dict()}) # Predict output shape os = f.shape.columnsDict os['col2'] = Types.Nominal os['col4'] = Types.Nominal assert op.getOutputShape().columnsDict == os # Removing options/input_shape causes None to be returned op.removeInputShape(0) assert op.getOutputShape() is None op.addInputShape(f.shape, pos=0) op.unsetOptions() assert op.getOutputShape() is None op.setOptions(attributes={1: dict(), 2: dict()}) assert op.getOutputShape().columnsDict == os # Re-adding everything g = op.execute(f) gd = { 'col1': [3, 0, 5, 6, 0], 'col2': ["3.0", "4.0", "5.1", "6.0", "0.0"], 'col3': ['123', '2', '0.43', '4', '90'], 'col4': ["1.0", "2.0", "3.0", "4.0", "5.0"] } assert g.to_dict() == gd assert g.shape.columnsDict == os
def test_add_removeInputShape(): d = {'col1': [1, 2, 3, 4.0, 10], 'col2': [3, 4, 5, 6, 0], 'col3': ['q', '2', 'c', '4', 'x'], 'date': ['05-09-1988', '22-12-1994', '21-11-1995', '22-06-1994', '12-12-2012']} f = Frame(d) class MyOp(DummyOp): def maxInputNumber(self) -> int: return 2 mop = MyOp() gs = f.shape gs.colNames[0] = 'hola' mop.addInputShape(f.shape, pos=1) mop.addInputShape(gs, pos=0) assert mop._shapes == [gs, f.shape] assert mop._shapes != [f.shape, gs] # Test if exception is thrown with pytest.raises(IndexError): mop.addInputShape(gs, pos=2) # Test if exception is thrown in DummyOp with pytest.raises(IndexError): DummyOp().addInputShape(gs, pos=1) # Adding / removing input shape should preserve the list length ss = mop._shapes.copy() mop.removeInputShape(pos=0) assert mop._shapes == [None, ss[1]] mop.addInputShape(gs, pos=0) assert mop._shapes == [gs, f.shape]
def test_cat_toCategory(): d = { 'col1': pd.Categorical(["5", "0", "5", "U", "0"]), 'col2': [3, 4, 5.1, 6, 0], 'col4': [1, 2, 3, 4, 5], # this will become a float 'col3': ['123', '2', '0.43', '4', '90'] } f = Frame(d) op = ToCategorical() op.addInputShape(f.shape, pos=0) op.setOptions(attributes={0: {'cat': '5 0'}}) # Predict output shape os = f.shape.columnsDict assert op.getOutputShape().columnsDict == os # Removing options/input_shape causes None to be returned op.removeInputShape(0) assert op.getOutputShape() is None op.addInputShape(f.shape, pos=0) op.unsetOptions() assert op.getOutputShape() is None op.setOptions(attributes={0: {'cat': '5 0'}}) assert op.getOutputShape().columnsDict == os # Re-adding everything g = op.execute(f) gd = { 'col1': ["5", "0", "5", None, "0"], 'col2': [3.0, 4.0, 5.1, 6.0, 0.0], 'col3': ['123', '2', '0.43', '4', '90'], 'col4': [1.0, 2.0, 3.0, 4.0, 5.0] } assert nan_to_None(g.to_dict()) == gd assert g.shape.columnsDict == os
def run(self): header = pd.read_csv(self.__path, sep=self.__sep, index_col=False, nrows=0) self.resultReady.emit(Frame(header))
def test_str_to_Timestamp_coerce(): d = { 'col1': ['3', '0', '5', '6', '0'], 'col2': [3, 4, 5.1, 6, 0], 'col3': ['123', '2', '0.43', '4', '2021 January'], 'cold': ['05091988', '22121994', '21111995', '22061994', '12122012'] } f = Frame(d) op = ToTimestamp() assert op.getOutputShape() is None op.addInputShape(f.shape, 0) assert op.getOutputShape() is None assert op.getOptions() == {'attributes': {}, 'errors': 'raise'} op.setOptions(attributes={ 0: dict(), 2: { 'format': '%Y %B' }, 3: { 'format': '%d%m%Y' } }, errors='raise') assert op.getOutputShape().colTypes == [ Types.Datetime, Types.Numeric, Types.Datetime, Types.Datetime ] assert op.getOptions() == { 'attributes': { 0: { 'format': '' }, 2: { 'format': '%Y %B' }, 3: { 'format': '%d%m%Y' } }, 'errors': 'raise' } # Raise exception since pandas cannot convert all values to datetime with pytest.raises(Exception): op.execute(f) op.setOptions(attributes={ 0: dict(), 2: { 'format': '%Y %B' }, 3: { 'format': '%d%m%Y' } }, errors='coerce') g = op.execute(f) pg = g.getRawFrame() assert op.getOutputShape() == g.shape