def add_datasource(self, datasource: BaseDatasource): """ Add datasource to pipeline. Args: datasource: class of type BaseDatasource """ self.datasource = datasource self.steps_dict[keys.TrainingSteps.DATA] = datasource.get_data_step()
def wrapper(ds1: BaseDatasource, ds2: BaseDatasource, loaded=True): # There can be a "None" datasource in a pipeline if ds1 is None and ds2 is None: return True if sum(d is None for d in [ds1, ds2]) == 1: return False equal = False equal |= ds1.name == ds2.name equal |= ds1.schema == ds2.schema equal |= ds1._id == ds2._id equal |= ds1._source == ds2._source equal |= equal_steps(ds1.get_data_step(), ds2.get_data_step(), loaded=loaded) if loaded: equal |= ds1._immutable != ds2._immutable else: equal |= ds1._immutable == ds2._immutable return equal
def wrapper(ds1: BaseDatasource, ds2: BaseDatasource, loaded=True): # There can be a "None" datasource in a pipeline if ds1 is None and ds2 is None: return True if sum(d is None for d in [ds1, ds2]) == 1: return False equal = False equal |= ds1.name == ds2.name equal |= ds1._id == ds2._id equal |= ds1._source == ds2._source equal |= equal_steps(ds1.get_data_step(), ds2.get_data_step(), loaded=loaded) # TODO[LOW]: Add more checks for constructor kwargs, __dict__ etc. if loaded: equal |= ds1._immutable != ds2._immutable else: equal |= ds1._immutable == ds2._immutable return equal
def test_get_datastep(): first_ds = BaseDatasource(name="my_datasource") assert not first_ds.get_data_step()