def add_datasource(self, datasource: BaseDatasource):
"""
Add datasource to pipeline.
Args:
datasource: class of type BaseDatasource
"""
self.datasource = datasource
self.steps_dict[keys.TrainingSteps.DATA] = datasource.get_data_step()
def wrapper(ds1: BaseDatasource, ds2: BaseDatasource, loaded=True):
# There can be a "None" datasource in a pipeline
if ds1 is None and ds2 is None:
return True
if sum(d is None for d in [ds1, ds2]) == 1:
return False
equal = False
equal |= ds1.name == ds2.name
equal |= ds1.schema == ds2.schema
equal |= ds1._id == ds2._id
equal |= ds1._source == ds2._source
equal |= equal_steps(ds1.get_data_step(), ds2.get_data_step(),
loaded=loaded)
if loaded:
equal |= ds1._immutable != ds2._immutable
else:
equal |= ds1._immutable == ds2._immutable
return equal
def wrapper(ds1: BaseDatasource, ds2: BaseDatasource, loaded=True):
# There can be a "None" datasource in a pipeline
if ds1 is None and ds2 is None:
return True
if sum(d is None for d in [ds1, ds2]) == 1:
return False
equal = False
equal |= ds1.name == ds2.name
equal |= ds1._id == ds2._id
equal |= ds1._source == ds2._source
equal |= equal_steps(ds1.get_data_step(), ds2.get_data_step(),
loaded=loaded)
# TODO[LOW]: Add more checks for constructor kwargs, __dict__ etc.
if loaded:
equal |= ds1._immutable != ds2._immutable
else:
equal |= ds1._immutable == ds2._immutable
return equal
def test_get_datastep():
first_ds = BaseDatasource(name="my_datasource")
assert not first_ds.get_data_step()