def test_create_data_reader(self):
reader = create_data_reader(data_origin="table",
records_per_task=10,
**{"columns": ["a", "b"]})
self.assertEqual(reader._kwargs["columns"], ["a", "b"])
self.assertEqual(reader._kwargs["records_per_task"], 10)
reader = create_data_reader(data_origin="table", records_per_task=10)
self.assertEqual(reader._kwargs["records_per_task"], 10)
self.assertTrue("columns" not in reader._kwargs)
def __init__(self,
worker,
training_with_evaluation,
data_reader_params=None):
self._worker = worker
self._training_with_evaluation = training_with_evaluation
self._lock = threading.Lock()
self._pending_dataset = True
self._pending_eval_tasks = []
self._reset()
if data_reader_params:
self._data_reader = create_data_reader(data_origin=None,
**data_reader_params)
else:
self._data_reader = create_data_reader(data_origin=None)
def _maybe_create_shards(data_origin):
return (
create_data_reader(
data_origin=data_origin, records_per_task=records_per_task
).create_shards()
if data_origin
else {}
)
def __init__(self,
worker,
training_with_evaluation,
data_reader_params=None):
self._worker = worker
self._training_with_evaluation = training_with_evaluation
self._lock = threading.Lock()
self._pending_dataset = True
self._pending_save_model_task = None
self._reset()
if data_reader_params:
self.data_reader = create_data_reader(data_origin=None,
**data_reader_params)
else:
self.data_reader = create_data_reader(data_origin=None)
self._warm_up_task = None
self._has_warmed_up = False
def _maybe_create_shards(data_origin):
wkargs = get_dict_from_params_str(data_reader_params)
partition = wkargs.get("partition", None) if wkargs else None
return (create_data_reader(
data_origin=data_origin,
records_per_task=records_per_task,
partition=partition,
).create_shards() if data_origin else {})
def __init__(
self, worker, training_with_evaluation, data_reader_params=None
):
self._worker = worker
self._training_with_evaluation = training_with_evaluation
self._lock = threading.Lock()
self._pending_dataset = True
self._pending_save_model_task = None
if data_reader_params:
self.data_reader = create_data_reader(
data_origin=None, **data_reader_params
)
else:
self.data_reader = create_data_reader(data_origin=None)
self._warm_up_task = None
self._has_warmed_up = False
self._failed_record_count = 0
self._reported_record_count = 0
self._current_task = None
self._pending_tasks = deque()