Python create_data_readerの例

コード例 #1

ファイル: data_reader_test.py プロジェクト: chunyang-wen/elasticdl

 def test_create_data_reader(self):
     reader = create_data_reader(data_origin="table",
                                 records_per_task=10,
                                 **{"columns": ["a", "b"]})
     self.assertEqual(reader._kwargs["columns"], ["a", "b"])
     self.assertEqual(reader._kwargs["records_per_task"], 10)
     reader = create_data_reader(data_origin="table", records_per_task=10)
     self.assertEqual(reader._kwargs["records_per_task"], 10)
     self.assertTrue("columns" not in reader._kwargs)

コード例 #2

 def __init__(self,
              worker,
              training_with_evaluation,
              data_reader_params=None):
     self._worker = worker
     self._training_with_evaluation = training_with_evaluation
     self._lock = threading.Lock()
     self._pending_dataset = True
     self._pending_eval_tasks = []
     self._reset()
     if data_reader_params:
         self._data_reader = create_data_reader(data_origin=None,
                                                **data_reader_params)
     else:
         self._data_reader = create_data_reader(data_origin=None)

コード例 #3

 def _maybe_create_shards(data_origin):
     return (
         create_data_reader(
             data_origin=data_origin, records_per_task=records_per_task
         ).create_shards()
         if data_origin
         else {}
     )

コード例 #4

 def __init__(self,
              worker,
              training_with_evaluation,
              data_reader_params=None):
     self._worker = worker
     self._training_with_evaluation = training_with_evaluation
     self._lock = threading.Lock()
     self._pending_dataset = True
     self._pending_save_model_task = None
     self._reset()
     if data_reader_params:
         self.data_reader = create_data_reader(data_origin=None,
                                               **data_reader_params)
     else:
         self.data_reader = create_data_reader(data_origin=None)
     self._warm_up_task = None
     self._has_warmed_up = False

コード例 #5

ファイル: master.py プロジェクト: zuodh/elasticdl

 def _maybe_create_shards(data_origin):
     wkargs = get_dict_from_params_str(data_reader_params)
     partition = wkargs.get("partition", None) if wkargs else None
     return (create_data_reader(
         data_origin=data_origin,
         records_per_task=records_per_task,
         partition=partition,
     ).create_shards() if data_origin else {})

コード例 #6

ファイル: task_data_service.py プロジェクト: zuodh/elasticdl

 def __init__(
     self, worker, training_with_evaluation, data_reader_params=None
 ):
     self._worker = worker
     self._training_with_evaluation = training_with_evaluation
     self._lock = threading.Lock()
     self._pending_dataset = True
     self._pending_save_model_task = None
     if data_reader_params:
         self.data_reader = create_data_reader(
             data_origin=None, **data_reader_params
         )
     else:
         self.data_reader = create_data_reader(data_origin=None)
     self._warm_up_task = None
     self._has_warmed_up = False
     self._failed_record_count = 0
     self._reported_record_count = 0
     self._current_task = None
     self._pending_tasks = deque()