def __init__(self, dataset): self.loop_size = 1 if not hasattr(dataset, '__ME_INITED__'): if not hasattr(dataset, '__loop_size__'): self.loop_size = dataset.get_dataset_size() else: self.loop_size = dataset.__loop_size__ dataset.__ME_INITED__ = _exec_datagraph(dataset, self.loop_size).queue_name self.ind = 0 self.dataset = dataset dataset_types, dataset_shapes = _get_types_and_shapes(dataset) self.dataset_types, self.dataset_shapes = dataset_types, dataset_shapes
def __init__(self, dataset, sink_size, epoch_num): self.dataset = dataset self.sink_size = sink_size self.sink_count = 1 if not hasattr(dataset, '__TRANSFER_DATASET__'): if hasattr(dataset, '__loop_size__'): self.sink_size = dataset.__loop_size__ dataset.__TRANSFER_DATASET__ = _exec_datagraph(dataset, self.sink_size) if not hasattr(dataset, '__no_send__'): _send_data(dataset, epoch_num) else: _send_data_no_flag(dataset, epoch_num) self.stop_send = dataset.__TRANSFER_DATASET__.stop_send self.dataset_types, self.dataset_shapes = _get_types_and_shapes(dataset)
def __init__(self, dataset): self.loop_size = 1 if not hasattr(dataset, '__ME_INITED__'): if not hasattr(dataset, '__loop_size__'): self.loop_size = dataset.get_dataset_size() else: self.loop_size = dataset.__loop_size__ dataset.__TRANSFER_DATASET__ = _exec_datagraph(dataset, self.loop_size) dataset.__ME_INITED__ = dataset.__TRANSFER_DATASET__.queue_name if not hasattr(dataset, '__no_send__'): _send_data(dataset) else: _send_data(dataset) self.ind = 0 self.dataset = dataset dataset_types, dataset_shapes = _get_types_and_shapes(dataset) self.dataset_types, self.dataset_shapes = dataset_types, dataset_shapes
def __init__(self, dataset): self.loop_size = 1 if not hasattr(dataset, '__ME_INITED__'): if not hasattr(dataset, '__loop_size__'): self.loop_size = dataset.get_dataset_size() else: self.loop_size = dataset.__loop_size__ dataset.__ME_INITED__ = _exec_datagraph(dataset, self.loop_size).queue_name self.ind = 0 self.dataset = dataset dataset_types, dataset_shapes = _get_types_and_shapes(dataset) self.dataset_types, self.dataset_shapes = dataset_types, dataset_shapes # for self._parallel_mode equal to semi_auto_parallel or auto_parallel, use a complete tensor to # compile, and slice tensor to run. The batch dimension of tensors for compile is device_number # times the batch dimension of tensors for run if _get_parallel_mode() in (ParallelMode.SEMI_AUTO_PARALLEL, ParallelMode.AUTO_PARALLEL): device_num = _get_device_num() self.dataset_shapes = _to_full_shapes(dataset_shapes, device_num)