def _parse_service(service):
"""Converts a tf.data service string into a (protocol, address) tuple.
Args:
service: A string in the format "protocol://address" or just "address". If
the string is only an address, the default protocol will be used.
Returns:
The (protocol, address) tuple
"""
if not isinstance(service, six.string_types):
raise ValueError(
"service must be a string, but service was of type {0}. service={1}"
.format(type(service), service))
if not service:
raise ValueError("service must not be empty")
parts = service.split("://")
if len(parts) == 2:
protocol, address = parts
elif len(parts) == 1:
address = parts[0]
protocol = _pywrap_utils.TF_DATA_DefaultProtocol()
else:
raise ValueError("malformed service string has multiple '://': %s" %
service)
# TODO(aaudibert): Considering validating reachability of address here.
return (protocol, address)
def __new__(cls,
port=0,
protocol=None,
work_dir=None,
fault_tolerant_mode=False,
worker_addresses=None,
job_gc_check_interval_ms=None,
job_gc_timeout_ms=None):
if protocol is None:
protocol = _pywrap_utils.TF_DATA_DefaultProtocol()
job_gc_check_interval_ms = _get_time_or_placeholder(
job_gc_check_interval_ms)
job_gc_timeout_ms = _get_time_or_placeholder(job_gc_timeout_ms)
return super(DispatcherConfig,
cls).__new__(cls, port, protocol, work_dir,
fault_tolerant_mode, worker_addresses,
job_gc_check_interval_ms, job_gc_timeout_ms)
def __new__(cls,
port=0,
protocol=None,
work_dir=None,
fault_tolerant_mode=False,
job_gc_check_interval_ms=None,
job_gc_timeout_ms=None):
if protocol is None:
protocol = _pywrap_utils.TF_DATA_DefaultProtocol()
if job_gc_check_interval_ms is None:
job_gc_check_interval_ms = 10 * 60 * 1000 # 10 minutes.
if job_gc_timeout_ms is None:
job_gc_timeout_ms = 5 * 60 * 1000 # 5 minutes.
return super(DispatcherConfig,
cls).__new__(cls, port, protocol, work_dir,
fault_tolerant_mode, job_gc_check_interval_ms,
job_gc_timeout_ms)
def __new__(cls,
dispatcher_address,
worker_address=None,
port=0,
protocol=None,
heartbeat_interval_ms=None,
dispatcher_timeout_ms=None):
if worker_address is None:
worker_address = "localhost:%port%"
if protocol is None:
protocol = _pywrap_utils.TF_DATA_DefaultProtocol()
heartbeat_interval_ms = _get_time_or_placeholder(heartbeat_interval_ms)
dispatcher_timeout_ms = _get_time_or_placeholder(dispatcher_timeout_ms)
return super(WorkerConfig,
cls).__new__(cls, dispatcher_address, worker_address, port,
protocol, heartbeat_interval_ms,
dispatcher_timeout_ms)
def __new__(cls,
dispatcher_address,
worker_address=None,
port=0,
protocol=None,
heartbeat_interval_ms=None,
dispatcher_timeout_ms=None):
if worker_address is None:
worker_address = "localhost:%port%"
if protocol is None:
protocol = _pywrap_utils.TF_DATA_DefaultProtocol()
if heartbeat_interval_ms is None:
heartbeat_interval_ms = 30 * 1000 # 30 seconds
if dispatcher_timeout_ms is None:
dispatcher_timeout_ms = 60 * 60 * 1000 # 1 hour
return super(WorkerConfig,
cls).__new__(cls, dispatcher_address, worker_address, port,
protocol, heartbeat_interval_ms,
dispatcher_timeout_ms)
def __new__(cls,
port=0,
protocol=None,
work_dir=None,
fault_tolerant_mode=False,
job_gc_check_interval_ms=None,
job_gc_timeout_ms=None,
cache_policy=1,
cache_format=2,
cache_compression=1,
cache_ops_parallelism=8,
cache_path="./outputs",
scaling_policy=1,
log_dir="",
log_dumps_interval_ms=None):
if protocol is None:
protocol = _pywrap_utils.TF_DATA_DefaultProtocol()
if job_gc_check_interval_ms is None:
job_gc_check_interval_ms = 10 * 60 * 1000 # 10 minutes.
if job_gc_timeout_ms is None:
job_gc_timeout_ms = 5 * 60 * 1000 # 5 minutes.
if log_dumps_interval_ms is None:
log_dumps_interval_ms = 100 # 100msec
"""
if cache_policy is None:
cache_policy=1
if cache_format is None:
cache_format=1
if cache_ops_parallelism is None:
cache_ops_parallelism = 8
if scaling_policy is None:
scaling_policy = 1"""
return super(DispatcherConfig, cls).__new__(
cls, port, protocol, work_dir, fault_tolerant_mode,
job_gc_check_interval_ms, job_gc_timeout_ms, cache_policy,
cache_format, cache_compression, cache_ops_parallelism, cache_path,
scaling_policy, log_dir, log_dumps_interval_ms)
