def experiment_fn() -> Experiment:
# To mitigate issue https://github.com/tensorflow/tensorflow/issues/32159 for tf >= 1.15
import tensorflow as tf
def train_input_fn():
dataset = winequality.get_dataset(WINE_EQUALITY_FILE, split="train")
return dataset.shuffle(1000).batch(128).repeat()
def eval_input_fn():
dataset = winequality.get_dataset(WINE_EQUALITY_FILE, split="test")
return dataset.shuffle(1000).batch(128)
estimator = tf.estimator.LinearClassifier(
feature_columns=winequality.get_feature_columns(),
model_dir=f"{HDFS_DIR}",
n_classes=winequality.get_n_classes(),
optimizer=lambda: hvd.DistributedOptimizer(tf.train.AdamOptimizer()))
return Experiment(
estimator,
tf.estimator.TrainSpec(train_input_fn,
max_steps=10,
hooks=[hvd.BroadcastGlobalVariablesHook(0)]),
tf.estimator.EvalSpec(eval_input_fn,
steps=10,
start_delay_secs=0,
throttle_secs=30))
def experiment_fn() -> Experiment:
train_data, test_data = winequality.get_train_eval_datasets(WINE_EQUALITY_FILE)
def train_input_fn():
return (train_data.shuffle(1000)
.batch(128)
.repeat()
.make_one_shot_iterator()
.get_next())
def eval_input_fn():
return (test_data.shuffle(1000)
.batch(128)
.make_one_shot_iterator()
.get_next())
estimator = tf.estimator.LinearClassifier(
feature_columns=winequality.get_feature_columns(),
model_dir=f"{HDFS_DIR}",
n_classes=winequality.get_n_classes())
return Experiment(
estimator,
tf.estimator.TrainSpec(train_input_fn, max_steps=10),
tf.estimator.EvalSpec(
eval_input_fn,
steps=10,
start_delay_secs=0,
throttle_secs=30))
def experiment_fn(dataset_path: str) -> Experiment:
train_data, test_data = winequality.get_train_eval_datasets(dataset_path)
def train_input_fn():
return (train_data.shuffle(1000).batch(
128).repeat().make_one_shot_iterator().get_next())
def eval_input_fn():
return (test_data.shuffle(1000).batch(
128).make_one_shot_iterator().get_next())
fs = check_output(
"hdfs getconf -confKey fs.defaultFS".split()).strip().decode()
user = pwd.getpwuid(os.getuid()).pw_name
config = tf.estimator.RunConfig(
tf_random_seed=42, model_dir=f"{fs}/user/{user}/examples/{run_id}")
estimator = tf.estimator.LinearClassifier(
winequality.get_feature_columns(),
n_classes=winequality.get_n_classes(),
config=config)
return Experiment(
estimator, tf.estimator.TrainSpec(train_input_fn, max_steps=10),
tf.estimator.EvalSpec(eval_input_fn,
steps=10,
start_delay_secs=0,
throttle_secs=30))
def experiment_fn() -> Experiment:
def train_input_fn():
train_data, test_data = winequality.get_train_eval_datasets(
WINE_EQUALITY_FILE)
return (train_data.shuffle(1000).batch(128).repeat())
estimator = tf.estimator.LinearClassifier(
optimizer=DistributedOptimizer(
tf.train.FtrlOptimizer(learning_rate=0.1)),
feature_columns=winequality.get_feature_columns(),
model_dir=f"{HDFS_DIR}",
n_classes=winequality.get_n_classes())
train_spec = tf.estimator.TrainSpec(train_input_fn,
max_steps=1000,
hooks=[BroadcastGlobalVariablesHook()])
return Experiment(estimator, train_spec,
tf.estimator.EvalSpec(lambda: True))
def experiment_fn() -> Experiment:
# To mitigate issue https://github.com/tensorflow/tensorflow/issues/32159 for tf >= 1.15
import tensorflow as tf
def train_input_fn():
dataset = winequality.get_dataset(WINE_EQUALITY_FILE, split="train")
return (dataset.shuffle(1000).batch(128).repeat())
def eval_input_fn():
dataset = winequality.get_dataset(WINE_EQUALITY_FILE, split="test")
return (dataset.shuffle(1000).batch(128))
estimator = tf.estimator.LinearClassifier(
feature_columns=winequality.get_feature_columns(),
model_dir=HDFS_DIR,
n_classes=winequality.get_n_classes())
return Experiment(
estimator, tf.estimator.TrainSpec(train_input_fn, max_steps=100),
tf.estimator.EvalSpec(eval_input_fn,
steps=10,
start_delay_secs=0,
throttle_secs=30))
pyenv_zip_path,
task_specs={
"worker": TaskSpec(memory="2 GiB", vcores=4, instances=2)
},
files=editable_requirements,
acls=skein.model.ACLs(enable=True,
view_users=['*'])) as cluster_spec:
distrib_config = tf.contrib.distribute.DistributeConfig(
train_distribute=tf.contrib.distribute.CollectiveAllReduceStrategy(
),
eval_distribute=tf.contrib.distribute.CollectiveAllReduceStrategy(
),
remote_cluster=cluster_spec)
run_config = tf.estimator.RunConfig(
experimental_distribute=distrib_config)
estimator = tf.estimator.LinearClassifier(
feature_columns=winequality.get_feature_columns(),
model_dir=f"{HDFS_DIR}",
n_classes=winequality.get_n_classes(),
optimizer='Adam',
config=run_config)
tf.estimator.train_and_evaluate(
estimator, tf.estimator.TrainSpec(train_input_fn, max_steps=1000),
tf.estimator.EvalSpec(eval_input_fn,
steps=10,
start_delay_secs=0,
throttle_secs=30))