def train_network(network, in_train, in_valid, max_iters):
valid_fn = canopy.handled_fn(
network,
[
canopy.handlers.time_call(key="valid_time"),
canopy.handlers.override_hyperparameters(deterministic=True),
canopy.handlers.chunk_variables(batch_size=BATCH_SIZE, variables=["x", "y"]),
],
{"x": "x", "y": "y"},
{"valid_cost": "cost", "pred": "pred"},
)
def validate(in_dict, result_dict):
valid_out = valid_fn(in_valid)
probabilities = valid_out.pop("pred")
predicted_classes = np.argmax(probabilities, axis=1)
result_dict["valid_accuracy"] = sklearn.metrics.accuracy_score(in_valid["y"], predicted_classes)
result_dict.update(valid_out)
train_fn = canopy.handled_fn(
network,
[
canopy.handlers.time_call(key="total_time"),
canopy.handlers.call_after_every(1, validate),
canopy.handlers.time_call(key="train_time"),
canopy.handlers.chunk_variables(batch_size=BATCH_SIZE, variables=["x", "y"]),
],
{"x": "x", "y": "y"},
{"train_cost": "cost"},
include_updates=True,
)
def callback(results_dict):
print(
"{_iter:3d}: "
"train_cost: {train_cost:0.3f} "
"valid_cost: {valid_cost:0.3f} "
"valid_accuracy: {valid_accuracy:0.3f}".format(**results_dict)
)
print("Starting training...")
canopy.evaluate_until(fn=train_fn, gen=itertools.repeat(in_train), max_iters=max_iters, callback=callback)
def train_network(network, in_train, in_valid, max_iters):
valid_fn = canopy.handled_fn(
network,
[canopy.handlers.time_call(key="valid_time"),
canopy.handlers.override_hyperparameters(deterministic=True),
canopy.handlers.chunk_variables(batch_size=BATCH_SIZE,
variables=["x", "y"])],
{"x": "x", "y": "y"},
{"valid_cost": "cost", "pred": "pred"})
def validate(in_dict, result_dict):
valid_out = valid_fn(in_valid)
probabilities = valid_out.pop("pred")
predicted_classes = np.argmax(probabilities, axis=1)
result_dict["valid_accuracy"] = sklearn.metrics.accuracy_score(
in_valid["y"], predicted_classes)
result_dict.update(valid_out)
train_fn = canopy.handled_fn(
network,
[canopy.handlers.time_call(key="total_time"),
canopy.handlers.call_after_every(1, validate),
canopy.handlers.time_call(key="train_time"),
canopy.handlers.chunk_variables(batch_size=BATCH_SIZE,
variables=["x", "y"])],
{"x": "x", "y": "y"},
{"train_cost": "cost"},
include_updates=True)
def callback(results_dict):
print("{_iter:3d}: "
"train_cost: {train_cost:0.3f} "
"valid_cost: {valid_cost:0.3f} "
"valid_accuracy: {valid_accuracy:0.3f}".format(**results_dict))
print("Starting training...")
canopy.evaluate_until(fn=train_fn,
gen=itertools.repeat(in_train),
max_iters=max_iters,
callback=callback)
"pred": "pred"
})
def validate(in_dict, results_dict):
valid_out = valid_fn(valid)
valid_y = valid["y"]
probabilities = valid_out.pop("pred")[:len(valid_y)]
predicted_classes = np.argmax(probabilities, axis=1)
valid_out["valid_accuracy"] = (valid_y == predicted_classes).mean()
results_dict.update(valid_out)
train_fn = canopy.handled_fn(network, [
canopy.handlers.time_call(key="total_time"),
canopy.handlers.call_after_every(1, validate),
canopy.handlers.time_call(key="train_time"),
canopy.handlers.batch_pad(BATCH_SIZE, keys=["x", "y"]),
canopy.handlers.chunk_variables(batch_size=BATCH_SIZE,
variables=["x", "y"])
], {
"x": "x",
"y": "y"
}, {"train_cost": "cost"},
include_updates=True)
# ################################# training #################################
print("Starting training...")
canopy.evaluate_until(fn=train_fn, gen=itertools.repeat(train), max_iters=200)
def validate(in_dict, results_dict):
valid_out = valid_fn(in_valid)
probabilities = valid_out["pred"]
predicted_classes = np.argmax(probabilities, axis=1)
results_dict["valid_cost"] = valid_out["total_cost"]
results_dict["valid_time"] = valid_out["valid_time"]
results_dict["valid_accuracy"] = sklearn.metrics.accuracy_score(
y_valid, predicted_classes)
train_fn = canopy.handled_fn(
network,
[canopy.handlers.time_call(key="total_time"),
canopy.handlers.call_after_every(1, validate),
canopy.handlers.time_call(key="train_time"),
canopy.handlers.chunk_variables(batch_size=BATCH_SIZE,
variables=["x", "y"])],
{"x": "x", "y": "y"},
{"train_cost": "total_cost",
"train_cp_cost1": "cp1_sendto",
"train_cp_cost2": "cp2_sendto",
"train_classification_cost": "cost"},
include_updates=True)
# ################################# training #################################
print("Starting training...")
canopy.evaluate_until(fn=train_fn,
gen=itertools.repeat(in_train),
max_iters=25)
BEST_ACC[0] = max(BEST_ACC[0], log["valid_accuracy"])
print("best: %0.4g" % BEST_ACC[0])
def train_generator():
x = train["x"]
y = train["y"]
chunk_size = len(train["x"])
while True:
x_chunk = []
y_chunk = []
for _ in range(chunk_size):
idx = np.random.randint(len(x))
tmp_x = x[idx]
# flip axes
if np.random.rand() < 0.5:
tmp_x = tmp_x[:, :, ::-1]
x_chunk.append(tmp_x)
y_chunk.append(y[idx])
yield {"x": np.array(x_chunk), "y": np.array(y_chunk)}
print("Starting training...")
canopy.evaluate_until(
fn=train_fn,
gen=train_generator(),
# max_iters=200,
# HACK
max_iters=10000,
callback=callback)
def validate(in_dict, results_dict):
valid_out = valid_fn(valid)
probabilities = valid_out["pred"]
predicted_classes = np.argmax(probabilities, axis=1)
results_dict["valid_cost"] = valid_out["cost"]
results_dict["valid_time"] = valid_out["valid_time"]
results_dict["valid_accuracy"] = sklearn.metrics.accuracy_score(
valid["y"], predicted_classes)
train_fn = canopy.handled_fn(network, [
canopy.handlers.time_call(key="total_time"),
canopy.handlers.call_after_every(1, validate),
canopy.handlers.time_call(key="train_time"),
canopy.handlers.chunk_variables(batch_size=BATCH_SIZE,
variables=["x", "y"])
], {
"x": "x",
"y": "y"
}, {"train_cost": "cost"},
include_updates=True)
# ################################# training #################################
print("Starting training...")
canopy.evaluate_until(fn=train_fn,
gen=itertools.repeat(train),
max_iters=NUM_EPOCHS)
result_dict["valid_accuracy"] = sklearn.metrics.accuracy_score(
valid["y"], predicted_classes)
result_dict.update(valid_out)
train_fn = canopy.handled_fn(network, [
canopy.handlers.time_call(key="total_time"),
canopy.handlers.call_after_every(1, validate),
canopy.handlers.time_call(key="train_time"),
canopy.handlers.chunk_variables(batch_size=BATCH_SIZE,
variables=["x", "y"])
], {
"x": "x",
"y": "y"
}, {"train_cost": "cost"},
include_updates=True)
def callback(results_dict):
print("{_iter:3d}: "
"train_cost: {train_cost:0.3f} "
"valid_cost: {valid_cost:0.3f} "
"valid_accuracy: {valid_accuracy:0.3f}".format(**results_dict))
print("Starting training...")
canopy.evaluate_until(fn=train_fn,
gen=itertools.repeat(train),
max_iters=MAX_ITERS,
callback=callback)
LOGS.append(log)
pprint.pprint(log)
BEST_ACC[0] = max(BEST_ACC[0], log["valid_accuracy"])
print("best: %0.4g" % BEST_ACC[0])
def train_generator():
x = train["x"]
y = train["y"]
chunk_size = len(train["x"])
while True:
x_chunk = []
y_chunk = []
for _ in range(chunk_size):
idx = np.random.randint(len(x))
tmp_x = x[idx]
# flip axes
if np.random.rand() < 0.5:
tmp_x = tmp_x[:, :, ::-1]
x_chunk.append(tmp_x)
y_chunk.append(y[idx])
yield {"x": np.array(x_chunk), "y": np.array(y_chunk)}
print("Starting training...")
canopy.evaluate_until(fn=train_fn,
gen=train_generator(),
# max_iters=200,
# HACK
max_iters=10000,
callback=callback)
{"cost": "cost", "pred": "pred"})
def validate(in_dict, results_dict):
valid_out = valid_fn(valid)
probabilities = valid_out["pred"]
predicted_classes = np.argmax(probabilities, axis=1)
results_dict["valid_cost"] = valid_out["cost"]
results_dict["valid_time"] = valid_out["valid_time"]
results_dict["valid_accuracy"] = sklearn.metrics.accuracy_score(
valid["y"], predicted_classes)
train_fn = canopy.handled_fn(
network,
[canopy.handlers.time_call(key="total_time"),
canopy.handlers.call_after_every(1, validate),
canopy.handlers.time_call(key="train_time"),
canopy.handlers.chunk_variables(batch_size=BATCH_SIZE,
variables=["x", "y"])],
{"x": "x", "y": "y"},
{"train_cost": "cost"},
include_updates=True)
# ################################# training #################################
print("Starting training...")
canopy.evaluate_until(fn=train_fn,
gen=itertools.repeat(train),
max_iters=NUM_EPOCHS)
results_dict["valid_cost"] = valid_out["cost"]
results_dict["valid_time"] = valid_out["valid_time"]
results_dict["valid_accuracy"] = sklearn.metrics.accuracy_score(
y_valid, predicted_classes)
train_fn = canopy.handled_fn(network, [
canopy.handlers.time_call(key="total_time"),
canopy.handlers.call_after_every(1, validate),
canopy.handlers.time_call(key="train_time"),
canopy.handlers.evaluate_monitoring_variables(fmt="train_%s"),
canopy.handlers.chunk_variables(batch_size=BATCH_SIZE,
variables=["x", "y"])
], {
"x": "x",
"y": "y"
}, {"train_cost": "cost"},
include_updates=True)
# ################################# training #################################
import canopy.sandbox.monitor_ui
result_writer = canopy.sandbox.monitor_ui.ResultWriter(dirname="monitor_dir",
pattern="")
print("Starting training...")
canopy.evaluate_until(fn=train_fn,
gen=itertools.repeat(in_train),
max_iters=1000,
callback=result_writer.write)
"train_time",
"train_cost",
"train_iou",
"valid_time",
"valid_cost",
"valid_iou",
"window_min",
"window_max"
}
def store_to_trial(train_log):
trial.store("train_log", train_log, silent=True)
if train_log["_iter"] % 10 == 0:
trial.store_important("iter", train_log["_iter"])
summary.update(train_log)
result_writer.write(train_log)
pprint(dslib.toolz.keyfilter(lambda k: k in print_keys,
train_log))
# Run train loop
canopy.evaluate_until(fn=train_fn,
gen=train_gen,
max_iters=params.num_chunks,
max_seconds=params.time_limit,
callback=store_to_trial)
# Store summary
summary_filepath = trial.file_path('summary.json')
# value_dict has numpy values, need a numpy-aware serializer
dslib.io_utils.json_dump(summary.to_value_dict(), summary_filepath)
st.monitor_affine_parameters(st_node.name),
canopy.handlers.schedule_hyperparameter(
canopy.schedules.PiecewiseLinearSchedule([(1, 0.0), (5, 1.0)]),
node_name="late_gate"),
canopy.handlers.schedule_hyperparameter(
canopy.schedules.PiecewiseLinearSchedule([(1, 0.2),
(1000, 0.001)]),
node_name="sigma_hp"),
canopy.handlers.time_call(key="train_time"),
canopy.handlers.chunk_variables(batch_size=BATCH_SIZE,
variables=["x", "y"])],
{"x": "x", "y": "y"},
{"train_cost": "cost"},
include_updates=True)
BEST_VALID_ACCURACY = [0]
def callback(results_dict):
BEST_VALID_ACCURACY[0] = max(BEST_VALID_ACCURACY[0],
results_dict["valid_accuracy"])
import pprint
pprint.pprint(results_dict)
print("best: %0.3f" % BEST_VALID_ACCURACY[0])
print("Starting training...")
canopy.evaluate_until(fn=train_fn,
gen=itertools.repeat(train),
max_iters=max_iters,
callback=callback)
y_valid, predicted_classes)
train_fn = canopy.handled_fn(
network,
[canopy.handlers.time_call(key="total_time"),
canopy.handlers.schedule_hyperparameter(
canopy.schedules.PiecewiseLinearSchedule([(10, 2e-3),
(15, 2e-4),
(20, 2e-5)]),
hyperparameter="learning_rate"),
canopy.handlers.call_after_every(1, validate),
canopy.handlers.time_call(key="train_time"),
canopy.handlers.chunk_variables(batch_size=BATCH_SIZE,
variables=["x", "y"])],
{"x": "x", "y": "y"},
{"train_cost": "cost"},
include_updates=True)
# ################################# training #################################
import canopy.sandbox.monitor_ui
result_writer = canopy.sandbox.monitor_ui.ResultWriter(dirname="monitor_dir",
pattern="")
print("Starting training...")
canopy.evaluate_until(fn=train_fn,
gen=itertools.repeat(in_train),
max_iters=25,
callback=result_writer.write)
