def train(working_dir):
model_num, model_name = fsdb.get_latest_model()
games = gfile.Glob(os.path.join(fsdb.selfplay_dir(), model_name, '*.zz'))
if len(games) < MIN_GAMES_PER_GENERATION:
print("{} doesn't have enough games to train a new model yet ({})".
format(model_name, len(games)))
print("Sleeping...")
time.sleep(10 * 60)
print("Done...")
sys.exit(1)
print("Training on gathered game data, initializing from {}".format(
model_name))
new_model_num = model_num + 1
new_model_name = shipname.generate(new_model_num)
print("New model will be {}".format(new_model_name))
training_file = os.path.join(fsdb.golden_chunk_dir(),
str(new_model_num) + '.tfrecord.zz')
while not gfile.Exists(training_file):
print("Waiting for", training_file)
time.sleep(1 * 60)
print("Using Golden File:", training_file)
local_copy = os.path.join(working_dir, str(new_model_num) + '.tfrecord.zz')
print("Copying locally to ", local_copy)
gfile.Copy(training_file, local_copy, overwrite=True)
try:
save_file = os.path.join(fsdb.models_dir(), new_model_name)
main.train(working_dir, [local_copy], save_file)
except:
logging.exception("Train error")
finally:
os.remove(local_copy)
def save_config_file(config_file, dest_dir):
if not gfile.Exists(dest_dir):
gfile.MkDir(dest_dir)
return gfile.Copy(
config_file,
os.path.join(dest_dir, 'blueoil_config.yaml')
)
def save(self, name):
checkpoint_path_json, checkpoint_path_h5 = self._get_checkpoint_paths(
name)
model_json = self.model.to_json()
with gfile.Open(checkpoint_path_json, 'w') as json_file:
json_file.write(model_json)
self.model.save_weights('/tmp/tmp_model_weights.h5')
gfile.Copy('/tmp/tmp_model_weights.h5', checkpoint_path_h5)
def load_model(net, output_path, file_name):
weights_path = os.path.join(output_path, file_name, '.h5')
local_filename = weights_path.split('/')[-1]
tmp_filename = os.path.join(tempfile.gettempdir(),
str(int(time.time())) + '_' + local_filename)
gfile.Copy(weights_path, tmp_filename)
net.load_weights(tmp_filename)
gfile.Remove(tmp_filename)
def load(self, name):
checkpoint_path_json, checkpoint_path_h5 = self._get_checkpoint_paths(
name)
with gfile.Open(checkpoint_path_json, 'r') as json_file:
loaded_model_json = json_file.read()
model = model_from_json(loaded_model_json)
gfile.Copy(checkpoint_path_h5, '/tmp/tmp_model_weights.h5')
model.load_weights('/tmp/tmp_model_weights.h5')
logging.info('Loaded model from disk.')
return model
def CopyRecursively(src, dst, overwrite=False):
entries = gfile.ListDirectory(src)
for entry in entries:
src_path = os.path.join(src, entry)
dst_path = os.path.join(dst, entry)
if gfile.IsDirectory(src_path):
gfile.MkDir(dst_path)
CopyRecursively(src_path, dst_path, overwrite)
else:
gfile.Copy(src_path, dst_path, overwrite)
def copy_to_experiment_dir(config_file):
# copy config file to the experiment directory
saved_config_file_path = _config_file_path_to_copy(config_file)
# HACK: This is for tensorflow bug workaround.
# We can remove following 2 lines once it's been resolved in tensorflow
# issue link: https://github.com/tensorflow/tensorflow/issues/28508
if gfile.Exists(saved_config_file_path):
gfile.Remove(saved_config_file_path)
gfile.Copy(config_file, saved_config_file_path)
def upload_to_gcs(local_path, gcs_path):
"""Upload local file to Google Cloud Storage.
Args:
local_path: (string) Local file
gcs_path: (string) Google Cloud Storage destination
Returns:
None
"""
gfile.Copy(local_path, gcs_path)
def save_model(net, model_json, output_path, file_name):
"""serialize weights to HDF5."""
with gfile.Open(output_path + file_name + '.json', 'w') as json_file:
json_file.write(model_json)
# serialize weights to HDF5
weight_path = os.path.join(output_path, file_name, '.h5')
local_filename = weight_path.split('/')[-1]
tmp_filename = os.path.join(tempfile.gettempdir(),
str(int(time.time())) + '_' + local_filename)
net.save_weights(tmp_filename)
gfile.Copy(tmp_filename, weight_path, overwrite=True)
gfile.Remove(tmp_filename)
def evaluate_model(
dataset_path: str, model_path: str, metric_name: str
) -> NamedTuple('Outputs', [('metric_name', str), ('metric_value', float),
('mlpipeline_metrics', 'Metrics')]):
"""Evaluates a trained sklearn model."""
import joblib
# import pickle
import json
import pandas as pd
import subprocess
import sys
from tensorflow import gfile
from sklearn.metrics import accuracy_score, recall_score
df_test = pd.read_csv(dataset_path)
X_test = df_test.drop('Cover_Type', axis=1)
y_test = df_test['Cover_Type']
# Copy the model from GCS
model_filename = 'model.pkl'
gcs_model_filepath = '{}/{}'.format(model_path, model_filename)
print(gcs_model_filepath)
if gfile.Exists(model_filename):
gfile.Remove(model_filename)
gfile.Copy(gcs_model_filepath, model_filename)
with open(model_filename, 'rb') as model_file:
model = joblib.load(model_file)
y_hat = model.predict(X_test)
if metric_name == 'accuracy':
metric_value = accuracy_score(y_test, y_hat)
elif metric_name == 'recall':
metric_value = recall_score(y_test, y_hat)
else:
metric_name = 'N/A'
metric_value = 0
# Export the metric
metrics = {
'metrics': [{
'name': metric_name,
'numberValue': float(metric_value)
}]
}
return metric_name, metric_value, json.dumps(metrics)
def save_config_file(config_file, dest_dir):
if not gfile.Exists(dest_dir):
gfile.MkDir(dest_dir)
config_file_dest = os.path.join(dest_dir, 'blueoil_config.yaml')
# HACK: This is for tensorflow bug workaround.
# We can remove following 2 lines once it's been resolved in tensorflow
# issue link: https://github.com/tensorflow/tensorflow/issues/28508
if gfile.Exists(config_file_dest):
gfile.Remove(config_file_dest)
return gfile.Copy(config_file, config_file_dest)
def atomic_file(path):
"""Atomically saves data to a target path.
Any existing data at the target path will be overwritten.
Args:
path: target path at which to save file
Yields:
file-like object
"""
with tempfile.NamedTemporaryFile() as tmp:
yield tmp
tmp.flush()
# Necessary when the destination is on CNS.
gfile.Copy(tmp.name, '%s.tmp' % path, overwrite=True)
gfile.Rename('%s.tmp' % path, path, overwrite=True)
def copy_file(old_path, new_path):
"""Copy the file from old_path to new_path
The paths can be local or on GCS
Args:
old_path: (string) original file location
new_path: (string) new location and name for the file to be copied to
Returns:
None
"""
# Make sure the local nested directory already exists:
out_dir = ntpath.split(new_path)[0]
if not (out_dir.startswith('gs://') and os.path.exists(out_dir)):
os.makedirs(out_dir)
gfile.Copy(old_path, new_path, overwrite=True)
def embed_data(x, dset, path):
"""embeds x into the code space using the autoencoder."""
if x:
return np.zeros(shape=(0, 10))
# load model and weights
json_path = os.path.join(path, 'ae_{}.json'.format(dset))
print('load model from json file:', json_path)
with gfile.Open(json_path) as f:
pt_ae = model_from_json(f.read())
weights_path = os.path.join(path, 'ae_{}_weights.h5'.format(dset))
print('load code spase from:', weights_path)
local_filename = weights_path.split('/')[-1]
tmp_filename = os.path.join(tempfile.gettempdir(),
str(int(time.time())) + '_' + local_filename)
gfile.Copy(weights_path, tmp_filename)
pt_ae.load_weights(tmp_filename)
gfile.Remove(tmp_filename)
print('***********************', x.shape)
x = x.reshape(-1, np.prod(x.shape[1:]))
print('***********************', x.shape)
get_embeddings = K.function([pt_ae.input], [pt_ae.layers[3].output])
get_reconstruction = K.function([pt_ae.layers[4].input], [pt_ae.output])
x_embedded = predict_with_k_fn(get_embeddings, x)[0]
x_recon = predict_with_k_fn(get_reconstruction, x_embedded)[0]
reconstruction_mse = np.mean(np.square(x - x_recon))
print(
'using pretrained embeddings; sanity check, total reconstruction error:',
np.mean(reconstruction_mse))
del pt_ae
return x_embedded
def copy_to_experiment_dir(config_file):
# copy config file to the experiment directory
saved_config_file_path = _config_file_path_to_copy(config_file)
gfile.Copy(config_file, saved_config_file_path)
def verbose_copy(src, tgt, overwrite=True):
logging.info('Copying "%s" -> "%s"', src, tgt)
gfile.Copy(src, tgt, overwrite=overwrite)
def copy_checkpoint(source, target):
for ext in (".index", ".data-00000-of-00001"):
gfile.Copy(source + ext, target + ext)
