def train(working_dir): model_num, model_name = fsdb.get_latest_model() games = gfile.Glob(os.path.join(fsdb.selfplay_dir(), model_name, '*.zz')) if len(games) < MIN_GAMES_PER_GENERATION: print("{} doesn't have enough games to train a new model yet ({})". format(model_name, len(games))) print("Sleeping...") time.sleep(10 * 60) print("Done...") sys.exit(1) print("Training on gathered game data, initializing from {}".format( model_name)) new_model_num = model_num + 1 new_model_name = shipname.generate(new_model_num) print("New model will be {}".format(new_model_name)) training_file = os.path.join(fsdb.golden_chunk_dir(), str(new_model_num) + '.tfrecord.zz') while not gfile.Exists(training_file): print("Waiting for", training_file) time.sleep(1 * 60) print("Using Golden File:", training_file) local_copy = os.path.join(working_dir, str(new_model_num) + '.tfrecord.zz') print("Copying locally to ", local_copy) gfile.Copy(training_file, local_copy, overwrite=True) try: save_file = os.path.join(fsdb.models_dir(), new_model_name) main.train(working_dir, [local_copy], save_file) except: logging.exception("Train error") finally: os.remove(local_copy)
def save_config_file(config_file, dest_dir): if not gfile.Exists(dest_dir): gfile.MkDir(dest_dir) return gfile.Copy( config_file, os.path.join(dest_dir, 'blueoil_config.yaml') )
def save(self, name): checkpoint_path_json, checkpoint_path_h5 = self._get_checkpoint_paths( name) model_json = self.model.to_json() with gfile.Open(checkpoint_path_json, 'w') as json_file: json_file.write(model_json) self.model.save_weights('/tmp/tmp_model_weights.h5') gfile.Copy('/tmp/tmp_model_weights.h5', checkpoint_path_h5)
def load_model(net, output_path, file_name): weights_path = os.path.join(output_path, file_name, '.h5') local_filename = weights_path.split('/')[-1] tmp_filename = os.path.join(tempfile.gettempdir(), str(int(time.time())) + '_' + local_filename) gfile.Copy(weights_path, tmp_filename) net.load_weights(tmp_filename) gfile.Remove(tmp_filename)
def load(self, name): checkpoint_path_json, checkpoint_path_h5 = self._get_checkpoint_paths( name) with gfile.Open(checkpoint_path_json, 'r') as json_file: loaded_model_json = json_file.read() model = model_from_json(loaded_model_json) gfile.Copy(checkpoint_path_h5, '/tmp/tmp_model_weights.h5') model.load_weights('/tmp/tmp_model_weights.h5') logging.info('Loaded model from disk.') return model
def CopyRecursively(src, dst, overwrite=False): entries = gfile.ListDirectory(src) for entry in entries: src_path = os.path.join(src, entry) dst_path = os.path.join(dst, entry) if gfile.IsDirectory(src_path): gfile.MkDir(dst_path) CopyRecursively(src_path, dst_path, overwrite) else: gfile.Copy(src_path, dst_path, overwrite)
def copy_to_experiment_dir(config_file): # copy config file to the experiment directory saved_config_file_path = _config_file_path_to_copy(config_file) # HACK: This is for tensorflow bug workaround. # We can remove following 2 lines once it's been resolved in tensorflow # issue link: https://github.com/tensorflow/tensorflow/issues/28508 if gfile.Exists(saved_config_file_path): gfile.Remove(saved_config_file_path) gfile.Copy(config_file, saved_config_file_path)
def upload_to_gcs(local_path, gcs_path): """Upload local file to Google Cloud Storage. Args: local_path: (string) Local file gcs_path: (string) Google Cloud Storage destination Returns: None """ gfile.Copy(local_path, gcs_path)
def save_model(net, model_json, output_path, file_name): """serialize weights to HDF5.""" with gfile.Open(output_path + file_name + '.json', 'w') as json_file: json_file.write(model_json) # serialize weights to HDF5 weight_path = os.path.join(output_path, file_name, '.h5') local_filename = weight_path.split('/')[-1] tmp_filename = os.path.join(tempfile.gettempdir(), str(int(time.time())) + '_' + local_filename) net.save_weights(tmp_filename) gfile.Copy(tmp_filename, weight_path, overwrite=True) gfile.Remove(tmp_filename)
def evaluate_model( dataset_path: str, model_path: str, metric_name: str ) -> NamedTuple('Outputs', [('metric_name', str), ('metric_value', float), ('mlpipeline_metrics', 'Metrics')]): """Evaluates a trained sklearn model.""" import joblib # import pickle import json import pandas as pd import subprocess import sys from tensorflow import gfile from sklearn.metrics import accuracy_score, recall_score df_test = pd.read_csv(dataset_path) X_test = df_test.drop('Cover_Type', axis=1) y_test = df_test['Cover_Type'] # Copy the model from GCS model_filename = 'model.pkl' gcs_model_filepath = '{}/{}'.format(model_path, model_filename) print(gcs_model_filepath) if gfile.Exists(model_filename): gfile.Remove(model_filename) gfile.Copy(gcs_model_filepath, model_filename) with open(model_filename, 'rb') as model_file: model = joblib.load(model_file) y_hat = model.predict(X_test) if metric_name == 'accuracy': metric_value = accuracy_score(y_test, y_hat) elif metric_name == 'recall': metric_value = recall_score(y_test, y_hat) else: metric_name = 'N/A' metric_value = 0 # Export the metric metrics = { 'metrics': [{ 'name': metric_name, 'numberValue': float(metric_value) }] } return metric_name, metric_value, json.dumps(metrics)
def save_config_file(config_file, dest_dir): if not gfile.Exists(dest_dir): gfile.MkDir(dest_dir) config_file_dest = os.path.join(dest_dir, 'blueoil_config.yaml') # HACK: This is for tensorflow bug workaround. # We can remove following 2 lines once it's been resolved in tensorflow # issue link: https://github.com/tensorflow/tensorflow/issues/28508 if gfile.Exists(config_file_dest): gfile.Remove(config_file_dest) return gfile.Copy(config_file, config_file_dest)
def atomic_file(path): """Atomically saves data to a target path. Any existing data at the target path will be overwritten. Args: path: target path at which to save file Yields: file-like object """ with tempfile.NamedTemporaryFile() as tmp: yield tmp tmp.flush() # Necessary when the destination is on CNS. gfile.Copy(tmp.name, '%s.tmp' % path, overwrite=True) gfile.Rename('%s.tmp' % path, path, overwrite=True)
def copy_file(old_path, new_path): """Copy the file from old_path to new_path The paths can be local or on GCS Args: old_path: (string) original file location new_path: (string) new location and name for the file to be copied to Returns: None """ # Make sure the local nested directory already exists: out_dir = ntpath.split(new_path)[0] if not (out_dir.startswith('gs://') and os.path.exists(out_dir)): os.makedirs(out_dir) gfile.Copy(old_path, new_path, overwrite=True)
def embed_data(x, dset, path): """embeds x into the code space using the autoencoder.""" if x: return np.zeros(shape=(0, 10)) # load model and weights json_path = os.path.join(path, 'ae_{}.json'.format(dset)) print('load model from json file:', json_path) with gfile.Open(json_path) as f: pt_ae = model_from_json(f.read()) weights_path = os.path.join(path, 'ae_{}_weights.h5'.format(dset)) print('load code spase from:', weights_path) local_filename = weights_path.split('/')[-1] tmp_filename = os.path.join(tempfile.gettempdir(), str(int(time.time())) + '_' + local_filename) gfile.Copy(weights_path, tmp_filename) pt_ae.load_weights(tmp_filename) gfile.Remove(tmp_filename) print('***********************', x.shape) x = x.reshape(-1, np.prod(x.shape[1:])) print('***********************', x.shape) get_embeddings = K.function([pt_ae.input], [pt_ae.layers[3].output]) get_reconstruction = K.function([pt_ae.layers[4].input], [pt_ae.output]) x_embedded = predict_with_k_fn(get_embeddings, x)[0] x_recon = predict_with_k_fn(get_reconstruction, x_embedded)[0] reconstruction_mse = np.mean(np.square(x - x_recon)) print( 'using pretrained embeddings; sanity check, total reconstruction error:', np.mean(reconstruction_mse)) del pt_ae return x_embedded
def copy_to_experiment_dir(config_file): # copy config file to the experiment directory saved_config_file_path = _config_file_path_to_copy(config_file) gfile.Copy(config_file, saved_config_file_path)
def verbose_copy(src, tgt, overwrite=True): logging.info('Copying "%s" -> "%s"', src, tgt) gfile.Copy(src, tgt, overwrite=overwrite)
def copy_checkpoint(source, target): for ext in (".index", ".data-00000-of-00001"): gfile.Copy(source + ext, target + ext)