def train_and_evaluate(args): INPUT_DIM = args.training_history*ONE_HOUR CLASS_SIZE = len(bins)+1 hidden_units = args.hidden_units # hidden_units = [int(units) for units in args.hidden_units.split(',')] learning_rate = args.learning_rate disk_model = model.model_fn(INPUT_DIM, CLASS_SIZE, hidden_units,learning_rate ) try: os.makedirs(args.job_dir) except: pass # Unhappy hack to workaround h5py not being able to write to GCS. # Force snapshots and saves to local filesystem, then copy them over to GCS. checkpoint_path = CHECKPOINT_FILE_PATH if not args.job_dir.startswith('gs://'): checkpoint_path = os.path.join(args.job_dir, checkpoint_path) # Model checkpoint callback. checkpoint = ModelCheckpoint( checkpoint_path, monitor='val_loss', verbose=1, period=args.checkpoint_epochs, mode='min') # Continuous eval callback. # evaluation = ContinuousEval(args.eval_frequency, args.eval_files, # args.learning_rate, args.job_dir) # Tensorboard logs callback. tb_log = TensorBoard( log_dir=os.path.join(args.job_dir, 'logs'), histogram_freq=0, write_graph=True, embeddings_freq=0) callbacks = [checkpoint,tb_log] history = disk_model.fit_generator( model.generator_input(args.train_files, args.training_history,args.train_batch_size), validation_data=model.generator_input(args.eval_files, args.training_history,args.eval_batch_size), steps_per_epoch=args.train_steps,validation_steps = 10, epochs=args.num_epochs, callbacks=callbacks) # Unhappy hack to workaround h5py not being able to write to GCS. # Force snapshots and saves to local filesystem, then copy them over to GCS. if args.job_dir.startswith('gs://'): disk_model.save(DISK_MODEL) copy_file_to_gcs(args.job_dir, DISK_MODEL) else: disk_model.save(os.path.join(args.job_dir, DISK_MODEL)) with file_io.FileIO( os.path.join(args.job_dir, 'history'), mode='w+') as output_f: pickle.dump(history.history, output_f)
def on_epoch_begin(self, epoch, logs={}): if epoch > 0 and epoch % self.eval_frequency == 0: # Unhappy hack to work around h5py not being able to write to GCS. # Force snapshots and saves to local filesystem, then copy them over to GCS. model_path_glob = 'checkpoint.*' if not self.job_dir.startswith("gs://"): model_path_glob = os.path.join(self.job_dir, model_path_glob) checkpoints = glob.glob(model_path_glob) if len(checkpoints) > 0: checkpoints.sort() CHURN_MODEL = load_model(checkpoints[-1]) CHURN_MODEL = model.compile_model(churn_model, self.learning_rate) loss, acc = churn_model.evaluate_generator( model.generator_input(self.eval_files, chunk_size=CHUNK_SIZE), steps=self.steps) print( '\nEvaluation epoch[{}] metrics[{:.2f}, {:.2f}] {}'.format( epoch, loss, acc, churn_model.metrics_names)) if self.job_dir.startswith("gs://"): copy_file_to_gcs(self.job_dir, checkpoints[-1]) else: print('\nEvaluation epoch[{}] (no checkpoints found)'.format( epoch))
def dispatch(train_files, eval_files, job_dir, train_steps, eval_steps, train_batch_size, eval_batch_size, learning_rate, eval_frequency, first_layer_size, num_layers, scale_factor, eval_num_epochs, num_epochs, checkpoint_epochs): census_model = model.model_fn(INPUT_SIZE, CLASS_SIZE) try: os.makedirs(job_dir) except: pass # Unhappy hack to work around h5py not being able to write to GCS. # Force snapshots and saves to local filesystem, then copy them over to GCS. checkpoint_path = FILE_PATH if not job_dir.startswith("gs://"): checkpoint_path = os.path.join(job_dir, checkpoint_path) # Model checkpoint callback checkpoint = ModelCheckpoint(checkpoint_path, monitor='val_loss', verbose=1, period=checkpoint_epochs, mode='max') # Continuous eval callback evaluation = ContinuousEval(eval_frequency, eval_files, learning_rate, job_dir) # Tensorboard logs callback tblog = TensorBoard(log_dir=os.path.join(job_dir, 'logs'), histogram_freq=0, write_graph=True) callbacks = [checkpoint, evaluation, tblog] census_model.fit_generator(model.generator_input(train_files, chunk_size=CHUNK_SIZE), steps_per_epoch=train_steps, epochs=num_epochs, callbacks=callbacks) # Unhappy hack to work around h5py not being able to write to GCS. # Force snapshots and saves to local filesystem, then copy them over to GCS. if job_dir.startswith("gs://"): census_model.save(CENSUS_MODEL) copy_file_to_gcs(job_dir, CENSUS_MODEL) else: census_model.save(os.path.join(job_dir, CENSUS_MODEL)) # Convert the Keras model to TensorFlow SavedModel if os.path.exists(os.path.join(job_dir, 'export')): shutil.rmtree(os.path.join(job_dir, 'export')) model.to_savedmodel(census_model, os.path.join(job_dir, 'export'))
def train_and_evaluate(args): census_model = model.model_fn(INPUT_SIZE, CLASS_SIZE) try: os.makedirs(args.job_dir) except: pass # Unhappy hack to workaround h5py not being able to write to GCS. # Force snapshots and saves to local filesystem, then copy them over to GCS. checkpoint_path = CHECKPOINT_FILE_PATH if not args.job_dir.startswith('gs://'): checkpoint_path = os.path.join(args.job_dir, checkpoint_path) # Model checkpoint callback. checkpoint = ModelCheckpoint( checkpoint_path, monitor='val_loss', verbose=1, period=args.checkpoint_epochs, mode='min') # Continuous eval callback. evaluation = ContinuousEval(args.eval_frequency, args.eval_files, args.learning_rate, args.job_dir) # Tensorboard logs callback. tb_log = TensorBoard( log_dir=os.path.join(args.job_dir, 'logs'), histogram_freq=0, write_graph=True, embeddings_freq=0) callbacks = [checkpoint, evaluation, tb_log] census_model.fit_generator( model.generator_input(args.train_files, chunk_size=CHUNK_SIZE), steps_per_epoch=args.train_steps, epochs=args.num_epochs, callbacks=callbacks) # Unhappy hack to workaround h5py not being able to write to GCS. # Force snapshots and saves to local filesystem, then copy them over to GCS. if args.job_dir.startswith('gs://'): census_model.save(CENSUS_MODEL) copy_file_to_gcs(args.job_dir, CENSUS_MODEL) else: census_model.save(os.path.join(args.job_dir, CENSUS_MODEL)) # Convert the Keras model to TensorFlow SavedModel. model.to_savedmodel(census_model, os.path.join(args.job_dir, 'export'))
def train_and_evaluate(hparams): census_model = model.model_fn(INPUT_SIZE, CLASS_SIZE) try: os.makedirs(hparams.job_dir) except: pass # Unhappy hack to workaround h5py not being able to write to GCS. # Force snapshots and saves to local filesystem, then copy them over to GCS. checkpoint_path = CHECKPOINT_FILE_PATH if not hparams.job_dir.startswith('gs://'): checkpoint_path = os.path.join(hparams.job_dir, checkpoint_path) # Model checkpoint callback. checkpoint = ModelCheckpoint(checkpoint_path, monitor='val_loss', verbose=1, period=hparams.checkpoint_epochs, mode='min') # Continuous eval callback. evaluation = ContinuousEval(hparams.eval_frequency, hparams.eval_files, hparams.learning_rate, hparams.job_dir) # Tensorboard logs callback. tb_log = TensorBoard(log_dir=os.path.join(hparams.job_dir, 'logs'), histogram_freq=0, write_graph=True, embeddings_freq=0) callbacks = [checkpoint, evaluation, tb_log] census_model.fit_generator(model.generator_input(hparams.train_files, chunk_size=CHUNK_SIZE), steps_per_epoch=hparams.train_steps, epochs=hparams.num_epochs, callbacks=callbacks) # Unhappy hack to workaround h5py not being able to write to GCS. # Force snapshots and saves to local filesystem, then copy them over to GCS. if hparams.job_dir.startswith('gs://'): census_model.save(CENSUS_MODEL) copy_file_to_gcs(hparams.job_dir, CENSUS_MODEL) else: census_model.save(os.path.join(hparams.job_dir, CENSUS_MODEL)) # Convert the Keras model to TensorFlow SavedModel. model.to_savedmodel(census_model, os.path.join(hparams.job_dir, 'export'))
def train_and_evaluate(args): logic_nn_model = model.model_fn(**vars(args)) try: os.makedirs(args.job_dir) except: pass # Unhappy hack to workaround h5py not being able to write to GCS. # Force snapshots and saves to local filesystem, then copy them over to GCS. checkpoint_path = CHECKPOINT_FILE_PATH if not args.job_dir.startswith('gs://'): checkpoint_path = os.path.join(args.job_dir, checkpoint_path) # Model checkpoint callback. checkpoint = ModelCheckpoint(checkpoint_path, monitor='val_loss', verbose=1, period=args.checkpoint_epochs, mode='min') # Continuous eval callback. evaluation = ContinuousEval(args.eval_frequency, args.learning_rate, args.job_dir, args.eval_steps) # Tensorboard logs callback. tb_log = TensorBoard(log_dir=os.path.join(args.job_dir, 'logs'), histogram_freq=0, write_graph=True, embeddings_freq=0) callbacks = [checkpoint, evaluation, tb_log] logic_nn_model.fit_generator(model.generator_input(), steps_per_epoch=args.train_steps, epochs=args.num_epochs, callbacks=callbacks) # Unhappy hack to workaround h5py not being able to write to GCS. # Force snapshots and saves to local filesystem, then copy them over to GCS. if args.job_dir.startswith('gs://'): logic_nn_model.save(LOGICAL_NN_MODEL) copy_file_to_gcs(args.job_dir, LOGICAL_NN_MODEL) else: logic_nn_model.save(os.path.join(args.job_dir, LOGICAL_NN_MODEL)) # Convert the Keras model to TensorFlow SavedModel. model.to_saved_model(logic_nn_model, os.path.join(args.job_dir, 'export')) print("...Finished actions for build and train with model...")
def on_epoch_begin(self, epoch, logs={}): """Compile and save model.""" if epoch > 0 and epoch % self.eval_frequency == 0: # Unhappy hack to work around h5py not being able to write to GCS. # Force snapshots and saves to local filesystem, then copy them over to GCS. model_path_glob = 'checkpoint.*' if not self.job_dir.startswith('gs://'): model_path_glob = os.path.join(self.job_dir, model_path_glob) checkpoints = glob.glob(model_path_glob) if len(checkpoints) > 0: checkpoints.sort() census_model = load_model(checkpoints[-1]) census_model = model.compile_model(census_model, self.learning_rate) loss, acc = census_model.evaluate_generator( model.generator_input(self.eval_files, chunk_size=CHUNK_SIZE), steps=self.steps) print('\nEvaluation epoch[{}] metrics[{:.2f}, {:.2f}] {}'.format( epoch, loss, acc, census_model.metrics_names)) if self.job_dir.startswith('gs://'): copy_file_to_gcs(self.job_dir, checkpoints[-1]) else: print('\nEvaluation epoch[{}] (no checkpoints found)'.format(epoch))
def train_and_evaluate(args): # confirm whether training datasets need to be created if args.create_data == True: import trainer.create_data_func as create_data_func logging.info('Begin creating datasets') for data_part in ['train', 'val', 'test']: create_data_func.create_data_func(data_part, args.project_id, args.bucket_name, args.dataset_id) logging.info('End creating datasets') # Create config file and store project id there so that model.py can read it. with open('config.py', 'w') as f: f.write("PROJECT_ID=\"{}\"\n".format(args.project_id)) f.write("BUCKET_NAME=\"{}\"\n".format(args.bucket_name)) # import after datasets are created as they are referenced immediately when this module is initiated import trainer.model as model # if new datasets are created, scaler also need to be created if args.create_data == True: import trainer.create_scaler_func as create_scaler_func logging.info('Begin fitting scaler') create_scaler_func.create_scaler_func(args.train_files, model.CSV_COLUMNS, model.LABEL_COLUMN, args.bucket_name, args.project_id) logging.info('End fitting scalers') # download the scaler if not path.exists('x_scaler'): logging.info('Downloading scaler') storage_client = storage.Client(project=args.project_id) bucket = storage_client.get_bucket(args.bucket_name) blob = bucket.blob('scalers/x_scaler') blob.download_to_filename('x_scaler') logging.info('Downloaded scaler') x_scaler = joblib.load('x_scaler') # build the model census_model = model.model_fn( learning_rate=args.learning_rate, num_deep_layers=args.num_deep_layers, first_deep_layer_size=args.first_deep_layer_size, first_wide_layer_size=args.first_wide_layer_size, wide_scale_factor=args.wide_scale_factor, dropout_rate=args.dropout_rate) logging.info(census_model.summary()) try: os.makedirs(args.job_dir) except: pass checkpoint_path = os.path.join(args.job_dir, CHECKPOINT_FILE_PATH) # Model checkpoint callback. checkpoint = ModelCheckpoint( checkpoint_path, monitor='val_mse', # 'mean_squared_error' verbose=1, period=args.checkpoint_epochs, save_best_only=True, mode='min') # Early stopping callback. early_stop = EarlyStopping(monitor='val_mse', patience=10) # 'mean_squared_error' # Tensorboard logs callback. tb_log = TensorBoard(log_dir=os.path.join(args.job_dir, 'logs'), histogram_freq=0, write_graph=True, embeddings_freq=0) callbacks = [checkpoint, early_stop, tb_log] # fit the model on the training set census_model.fit_generator( generator=model.generator_input(args.train_files, chunk_size=CHUNK_SIZE, project_id=args.project_id, bucket_name=args.bucket_name, x_scaler=x_scaler), steps_per_epoch=args.train_steps, epochs=args.num_epochs, callbacks=callbacks, validation_data=model.generator_input(args.eval_files, chunk_size=CHUNK_SIZE, project_id=args.project_id, bucket_name=args.bucket_name, x_scaler=x_scaler), validation_steps=args.eval_steps) # evaluate model on test set loss, mae, mse = census_model.evaluate_generator(model.generator_input( args.test_files, chunk_size=CHUNK_SIZE, project_id=args.project_id, bucket_name=args.bucket_name, x_scaler=x_scaler), steps=args.test_steps) logging.info('\nTest evaluation metrics[{:.2f}, {:.2f}, {:.2f}] {}'.format( loss, mae, mse, census_model.metrics_names)) # Unhappy hack to workaround h5py not being able to write to GCS. # Force snapshots and saves to local filesystem, then copy them over to GCS. if args.job_dir.startswith('gs://'): census_model.save(CENSUS_MODEL) copy_file_to_gcs(args.job_dir, CENSUS_MODEL) else: census_model.save(os.path.join(args.job_dir, CENSUS_MODEL)) # Convert the Keras model to TensorFlow SavedModel. model.to_savedmodel(census_model, os.path.join(args.job_dir, 'export'))
# You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== """Pre-processing script to generate a sample for SavedModel prediction.""" import json import sys from trainer import model if __name__=='__main__': gen = model.generator_input(['data/output_test.csv'], chunk_size=5000) sample = gen.next() input_sample = {} input_sample['input'] = sample[0].values[0].tolist() print('Produced sample with label {}'.format(sample[1].values[0].tolist())) with open(sys.argv[1], 'w') as outfile: json.dump(input_sample, outfile)
def train_and_evaluate(args): CLASS_SIZE = len(bins) + 1 # hidden_units = [int(units) for units in args.hidden_units.split(',')] try: os.makedirs(args.job_dir) except: pass # Unhappy hack to workaround h5py not being able to write to GCS. # Force snapshots and saves to local filesystem, then copy them over to GCS. checkpoint_path = CHECKPOINT_FILE_PATH if not args.job_dir.startswith('gs://'): checkpoint_path = os.path.join(args.job_dir, checkpoint_path) # Model checkpoint callback. checkpoint = ModelCheckpoint(checkpoint_path, monitor='val_loss', verbose=1, period=args.checkpoint_epochs, mode='min') tb_log = TensorBoard(log_dir=os.path.join(args.job_dir, 'logs'), histogram_freq=0, write_graph=True, embeddings_freq=0) callbacks = [checkpoint, tb_log] sequential_train = [ int(float(hour)) for hour in args.sequential_train.split(',') ] seq_id = 0 sequential_models = [] weights_all = [] for hours in sequential_train: history_all = [] if seq_id == 0: hidden_units = args.hidden_units learning_rate = args.learning_rate INPUT_DIM = hours * ONE_HOUR ###########fully connected model############# first_model = initial_model.model_fn(INPUT_DIM, CLASS_SIZE, hidden_units, 0.0001) assign_w = 0.03 first_model.compile(loss=initial_model.weighted_loss(assign_w), optimizer=keras.optimizers.Adam(lr=0.0001), metrics=[ initial_model.first_class_accuracy, initial_model.other_class_accuracy, initial_model.single_class_accuracy(1), initial_model.single_class_accuracy(2), initial_model.single_class_accuracy(3), initial_model.single_class_accuracy(4), initial_model.single_class_accuracy(5), initial_model.single_class_accuracy(6), initial_model.single_class_accuracy(7), 'accuracy' ]) train_file_names = os.path.join(args.train_files, str(hours) + 'hrs', 'train', '*npz') eval_file_names = os.path.join(args.eval_files, str(hours) + 'hrs', 'eval', '*npz') print("\n\ntraining " + str(hours) + 'hrs!\n\n') history_all.append( first_model.fit_generator( initial_model.generator_input(train_file_names, args.train_batch_size), validation_data=initial_model.generator_input( eval_file_names, args.eval_batch_size), steps_per_epoch=args.train_steps, validation_steps=args.eval_steps, epochs=args.num_epochs, callbacks=callbacks)) weights = first_model.get_weights() weights_all.append(weights) with open( os.path.join(args.job_dir, 'weights', str(hours) + 'hrs_weights'), 'wb') as fp: pickle.dump(weights, fp) DISK_MODEL = 'disk_model.hdf5' if args.job_dir.startswith('gs://'): first_model.save(DISK_MODEL) copy_file_to_gcs(args.job_dir, DISK_MODEL) else: first_model.save(os.path.join(args.job_dir, DISK_MODEL)) data, label = initial_model.generator_input_once( str(args.train_files) + str(hours) + 'hrs/train/input_' + str(hours) + 'hrs_8.npz', 3) first_model.compile(loss=initial_model.weighted_loss(0.00081), optimizer=keras.optimizers.Adam(lr=0.0001), metrics=[ initial_model.first_class_accuracy, initial_model.other_class_accuracy, initial_model.single_class_accuracy(1), initial_model.single_class_accuracy(2), initial_model.single_class_accuracy(3), initial_model.single_class_accuracy(4), initial_model.single_class_accuracy(5), initial_model.single_class_accuracy(6), initial_model.single_class_accuracy(7), 'accuracy' ]) first_model.fit_generator( initial_model.generator_input(train_file_names, args.train_batch_size), validation_data=initial_model.generator_input( eval_file_names, args.eval_batch_size), steps_per_epoch=args.train_steps, validation_steps=args.eval_steps, epochs=50, callbacks=callbacks) scores = first_model.evaluate(x=data, y=label, batch_size=None, verbose=1, sample_weight=None, steps=1) print("\ntest " + str(hours) + 'hrs after train\n') print(scores) seq_id = seq_id + 1 else: with open( os.path.join( args.job_dir, 'weights', str(sequential_train[seq_id - 1]) + 'hrs_weights'), 'rb') as fp: weights_0 = pickle.load(fp) ######sequential(weights, CONCAT_UNIT_SIZE, INPUT_SHAPE, learning_rate) hours = sequential_train[seq_id] seq = Sequential(weights_0, args.CONCAT_UNIT_SIZE, hours * ONE_HOUR, 'zeros') model = seq.build_sequential_model() # assign_w = 0.016+0.005*seq_id assign_w = 0.03 model.compile(loss=initial_model.weighted_loss(assign_w), optimizer=keras.optimizers.Adam(lr=0.0001), metrics=[ initial_model.first_class_accuracy, initial_model.other_class_accuracy, initial_model.single_class_accuracy(1), initial_model.single_class_accuracy(2), initial_model.single_class_accuracy(3), initial_model.single_class_accuracy(4), initial_model.single_class_accuracy(5), initial_model.single_class_accuracy(6), initial_model.single_class_accuracy(7), 'accuracy' ]) data, label = initial_model.generator_input_once( str(args.train_files) + str(hours) + 'hrs/train/input_' + str(hours) + 'hrs_8.npz', 6) scores = model.evaluate(x=data, y=label, batch_size=None, verbose=1, sample_weight=None, steps=1) print("\ntest " + str(hours) + 'hrs beofre train\n') print(scores) # data,label = initial_model.generator_input_once('/Volumes/TOSHIBA EXT/train_input/24hrs/train/input_24hrs_8.npz', 24) # # scores = model.evaluate(x=data, y=label, batch_size=None, verbose=1, sample_weight=None, steps=1) # print(scores) ###########sequential model############# train_file_names = os.path.join(str(args.train_files), str(hours) + 'hrs', 'train', '*npz') eval_file_names = os.path.join(args.eval_files, str(hours) + 'hrs', 'eval', '*npz') print("\n\ntraining " + str(hours) + 'hrs!\n\n') history_all.append( model.fit_generator( initial_model.generator_input(train_file_names, args.train_batch_size), validation_data=initial_model.generator_input( eval_file_names, args.eval_batch_size), steps_per_epoch=args.train_steps, validation_steps=args.eval_steps, epochs=args.num_epochs, callbacks=callbacks)) weights = model.get_weights() weights_all.append(weights) weights_0 = [] for i in range(int(len(weights) / 4)): if i == int(len(weights) / 4) - 1: weights_0.extend([ np.concatenate((weights[i * 4 + 2], weights[i * 4]), axis=0), (weights[i * 4 + 1] + weights[i * 4 + 3]) ]) elif i == int(len(weights) / 4) - 2: weights_0.extend([ np.concatenate((weights[i * 4 + 2], weights[i * 4]), axis=1), np.concatenate( (weights[i * 4 + 3], weights[i * 4 + 1])) ]) else: weights_0.extend([ np.concatenate((weights[i * 4], weights[i * 4 + 2]), axis=1), np.concatenate( (weights[i * 4 + 1], weights[i * 4 + 3])) ]) # weights_0 = [np.concatenate((weights[0], weights[2]), axis=1), # np.concatenate((weights[1], weights[3])), # np.concatenate((weights[4], weights[6]), axis=1), # np.concatenate((weights[5], weights[7])), # np.concatenate((weights[8], weights[10]), axis=0) # weights[9] + weights[11]] data, label = initial_model.generator_input_once( str(args.train_files) + str(hours) + 'hrs/train/input_' + str(hours) + 'hrs_8.npz', 6) scores = model.evaluate(x=data, y=label, batch_size=None, verbose=1, sample_weight=None, steps=1) print("\ntest " + str(hours) + 'hrs after train\n') print(scores) with open( os.path.join(str(args.job_dir), 'weights', str(hours) + 'hrs_weights'), 'wb') as fp: pickle.dump(weights_0, fp) DISK_MODEL = 'disk_model' + str(hours) + '.hdf5' if args.job_dir.startswith('gs://'): model.save(DISK_MODEL) copy_file_to_gcs(args.job_dir, DISK_MODEL) else: model.save(os.path.join(args.job_dir, DISK_MODEL)) seq_id = seq_id + 1 with open(os.path.join(str(args.job_dir), 'histroy_all'), 'wb') as fp: pickle.dump(history_all, fp)
from google.oauth2 import service_account from googleapiclient import discovery from trainer import model PATH_CREDENTIALS_GOOGLE = os.environ.get('GOOGLE_APPLICATION_CREDENTIALS', None) credentials = service_account.Credentials.from_service_account_file( PATH_CREDENTIALS_GOOGLE) api = discovery.build('ml', 'v1', credentials=credentials) parent = 'projects/%s/models/%s/versions/%s' % ('academic-motif-193414', 'neural_keras_model', 'v1') data = model.generator_input(batch_size=1) for i in range(0, 30): flag = True while flag: sample = next(data) flag = True if random() > 0.5 else False sample = next(data) input = sample[0][0].tolist() request_data = {'instances': [input]} response = api.projects().predict(body=request_data, name=parent).execute() result = response['predictions'][0]['income'][0] print("{:.3g} value predicted for the sample {}, binary output {}".format( result, input, round(result)))
default='ross-keras') args, _ = parser.parse_known_args() # download the scaler if not path.exists('x_scaler'): print('Downloading scaler') storage_client = storage.Client(project='mwpmltr') bucket = storage_client.get_bucket('ross-keras') blob = bucket.blob('scalers/x_scaler') blob.download_to_filename('x_scaler') print('Downloaded scaler') x_scaler = joblib.load('x_scaler') gen = model.generator_input(['gs://ross-keras/data/full_test_results.csv'], chunk_size=5000, project_id=args.project_id, bucket_name=args.bucket_name, x_scaler=x_scaler, batch_size=1) actuals = [] preds = [] for i in range(0, args.num_samples): sample = gen.__next__() input_sample = {} input_sample['input'] = sample[0] actual = int(round(np.exp(sample[1]))) actuals.append(actual) with open('input_sample.json', 'w') as outfile:
# You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== """Pre-processing script to generate a sample for SavedModel prediction.""" import json import sys from trainer import model if __name__=='__main__': gen = model.generator_input(['adult.data.csv'], chunk_size=5000) sample = gen.__next__() input_sample = {} input_sample['input'] = sample[0].values[0].tolist() print('Produced sample with label {}'.format(sample[1].values[0].tolist())) with open(sys.argv[1], 'w') as outfile: json.dump(input_sample, outfile)
"""Pre-processing script to generate a sample for SavedModel prediction.""" import json import sys from trainer import model TEST_CSV_PATH = "data/adult.data.csv" if __name__ == '__main__': gen = model.generator_input([TEST_CSV_PATH], chunk_size=5000) sample = gen.next() input_sample = {} input_sample['input'] = sample[0].values[0].tolist() print('Produced sample with label {}'.format(sample[1].values[0].tolist())) with open(sys.argv[1], 'w') as outfile: json.dump(input_sample, outfile)
'--bucket-name', type=str, default='sanofi-ml-workshop-chicago-taxi-demo', help='The Cloud Storage bucket to be used for process artifacts') args, _ = parser.parse_known_args() # download the scaler if not path.exists('x_scaler'): print('Downloading scaler') storage_client = storage.Client(project=args.project_id) bucket = storage_client.get_bucket(args.bucket_name) blob = bucket.blob('scalers/x_scaler') blob.download_to_filename('x_scaler') print('Downloaded scaler') x_scaler = joblib.load('x_scaler') gen = model.generator_input(['gs://{}/data/full_test_results.csv'.format(args.bucket_name)], chunk_size=5000, project_id=args.bucket_name, bucket_name=args.bucket_name, x_scaler=x_scaler, batch_size=1) for i in range(1, random.randint(1,100)): sample = gen.__next__() input_sample = {} input_sample['input'] = sample[0] print('Produced sample with label {} seconds.'.format(str(int(round(np.exp(sample[1])))))) with open('input_sample.json', 'w') as outfile: json.dump(input_sample, outfile, cls=NumpyEncoder)
# you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== """Pre-processing script to generate a sample for SavedModel prediction.""" import json import sys from trainer import model if __name__ == '__main__': gen = model.generator_input(['adult.data.csv'], chunk_size=500) sample = gen.__next__() sample = gen.__next__() input_sample = {} input_sample['input'] = sample[0].values[0].tolist() print('Produced sample with label {}'.format(sample[1].values[0].tolist())) with open(sys.argv[1], 'w') as outfile: json.dump(input_sample, outfile)
def train_and_evaluate(args): # Showcasing the hypertuning parameters here. # The first-layer-size is being tuned in this example hidden_units = [args.first_layer_size, 70, 50, 20] census_model = model.model_fn(INPUT_SIZE, CLASS_SIZE, hidden_units) try: os.makedirs(args.job_dir) except: pass # Unhappy hack to workaround h5py not being able to write to GCS. # Force snapshots and saves to local filesystem, then copy them over to GCS. checkpoint_path = CHECKPOINT_FILE_PATH if not args.job_dir.startswith('gs://'): checkpoint_path = os.path.join(args.job_dir, checkpoint_path) # Model checkpoint callback. checkpoint = ModelCheckpoint( checkpoint_path, monitor='val_loss', verbose=1, save_best_only=False, period=args.checkpoint_epochs, mode='min') # Continuous eval callback. evaluation = ContinuousEval(args.eval_frequency, args.eval_files, args.learning_rate, args.job_dir) # Tensorboard logs callback. tb_log = TensorBoard( log_dir=os.path.join(args.job_dir, 'logs'), histogram_freq=0, write_graph=True, embeddings_freq=0) callbacks = [checkpoint, evaluation, tb_log] census_model.fit_generator( model.generator_input(args.train_files, chunk_size=CHUNK_SIZE), steps_per_epoch=args.train_steps, epochs=args.num_epochs, use_multiprocessing=args.distributed, callbacks=callbacks) # Unhappy hack to workaround h5py not being able to write to GCS. # Force snapshots and saves to local filesystem, then copy them over to GCS. if args.job_dir.startswith('gs://'): census_model.save(CENSUS_MODEL) copy_file_to_gcs(args.job_dir, CENSUS_MODEL) else: census_model.save(os.path.join(args.job_dir, CENSUS_MODEL)) # Convert the Keras model to TensorFlow SavedModel. model.to_savedmodel(census_model, os.path.join(args.job_dir, 'export')) # The following is for hyperparameter tuning and is adapted from here: https://cloud.google.com/ml-engine/docs/tensorflow/using-hyperparameter-tuning # Note: the last_loss_val is updated after each checkpoint, but we only write the summary once. summary = Summary(value=[Summary.Value(tag='val_loss', simple_value=evaluation.last_loss_val)]) # more hypertune info here: https://cloud.google.com/solutions/machine-learning/recommendation-system-tensorflow-train-cloud-ml-engine job_dir = args.job_dir if args.hypertune: # if tuning, join the trial number to the output path trial = json.loads(os.environ.get('TF_CONFIG', '{}')).get('task', {}).get('trial', '') output_dir = os.path.join(job_dir, trial) else: output_dir = job_dir eval_path = os.path.join(output_dir, 'val_loss') summary_writer = tf.summary.FileWriter(eval_path) # Note: adding the summary to the writer is enough for hyperparameter tuning. # ML Engine looks for any summary added with the hyperparameter metric tag. summary_writer.add_summary(summary) summary_writer.flush()
if __name__ == '__main__': # download the scaler if not path.exists('x_scaler'): print('Downloading scaler') storage_client = storage.Client(project='mwpmltr') bucket = storage_client.get_bucket('ross-keras') blob = bucket.blob('scalers/x_scaler') blob.download_to_filename('x_scaler') print('Downloaded scaler') x_scaler = joblib.load('x_scaler') gen = model.generator_input(['gs://ross-keras/data/full_test_results.csv'], chunk_size=5000, project_id=PROJECT_ID, bucket_name=BUCKET_NAME, x_scaler=x_scaler, batch_size=1) for i in range(1, random.randint(1, 100)): sample = gen.__next__() input_sample = {} input_sample['input'] = sample[0] print('Produced sample with label {} seconds.'.format( str(int(round(np.exp(sample[1])))))) with open('input_sample.json', 'w') as outfile: json.dump(input_sample, outfile, cls=NumpyEncoder)