def train_and_evaluate(args):
INPUT_DIM = args.training_history*ONE_HOUR
CLASS_SIZE = len(bins)+1
hidden_units = args.hidden_units
# hidden_units = [int(units) for units in args.hidden_units.split(',')]
learning_rate = args.learning_rate
disk_model = model.model_fn(INPUT_DIM, CLASS_SIZE,
hidden_units,learning_rate
)
try:
os.makedirs(args.job_dir)
except:
pass
# Unhappy hack to workaround h5py not being able to write to GCS.
# Force snapshots and saves to local filesystem, then copy them over to GCS.
checkpoint_path = CHECKPOINT_FILE_PATH
if not args.job_dir.startswith('gs://'):
checkpoint_path = os.path.join(args.job_dir, checkpoint_path)
# Model checkpoint callback.
checkpoint = ModelCheckpoint(
checkpoint_path,
monitor='val_loss',
verbose=1,
period=args.checkpoint_epochs,
mode='min')
# Continuous eval callback.
# evaluation = ContinuousEval(args.eval_frequency, args.eval_files,
# args.learning_rate, args.job_dir)
# Tensorboard logs callback.
tb_log = TensorBoard(
log_dir=os.path.join(args.job_dir, 'logs'),
histogram_freq=0,
write_graph=True,
embeddings_freq=0)
callbacks = [checkpoint,tb_log]
history = disk_model.fit_generator(
model.generator_input(args.train_files, args.training_history,args.train_batch_size),
validation_data=model.generator_input(args.eval_files, args.training_history,args.eval_batch_size),
steps_per_epoch=args.train_steps,validation_steps = 10,
epochs=args.num_epochs,
callbacks=callbacks)
# Unhappy hack to workaround h5py not being able to write to GCS.
# Force snapshots and saves to local filesystem, then copy them over to GCS.
if args.job_dir.startswith('gs://'):
disk_model.save(DISK_MODEL)
copy_file_to_gcs(args.job_dir, DISK_MODEL)
else:
disk_model.save(os.path.join(args.job_dir, DISK_MODEL))
with file_io.FileIO(
os.path.join(args.job_dir, 'history'), mode='w+') as output_f:
pickle.dump(history.history, output_f)
def on_epoch_begin(self, epoch, logs={}):
if epoch > 0 and epoch % self.eval_frequency == 0:
# Unhappy hack to work around h5py not being able to write to GCS.
# Force snapshots and saves to local filesystem, then copy them over to GCS.
model_path_glob = 'checkpoint.*'
if not self.job_dir.startswith("gs://"):
model_path_glob = os.path.join(self.job_dir, model_path_glob)
checkpoints = glob.glob(model_path_glob)
if len(checkpoints) > 0:
checkpoints.sort()
CHURN_MODEL = load_model(checkpoints[-1])
CHURN_MODEL = model.compile_model(churn_model,
self.learning_rate)
loss, acc = churn_model.evaluate_generator(
model.generator_input(self.eval_files,
chunk_size=CHUNK_SIZE),
steps=self.steps)
print(
'\nEvaluation epoch[{}] metrics[{:.2f}, {:.2f}] {}'.format(
epoch, loss, acc, churn_model.metrics_names))
if self.job_dir.startswith("gs://"):
copy_file_to_gcs(self.job_dir, checkpoints[-1])
else:
print('\nEvaluation epoch[{}] (no checkpoints found)'.format(
epoch))
def dispatch(train_files, eval_files, job_dir, train_steps, eval_steps,
train_batch_size, eval_batch_size, learning_rate, eval_frequency,
first_layer_size, num_layers, scale_factor, eval_num_epochs,
num_epochs, checkpoint_epochs):
census_model = model.model_fn(INPUT_SIZE, CLASS_SIZE)
try:
os.makedirs(job_dir)
except:
pass
# Unhappy hack to work around h5py not being able to write to GCS.
# Force snapshots and saves to local filesystem, then copy them over to GCS.
checkpoint_path = FILE_PATH
if not job_dir.startswith("gs://"):
checkpoint_path = os.path.join(job_dir, checkpoint_path)
# Model checkpoint callback
checkpoint = ModelCheckpoint(checkpoint_path,
monitor='val_loss',
verbose=1,
period=checkpoint_epochs,
mode='max')
# Continuous eval callback
evaluation = ContinuousEval(eval_frequency, eval_files, learning_rate,
job_dir)
# Tensorboard logs callback
tblog = TensorBoard(log_dir=os.path.join(job_dir, 'logs'),
histogram_freq=0,
write_graph=True)
callbacks = [checkpoint, evaluation, tblog]
census_model.fit_generator(model.generator_input(train_files,
chunk_size=CHUNK_SIZE),
steps_per_epoch=train_steps,
epochs=num_epochs,
callbacks=callbacks)
# Unhappy hack to work around h5py not being able to write to GCS.
# Force snapshots and saves to local filesystem, then copy them over to GCS.
if job_dir.startswith("gs://"):
census_model.save(CENSUS_MODEL)
copy_file_to_gcs(job_dir, CENSUS_MODEL)
else:
census_model.save(os.path.join(job_dir, CENSUS_MODEL))
# Convert the Keras model to TensorFlow SavedModel
if os.path.exists(os.path.join(job_dir, 'export')):
shutil.rmtree(os.path.join(job_dir, 'export'))
model.to_savedmodel(census_model, os.path.join(job_dir, 'export'))
def train_and_evaluate(args):
census_model = model.model_fn(INPUT_SIZE, CLASS_SIZE)
try:
os.makedirs(args.job_dir)
except:
pass
# Unhappy hack to workaround h5py not being able to write to GCS.
# Force snapshots and saves to local filesystem, then copy them over to GCS.
checkpoint_path = CHECKPOINT_FILE_PATH
if not args.job_dir.startswith('gs://'):
checkpoint_path = os.path.join(args.job_dir, checkpoint_path)
# Model checkpoint callback.
checkpoint = ModelCheckpoint(
checkpoint_path,
monitor='val_loss',
verbose=1,
period=args.checkpoint_epochs,
mode='min')
# Continuous eval callback.
evaluation = ContinuousEval(args.eval_frequency, args.eval_files,
args.learning_rate, args.job_dir)
# Tensorboard logs callback.
tb_log = TensorBoard(
log_dir=os.path.join(args.job_dir, 'logs'),
histogram_freq=0,
write_graph=True,
embeddings_freq=0)
callbacks = [checkpoint, evaluation, tb_log]
census_model.fit_generator(
model.generator_input(args.train_files, chunk_size=CHUNK_SIZE),
steps_per_epoch=args.train_steps,
epochs=args.num_epochs,
callbacks=callbacks)
# Unhappy hack to workaround h5py not being able to write to GCS.
# Force snapshots and saves to local filesystem, then copy them over to GCS.
if args.job_dir.startswith('gs://'):
census_model.save(CENSUS_MODEL)
copy_file_to_gcs(args.job_dir, CENSUS_MODEL)
else:
census_model.save(os.path.join(args.job_dir, CENSUS_MODEL))
# Convert the Keras model to TensorFlow SavedModel.
model.to_savedmodel(census_model, os.path.join(args.job_dir, 'export'))
def train_and_evaluate(hparams):
census_model = model.model_fn(INPUT_SIZE, CLASS_SIZE)
try:
os.makedirs(hparams.job_dir)
except:
pass
# Unhappy hack to workaround h5py not being able to write to GCS.
# Force snapshots and saves to local filesystem, then copy them over to GCS.
checkpoint_path = CHECKPOINT_FILE_PATH
if not hparams.job_dir.startswith('gs://'):
checkpoint_path = os.path.join(hparams.job_dir, checkpoint_path)
# Model checkpoint callback.
checkpoint = ModelCheckpoint(checkpoint_path,
monitor='val_loss',
verbose=1,
period=hparams.checkpoint_epochs,
mode='min')
# Continuous eval callback.
evaluation = ContinuousEval(hparams.eval_frequency, hparams.eval_files,
hparams.learning_rate, hparams.job_dir)
# Tensorboard logs callback.
tb_log = TensorBoard(log_dir=os.path.join(hparams.job_dir, 'logs'),
histogram_freq=0,
write_graph=True,
embeddings_freq=0)
callbacks = [checkpoint, evaluation, tb_log]
census_model.fit_generator(model.generator_input(hparams.train_files,
chunk_size=CHUNK_SIZE),
steps_per_epoch=hparams.train_steps,
epochs=hparams.num_epochs,
callbacks=callbacks)
# Unhappy hack to workaround h5py not being able to write to GCS.
# Force snapshots and saves to local filesystem, then copy them over to GCS.
if hparams.job_dir.startswith('gs://'):
census_model.save(CENSUS_MODEL)
copy_file_to_gcs(hparams.job_dir, CENSUS_MODEL)
else:
census_model.save(os.path.join(hparams.job_dir, CENSUS_MODEL))
# Convert the Keras model to TensorFlow SavedModel.
model.to_savedmodel(census_model, os.path.join(hparams.job_dir, 'export'))
def train_and_evaluate(args):
logic_nn_model = model.model_fn(**vars(args))
try:
os.makedirs(args.job_dir)
except:
pass
# Unhappy hack to workaround h5py not being able to write to GCS.
# Force snapshots and saves to local filesystem, then copy them over to GCS.
checkpoint_path = CHECKPOINT_FILE_PATH
if not args.job_dir.startswith('gs://'):
checkpoint_path = os.path.join(args.job_dir, checkpoint_path)
# Model checkpoint callback.
checkpoint = ModelCheckpoint(checkpoint_path,
monitor='val_loss',
verbose=1,
period=args.checkpoint_epochs,
mode='min')
# Continuous eval callback.
evaluation = ContinuousEval(args.eval_frequency, args.learning_rate,
args.job_dir, args.eval_steps)
# Tensorboard logs callback.
tb_log = TensorBoard(log_dir=os.path.join(args.job_dir, 'logs'),
histogram_freq=0,
write_graph=True,
embeddings_freq=0)
callbacks = [checkpoint, evaluation, tb_log]
logic_nn_model.fit_generator(model.generator_input(),
steps_per_epoch=args.train_steps,
epochs=args.num_epochs,
callbacks=callbacks)
# Unhappy hack to workaround h5py not being able to write to GCS.
# Force snapshots and saves to local filesystem, then copy them over to GCS.
if args.job_dir.startswith('gs://'):
logic_nn_model.save(LOGICAL_NN_MODEL)
copy_file_to_gcs(args.job_dir, LOGICAL_NN_MODEL)
else:
logic_nn_model.save(os.path.join(args.job_dir, LOGICAL_NN_MODEL))
# Convert the Keras model to TensorFlow SavedModel.
model.to_saved_model(logic_nn_model, os.path.join(args.job_dir, 'export'))
print("...Finished actions for build and train with model...")
def on_epoch_begin(self, epoch, logs={}):
"""Compile and save model."""
if epoch > 0 and epoch % self.eval_frequency == 0:
# Unhappy hack to work around h5py not being able to write to GCS.
# Force snapshots and saves to local filesystem, then copy them over to GCS.
model_path_glob = 'checkpoint.*'
if not self.job_dir.startswith('gs://'):
model_path_glob = os.path.join(self.job_dir, model_path_glob)
checkpoints = glob.glob(model_path_glob)
if len(checkpoints) > 0:
checkpoints.sort()
census_model = load_model(checkpoints[-1])
census_model = model.compile_model(census_model, self.learning_rate)
loss, acc = census_model.evaluate_generator(
model.generator_input(self.eval_files, chunk_size=CHUNK_SIZE),
steps=self.steps)
print('\nEvaluation epoch[{}] metrics[{:.2f}, {:.2f}] {}'.format(
epoch, loss, acc, census_model.metrics_names))
if self.job_dir.startswith('gs://'):
copy_file_to_gcs(self.job_dir, checkpoints[-1])
else:
print('\nEvaluation epoch[{}] (no checkpoints found)'.format(epoch))
def train_and_evaluate(args):
# confirm whether training datasets need to be created
if args.create_data == True:
import trainer.create_data_func as create_data_func
logging.info('Begin creating datasets')
for data_part in ['train', 'val', 'test']:
create_data_func.create_data_func(data_part, args.project_id,
args.bucket_name,
args.dataset_id)
logging.info('End creating datasets')
# Create config file and store project id there so that model.py can read it.
with open('config.py', 'w') as f:
f.write("PROJECT_ID=\"{}\"\n".format(args.project_id))
f.write("BUCKET_NAME=\"{}\"\n".format(args.bucket_name))
# import after datasets are created as they are referenced immediately when this module is initiated
import trainer.model as model
# if new datasets are created, scaler also need to be created
if args.create_data == True:
import trainer.create_scaler_func as create_scaler_func
logging.info('Begin fitting scaler')
create_scaler_func.create_scaler_func(args.train_files,
model.CSV_COLUMNS,
model.LABEL_COLUMN,
args.bucket_name,
args.project_id)
logging.info('End fitting scalers')
# download the scaler
if not path.exists('x_scaler'):
logging.info('Downloading scaler')
storage_client = storage.Client(project=args.project_id)
bucket = storage_client.get_bucket(args.bucket_name)
blob = bucket.blob('scalers/x_scaler')
blob.download_to_filename('x_scaler')
logging.info('Downloaded scaler')
x_scaler = joblib.load('x_scaler')
# build the model
census_model = model.model_fn(
learning_rate=args.learning_rate,
num_deep_layers=args.num_deep_layers,
first_deep_layer_size=args.first_deep_layer_size,
first_wide_layer_size=args.first_wide_layer_size,
wide_scale_factor=args.wide_scale_factor,
dropout_rate=args.dropout_rate)
logging.info(census_model.summary())
try:
os.makedirs(args.job_dir)
except:
pass
checkpoint_path = os.path.join(args.job_dir, CHECKPOINT_FILE_PATH)
# Model checkpoint callback.
checkpoint = ModelCheckpoint(
checkpoint_path,
monitor='val_mse', # 'mean_squared_error'
verbose=1,
period=args.checkpoint_epochs,
save_best_only=True,
mode='min')
# Early stopping callback.
early_stop = EarlyStopping(monitor='val_mse',
patience=10) # 'mean_squared_error'
# Tensorboard logs callback.
tb_log = TensorBoard(log_dir=os.path.join(args.job_dir, 'logs'),
histogram_freq=0,
write_graph=True,
embeddings_freq=0)
callbacks = [checkpoint, early_stop, tb_log]
# fit the model on the training set
census_model.fit_generator(
generator=model.generator_input(args.train_files,
chunk_size=CHUNK_SIZE,
project_id=args.project_id,
bucket_name=args.bucket_name,
x_scaler=x_scaler),
steps_per_epoch=args.train_steps,
epochs=args.num_epochs,
callbacks=callbacks,
validation_data=model.generator_input(args.eval_files,
chunk_size=CHUNK_SIZE,
project_id=args.project_id,
bucket_name=args.bucket_name,
x_scaler=x_scaler),
validation_steps=args.eval_steps)
# evaluate model on test set
loss, mae, mse = census_model.evaluate_generator(model.generator_input(
args.test_files,
chunk_size=CHUNK_SIZE,
project_id=args.project_id,
bucket_name=args.bucket_name,
x_scaler=x_scaler),
steps=args.test_steps)
logging.info('\nTest evaluation metrics[{:.2f}, {:.2f}, {:.2f}] {}'.format(
loss, mae, mse, census_model.metrics_names))
# Unhappy hack to workaround h5py not being able to write to GCS.
# Force snapshots and saves to local filesystem, then copy them over to GCS.
if args.job_dir.startswith('gs://'):
census_model.save(CENSUS_MODEL)
copy_file_to_gcs(args.job_dir, CENSUS_MODEL)
else:
census_model.save(os.path.join(args.job_dir, CENSUS_MODEL))
# Convert the Keras model to TensorFlow SavedModel.
model.to_savedmodel(census_model, os.path.join(args.job_dir, 'export'))
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Pre-processing script to generate a sample for SavedModel prediction."""
import json
import sys
from trainer import model
if __name__=='__main__':
gen = model.generator_input(['data/output_test.csv'], chunk_size=5000)
sample = gen.next()
input_sample = {}
input_sample['input'] = sample[0].values[0].tolist()
print('Produced sample with label {}'.format(sample[1].values[0].tolist()))
with open(sys.argv[1], 'w') as outfile:
json.dump(input_sample, outfile)
def train_and_evaluate(args):
CLASS_SIZE = len(bins) + 1
# hidden_units = [int(units) for units in args.hidden_units.split(',')]
try:
os.makedirs(args.job_dir)
except:
pass
# Unhappy hack to workaround h5py not being able to write to GCS.
# Force snapshots and saves to local filesystem, then copy them over to GCS.
checkpoint_path = CHECKPOINT_FILE_PATH
if not args.job_dir.startswith('gs://'):
checkpoint_path = os.path.join(args.job_dir, checkpoint_path)
# Model checkpoint callback.
checkpoint = ModelCheckpoint(checkpoint_path,
monitor='val_loss',
verbose=1,
period=args.checkpoint_epochs,
mode='min')
tb_log = TensorBoard(log_dir=os.path.join(args.job_dir, 'logs'),
histogram_freq=0,
write_graph=True,
embeddings_freq=0)
callbacks = [checkpoint, tb_log]
sequential_train = [
int(float(hour)) for hour in args.sequential_train.split(',')
]
seq_id = 0
sequential_models = []
weights_all = []
for hours in sequential_train:
history_all = []
if seq_id == 0:
hidden_units = args.hidden_units
learning_rate = args.learning_rate
INPUT_DIM = hours * ONE_HOUR
###########fully connected model#############
first_model = initial_model.model_fn(INPUT_DIM, CLASS_SIZE,
hidden_units, 0.0001)
assign_w = 0.03
first_model.compile(loss=initial_model.weighted_loss(assign_w),
optimizer=keras.optimizers.Adam(lr=0.0001),
metrics=[
initial_model.first_class_accuracy,
initial_model.other_class_accuracy,
initial_model.single_class_accuracy(1),
initial_model.single_class_accuracy(2),
initial_model.single_class_accuracy(3),
initial_model.single_class_accuracy(4),
initial_model.single_class_accuracy(5),
initial_model.single_class_accuracy(6),
initial_model.single_class_accuracy(7),
'accuracy'
])
train_file_names = os.path.join(args.train_files,
str(hours) + 'hrs', 'train',
'*npz')
eval_file_names = os.path.join(args.eval_files,
str(hours) + 'hrs', 'eval', '*npz')
print("\n\ntraining " + str(hours) + 'hrs!\n\n')
history_all.append(
first_model.fit_generator(
initial_model.generator_input(train_file_names,
args.train_batch_size),
validation_data=initial_model.generator_input(
eval_file_names, args.eval_batch_size),
steps_per_epoch=args.train_steps,
validation_steps=args.eval_steps,
epochs=args.num_epochs,
callbacks=callbacks))
weights = first_model.get_weights()
weights_all.append(weights)
with open(
os.path.join(args.job_dir, 'weights',
str(hours) + 'hrs_weights'), 'wb') as fp:
pickle.dump(weights, fp)
DISK_MODEL = 'disk_model.hdf5'
if args.job_dir.startswith('gs://'):
first_model.save(DISK_MODEL)
copy_file_to_gcs(args.job_dir, DISK_MODEL)
else:
first_model.save(os.path.join(args.job_dir, DISK_MODEL))
data, label = initial_model.generator_input_once(
str(args.train_files) + str(hours) + 'hrs/train/input_' +
str(hours) + 'hrs_8.npz', 3)
first_model.compile(loss=initial_model.weighted_loss(0.00081),
optimizer=keras.optimizers.Adam(lr=0.0001),
metrics=[
initial_model.first_class_accuracy,
initial_model.other_class_accuracy,
initial_model.single_class_accuracy(1),
initial_model.single_class_accuracy(2),
initial_model.single_class_accuracy(3),
initial_model.single_class_accuracy(4),
initial_model.single_class_accuracy(5),
initial_model.single_class_accuracy(6),
initial_model.single_class_accuracy(7),
'accuracy'
])
first_model.fit_generator(
initial_model.generator_input(train_file_names,
args.train_batch_size),
validation_data=initial_model.generator_input(
eval_file_names, args.eval_batch_size),
steps_per_epoch=args.train_steps,
validation_steps=args.eval_steps,
epochs=50,
callbacks=callbacks)
scores = first_model.evaluate(x=data,
y=label,
batch_size=None,
verbose=1,
sample_weight=None,
steps=1)
print("\ntest " + str(hours) + 'hrs after train\n')
print(scores)
seq_id = seq_id + 1
else:
with open(
os.path.join(
args.job_dir, 'weights',
str(sequential_train[seq_id - 1]) + 'hrs_weights'),
'rb') as fp:
weights_0 = pickle.load(fp)
######sequential(weights, CONCAT_UNIT_SIZE, INPUT_SHAPE, learning_rate)
hours = sequential_train[seq_id]
seq = Sequential(weights_0, args.CONCAT_UNIT_SIZE,
hours * ONE_HOUR, 'zeros')
model = seq.build_sequential_model()
# assign_w = 0.016+0.005*seq_id
assign_w = 0.03
model.compile(loss=initial_model.weighted_loss(assign_w),
optimizer=keras.optimizers.Adam(lr=0.0001),
metrics=[
initial_model.first_class_accuracy,
initial_model.other_class_accuracy,
initial_model.single_class_accuracy(1),
initial_model.single_class_accuracy(2),
initial_model.single_class_accuracy(3),
initial_model.single_class_accuracy(4),
initial_model.single_class_accuracy(5),
initial_model.single_class_accuracy(6),
initial_model.single_class_accuracy(7),
'accuracy'
])
data, label = initial_model.generator_input_once(
str(args.train_files) + str(hours) + 'hrs/train/input_' +
str(hours) + 'hrs_8.npz', 6)
scores = model.evaluate(x=data,
y=label,
batch_size=None,
verbose=1,
sample_weight=None,
steps=1)
print("\ntest " + str(hours) + 'hrs beofre train\n')
print(scores)
# data,label = initial_model.generator_input_once('/Volumes/TOSHIBA EXT/train_input/24hrs/train/input_24hrs_8.npz', 24)
#
# scores = model.evaluate(x=data, y=label, batch_size=None, verbose=1, sample_weight=None, steps=1)
# print(scores)
###########sequential model#############
train_file_names = os.path.join(str(args.train_files),
str(hours) + 'hrs', 'train',
'*npz')
eval_file_names = os.path.join(args.eval_files,
str(hours) + 'hrs', 'eval', '*npz')
print("\n\ntraining " + str(hours) + 'hrs!\n\n')
history_all.append(
model.fit_generator(
initial_model.generator_input(train_file_names,
args.train_batch_size),
validation_data=initial_model.generator_input(
eval_file_names, args.eval_batch_size),
steps_per_epoch=args.train_steps,
validation_steps=args.eval_steps,
epochs=args.num_epochs,
callbacks=callbacks))
weights = model.get_weights()
weights_all.append(weights)
weights_0 = []
for i in range(int(len(weights) / 4)):
if i == int(len(weights) / 4) - 1:
weights_0.extend([
np.concatenate((weights[i * 4 + 2], weights[i * 4]),
axis=0),
(weights[i * 4 + 1] + weights[i * 4 + 3])
])
elif i == int(len(weights) / 4) - 2:
weights_0.extend([
np.concatenate((weights[i * 4 + 2], weights[i * 4]),
axis=1),
np.concatenate(
(weights[i * 4 + 3], weights[i * 4 + 1]))
])
else:
weights_0.extend([
np.concatenate((weights[i * 4], weights[i * 4 + 2]),
axis=1),
np.concatenate(
(weights[i * 4 + 1], weights[i * 4 + 3]))
])
# weights_0 = [np.concatenate((weights[0], weights[2]), axis=1),
# np.concatenate((weights[1], weights[3])),
# np.concatenate((weights[4], weights[6]), axis=1),
# np.concatenate((weights[5], weights[7])),
# np.concatenate((weights[8], weights[10]), axis=0)
# weights[9] + weights[11]]
data, label = initial_model.generator_input_once(
str(args.train_files) + str(hours) + 'hrs/train/input_' +
str(hours) + 'hrs_8.npz', 6)
scores = model.evaluate(x=data,
y=label,
batch_size=None,
verbose=1,
sample_weight=None,
steps=1)
print("\ntest " + str(hours) + 'hrs after train\n')
print(scores)
with open(
os.path.join(str(args.job_dir), 'weights',
str(hours) + 'hrs_weights'), 'wb') as fp:
pickle.dump(weights_0, fp)
DISK_MODEL = 'disk_model' + str(hours) + '.hdf5'
if args.job_dir.startswith('gs://'):
model.save(DISK_MODEL)
copy_file_to_gcs(args.job_dir, DISK_MODEL)
else:
model.save(os.path.join(args.job_dir, DISK_MODEL))
seq_id = seq_id + 1
with open(os.path.join(str(args.job_dir), 'histroy_all'), 'wb') as fp:
pickle.dump(history_all, fp)
from google.oauth2 import service_account
from googleapiclient import discovery
from trainer import model
PATH_CREDENTIALS_GOOGLE = os.environ.get('GOOGLE_APPLICATION_CREDENTIALS',
None)
credentials = service_account.Credentials.from_service_account_file(
PATH_CREDENTIALS_GOOGLE)
api = discovery.build('ml', 'v1', credentials=credentials)
parent = 'projects/%s/models/%s/versions/%s' % ('academic-motif-193414',
'neural_keras_model', 'v1')
data = model.generator_input(batch_size=1)
for i in range(0, 30):
flag = True
while flag:
sample = next(data)
flag = True if random() > 0.5 else False
sample = next(data)
input = sample[0][0].tolist()
request_data = {'instances': [input]}
response = api.projects().predict(body=request_data, name=parent).execute()
result = response['predictions'][0]['income'][0]
print("{:.3g} value predicted for the sample {}, binary output {}".format(
result, input, round(result)))
default='ross-keras')
args, _ = parser.parse_known_args()
# download the scaler
if not path.exists('x_scaler'):
print('Downloading scaler')
storage_client = storage.Client(project='mwpmltr')
bucket = storage_client.get_bucket('ross-keras')
blob = bucket.blob('scalers/x_scaler')
blob.download_to_filename('x_scaler')
print('Downloaded scaler')
x_scaler = joblib.load('x_scaler')
gen = model.generator_input(['gs://ross-keras/data/full_test_results.csv'], chunk_size=5000, project_id=args.project_id, bucket_name=args.bucket_name, x_scaler=x_scaler, batch_size=1)
actuals = []
preds = []
for i in range(0, args.num_samples):
sample = gen.__next__()
input_sample = {}
input_sample['input'] = sample[0]
actual = int(round(np.exp(sample[1])))
actuals.append(actual)
with open('input_sample.json', 'w') as outfile:
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Pre-processing script to generate a sample for SavedModel prediction."""
import json
import sys
from trainer import model
if __name__=='__main__':
gen = model.generator_input(['adult.data.csv'], chunk_size=5000)
sample = gen.__next__()
input_sample = {}
input_sample['input'] = sample[0].values[0].tolist()
print('Produced sample with label {}'.format(sample[1].values[0].tolist()))
with open(sys.argv[1], 'w') as outfile:
json.dump(input_sample, outfile)
"""Pre-processing script to generate a sample for SavedModel prediction."""
import json
import sys
from trainer import model
TEST_CSV_PATH = "data/adult.data.csv"
if __name__ == '__main__':
gen = model.generator_input([TEST_CSV_PATH], chunk_size=5000)
sample = gen.next()
input_sample = {}
input_sample['input'] = sample[0].values[0].tolist()
print('Produced sample with label {}'.format(sample[1].values[0].tolist()))
with open(sys.argv[1], 'w') as outfile:
json.dump(input_sample, outfile)
'--bucket-name',
type=str,
default='sanofi-ml-workshop-chicago-taxi-demo',
help='The Cloud Storage bucket to be used for process artifacts')
args, _ = parser.parse_known_args()
# download the scaler
if not path.exists('x_scaler'):
print('Downloading scaler')
storage_client = storage.Client(project=args.project_id)
bucket = storage_client.get_bucket(args.bucket_name)
blob = bucket.blob('scalers/x_scaler')
blob.download_to_filename('x_scaler')
print('Downloaded scaler')
x_scaler = joblib.load('x_scaler')
gen = model.generator_input(['gs://{}/data/full_test_results.csv'.format(args.bucket_name)], chunk_size=5000, project_id=args.bucket_name, bucket_name=args.bucket_name, x_scaler=x_scaler, batch_size=1)
for i in range(1, random.randint(1,100)):
sample = gen.__next__()
input_sample = {}
input_sample['input'] = sample[0]
print('Produced sample with label {} seconds.'.format(str(int(round(np.exp(sample[1]))))))
with open('input_sample.json', 'w') as outfile:
json.dump(input_sample, outfile, cls=NumpyEncoder)
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Pre-processing script to generate a sample for SavedModel prediction."""
import json
import sys
from trainer import model
if __name__ == '__main__':
gen = model.generator_input(['adult.data.csv'], chunk_size=500)
sample = gen.__next__()
sample = gen.__next__()
input_sample = {}
input_sample['input'] = sample[0].values[0].tolist()
print('Produced sample with label {}'.format(sample[1].values[0].tolist()))
with open(sys.argv[1], 'w') as outfile:
json.dump(input_sample, outfile)
def train_and_evaluate(args):
# Showcasing the hypertuning parameters here.
# The first-layer-size is being tuned in this example
hidden_units = [args.first_layer_size, 70, 50, 20]
census_model = model.model_fn(INPUT_SIZE, CLASS_SIZE, hidden_units)
try:
os.makedirs(args.job_dir)
except:
pass
# Unhappy hack to workaround h5py not being able to write to GCS.
# Force snapshots and saves to local filesystem, then copy them over to GCS.
checkpoint_path = CHECKPOINT_FILE_PATH
if not args.job_dir.startswith('gs://'):
checkpoint_path = os.path.join(args.job_dir, checkpoint_path)
# Model checkpoint callback.
checkpoint = ModelCheckpoint(
checkpoint_path,
monitor='val_loss',
verbose=1,
save_best_only=False,
period=args.checkpoint_epochs,
mode='min')
# Continuous eval callback.
evaluation = ContinuousEval(args.eval_frequency, args.eval_files,
args.learning_rate, args.job_dir)
# Tensorboard logs callback.
tb_log = TensorBoard(
log_dir=os.path.join(args.job_dir, 'logs'),
histogram_freq=0,
write_graph=True,
embeddings_freq=0)
callbacks = [checkpoint, evaluation, tb_log]
census_model.fit_generator(
model.generator_input(args.train_files, chunk_size=CHUNK_SIZE),
steps_per_epoch=args.train_steps,
epochs=args.num_epochs,
use_multiprocessing=args.distributed,
callbacks=callbacks)
# Unhappy hack to workaround h5py not being able to write to GCS.
# Force snapshots and saves to local filesystem, then copy them over to GCS.
if args.job_dir.startswith('gs://'):
census_model.save(CENSUS_MODEL)
copy_file_to_gcs(args.job_dir, CENSUS_MODEL)
else:
census_model.save(os.path.join(args.job_dir, CENSUS_MODEL))
# Convert the Keras model to TensorFlow SavedModel.
model.to_savedmodel(census_model, os.path.join(args.job_dir, 'export'))
# The following is for hyperparameter tuning and is adapted from here: https://cloud.google.com/ml-engine/docs/tensorflow/using-hyperparameter-tuning
# Note: the last_loss_val is updated after each checkpoint, but we only write the summary once.
summary = Summary(value=[Summary.Value(tag='val_loss', simple_value=evaluation.last_loss_val)])
# more hypertune info here: https://cloud.google.com/solutions/machine-learning/recommendation-system-tensorflow-train-cloud-ml-engine
job_dir = args.job_dir
if args.hypertune:
# if tuning, join the trial number to the output path
trial = json.loads(os.environ.get('TF_CONFIG', '{}')).get('task', {}).get('trial', '')
output_dir = os.path.join(job_dir, trial)
else:
output_dir = job_dir
eval_path = os.path.join(output_dir, 'val_loss')
summary_writer = tf.summary.FileWriter(eval_path)
# Note: adding the summary to the writer is enough for hyperparameter tuning.
# ML Engine looks for any summary added with the hyperparameter metric tag.
summary_writer.add_summary(summary)
summary_writer.flush()
if __name__ == '__main__':
# download the scaler
if not path.exists('x_scaler'):
print('Downloading scaler')
storage_client = storage.Client(project='mwpmltr')
bucket = storage_client.get_bucket('ross-keras')
blob = bucket.blob('scalers/x_scaler')
blob.download_to_filename('x_scaler')
print('Downloaded scaler')
x_scaler = joblib.load('x_scaler')
gen = model.generator_input(['gs://ross-keras/data/full_test_results.csv'],
chunk_size=5000,
project_id=PROJECT_ID,
bucket_name=BUCKET_NAME,
x_scaler=x_scaler,
batch_size=1)
for i in range(1, random.randint(1, 100)):
sample = gen.__next__()
input_sample = {}
input_sample['input'] = sample[0]
print('Produced sample with label {} seconds.'.format(
str(int(round(np.exp(sample[1]))))))
with open('input_sample.json', 'w') as outfile:
json.dump(input_sample, outfile, cls=NumpyEncoder)
