def test_experiment_fit_gen(self, get_model, get_loss_metric,
get_custom_l, get_callback_fix):
new_session()
model, metrics, cust_objects = prepare_model(get_model(get_custom_l),
get_loss_metric,
get_custom_l)
model_name = model.__class__.__name__
_, data_val_use = make_data(train_samples, test_samples)
expe = Experiment(model)
for val in [1, data_val_use]:
gen, data, data_stream = make_gen(batch_size)
if val == 1:
val, data_2, data_stream_2 = make_gen(batch_size)
expe.fit_gen([gen], [val], nb_epoch=2,
model=model,
metrics=metrics,
custom_objects=cust_objects,
samples_per_epoch=64,
nb_val_samples=128,
verbose=2, overwrite=True,
callbacks=get_callback_fix)
close_gens(gen, data, data_stream)
if val == 1:
close_gens(val, data_2, data_stream_2)
if K.backend() == 'tensorflow':
K.clear_session()
print(self)
def test_build_predict_func(self, get_model):
"""Test the build of a model"""
new_session()
X_tr = np.ones((train_samples, input_dim))
model = get_model()
model.compile(loss='categorical_crossentropy',
optimizer='rmsprop',
metrics=['accuracy'])
model_name = model.__class__.__name__
pred_func = KTB.build_predict_func(model)
tensors = [X_tr]
if model_name != 'Model':
tensors.append(1.)
res = pred_func(tensors)
assert len(res[0]) == len(X_tr)
if K.backend() == 'tensorflow':
K.clear_session()
print(self)
def create_tiny_model(input_shape, anchors, num_classes, load_pretrained=True, freeze_body=2,
weights_path='model_data/tiny_yolo_weights.h5'):
'''create the training model, for Tiny YOLOv3'''
K.clear_session() # get a new session
image_input = Input(shape=(None, None, 3))
h, w = input_shape
num_anchors = len(anchors)
y_true = [Input(shape=(h//{0:32, 1:16}[l], w//{0:32, 1:16}[l], \
num_anchors//2, num_classes+5)) for l in range(2)]
model_body = tiny_yolo_body(image_input, num_anchors//2, num_classes)
print('Create Tiny YOLOv3 model with {} anchors and {} classes.'.format(num_anchors, num_classes))
if load_pretrained:
model_body.load_weights(weights_path, by_name=True, skip_mismatch=True)
print('Load weights {}.'.format(weights_path))
if freeze_body in [1, 2]:
# Freeze the darknet body or freeze all but 2 output layers.
num = (20, len(model_body.layers)-2)[freeze_body-1]
for i in range(num): model_body.layers[i].trainable = False
print('Freeze the first {} layers of total {} layers.'.format(num, len(model_body.layers)))
model_loss = Lambda(yolo_loss, output_shape=(1,), name='yolo_loss',
arguments={'anchors': anchors, 'num_classes': num_classes, 'ignore_thresh': 0.7})(
[*model_body.output, *y_true])
model = Model([model_body.input, *y_true], model_loss)
return model
def run_one_eval(self, train_x, train_y, valid_x, valid_y, epochs, config):
model, history = self.train_model_for_data(train_x, train_y, epochs, config, valid=0.1)
loss = model.evaluate(valid_x, valid_y, verbose=0)
print("Running an eval with config: %s had validation loss %f" % (str(config), loss))
K.clear_session()
return loss
def predictor(q):
model = make_model(network, (None, None, channels))
model.load_weights('trained_models/' + weights)
while True:
f, image = q.get()
if image is None:
break
preds = []
for tta in [None, 'hflip', 'vflip', 'hflip+vflip']:
ttas = []
if tta:
ttas = tta.split("+")
img = do_tta(image, ttas)
pred = model.predict(np.expand_dims(img, axis=0), batch_size=1, verbose=0)[0]
pred = undo_tta(pred, ttas)
preds.append(pred)
mask = np.average(np.array(preds), axis=0)
all_masks_dir = "all_masks"
os.makedirs(all_masks_dir, exist_ok=True)
model_mask_dir = os.path.join(all_masks_dir, out_dir)
os.makedirs(model_mask_dir, exist_ok=True)
cv2.imwrite(os.path.join(model_mask_dir, f + ".png"), mask * 255)
del model
K.clear_session()
def test_experiment_fit(self, get_model, get_loss_metric,
get_custom_l, get_callback_fix):
new_session()
data, data_val = make_data(train_samples, test_samples)
model, metrics, cust_objects = prepare_model(get_model(get_custom_l),
get_loss_metric,
get_custom_l)
expe = Experiment(model)
for mod in [None, model]:
for data_val_loc in [None, data_val]:
expe.fit([data], [data_val_loc], model=mod, nb_epoch=2,
batch_size=batch_size, metrics=metrics,
custom_objects=cust_objects, overwrite=True,
callbacks=get_callback_fix)
expe.backend_name = 'another_backend'
expe.load_model()
expe.load_model(expe.mod_id, expe.data_id)
assert expe.data_id is not None
assert expe.mod_id is not None
assert expe.params_dump is not None
if K.backend() == 'tensorflow':
K.clear_session()
print(self)
def _run(self):
round_params(self)
try:
_hr_out, self.keras_model = self._model()
except TypeError:
print('The model needs to have Return in format "return history, model"')
self.epoch_entropy.append(epoch_entropy((_hr_out)))
_hr_out = run_round_results(self, _hr_out)
self._val_score = get_score(self)
write_log(self)
self.result.append(_hr_out)
save_result(self)
time_estimator(self)
if (self.round_counter + 1) % self.reduction_interval == 0:
if self.reduction_method == 'spear':
self = spear_reducer(self)
K.clear_session()
self.round_counter += 1
def start_session_get_args_and_model(intra_ops, inter_ops, semantics_json, weights_hd5=None, tensor_type=None):
K.clear_session()
K.get_session().close()
cfg = K.tf.ConfigProto(intra_op_parallelism_threads=intra_ops, inter_op_parallelism_threads=inter_ops)
cfg.gpu_options.allow_growth = True
K.set_session(K.tf.Session(config=cfg))
return args_and_model_from_semantics(semantics_json, weights_hd5, tensor_type)
def KerasEmit(original_framework, architecture_name, architecture_path, weight_path, image_path):
# IR to code
converted_file = original_framework + '_keras_' + architecture_name + "_converted"
converted_file = converted_file.replace('.', '_')
emitter = Keras2Emitter((architecture_path, weight_path))
emitter.run(converted_file + '.py', None, 'test')
del emitter
# import converted model
model_converted = __import__(converted_file).KitModel(weight_path)
func = TestKit.preprocess_func[original_framework][architecture_name]
img = func(image_path)
input_data = np.expand_dims(img, 0)
predict = model_converted.predict(input_data)
converted_predict = np.squeeze(predict)
del model_converted
del sys.modules[converted_file]
import keras.backend as K
K.clear_session()
os.remove(converted_file + '.py')
return converted_predict
def __call__(cls, *args, **kwargs):
obj = cls.__new__(cls)
from .keras_model import KerasModel
if issubclass(cls, KerasModel):
import keras.backend as K
if K.backend() != 'tensorflow':
obj.__init__(*args, **kwargs)
return obj
K.clear_session()
obj.graph = tf.Graph()
with obj.graph.as_default():
if hasattr(cls, '_config_session'):
obj.sess = cls._config_session()
else:
obj.sess = tf.Session()
else:
obj.graph = tf.Graph()
for meth in dir(obj):
if meth == '__class__':
continue
attr = getattr(obj, meth)
if callable(attr):
if issubclass(cls, KerasModel):
wrapped_attr = _keras_wrap(attr, obj.graph, obj.sess)
else:
wrapped_attr = _graph_wrap(attr, obj.graph)
setattr(obj, meth, wrapped_attr)
obj.__init__(*args, **kwargs)
return obj
def test_clone_sequential_model():
val_a = np.random.random((10, 4))
val_out = np.random.random((10, 4))
model = keras.models.Sequential()
model.add(keras.layers.Dense(4, input_shape=(4,)))
model.add(keras.layers.BatchNormalization())
model.add(keras.layers.Dropout(0.5))
model.add(keras.layers.Dense(4))
if K.backend() == 'tensorflow':
# Everything should work in a new session.
K.clear_session()
# With placeholder creation
new_model = keras.models.clone_model(model)
new_model.compile('rmsprop', 'mse')
new_model.train_on_batch(val_a, val_out)
# On top of new tensor
input_a = keras.Input(shape=(4,))
new_model = keras.models.clone_model(
model, input_tensors=input_a)
new_model.compile('rmsprop', 'mse')
new_model.train_on_batch(val_a, val_out)
# On top of new, non-Keras tensor
input_a = keras.backend.variable(val_a)
new_model = keras.models.clone_model(
model, input_tensors=input_a)
new_model.compile('rmsprop', 'mse')
new_model.train_on_batch(None, val_out)
def train(args):
with open("conf/fea.yaml") as fin:
cfg = yaml.load(fin)[args.p]
begin, end = map(int, cfg["train"].split("-"))
with TimeLog("load data"):
data = read_data(cfg["data"], begin, end)
if not args.m:
args.m = args.p
else:
args.m = args.p + "_" + args.m
print "model={m}, data={d}".format(d=cfg["data"], m=args.m)
model_dir = "model/" + args.m + "/"
makedirs(model_dir)
if False:
model = load_model(model_dir)
else:
model = make_model(model_dir, len(data.fea[0]))
gamma = 0.6
n_epochs = [10, 10]
lr = 0.001
batch_size = 1024
def lr_scheduler(epoch):
learning_rate = lr
ep = epoch
for n_epoch in n_epochs:
if ep - n_epoch < 0:
break
learning_rate *= gamma
ep -= n_epoch
print 'lr: %f' % learning_rate
return learning_rate
scheduler = LearningRateScheduler(lr_scheduler)
if args.opt == "adam":
optimizer = Adam(lr=lr)
else:
optimizer = SGD(lr=lr)
# model.compile(loss='binary_crossentropy', optimizer=optimizer)
model.compile(loss='mse', optimizer=optimizer)
filepath = model_dir + "/e{epoch:d}.hdf5"
checkpoint = ModelCheckpoint(
filepath, monitor='mse',
verbose=1, save_best_only=False, mode='auto')
callbacks_list = [scheduler, checkpoint]
model.fit(data.fea, data.tgt, batch_size=batch_size,
validation_split=0.1,
nb_epoch=sum(n_epochs), callbacks=callbacks_list)
print 'saving...'
weightPath = model_dir + '/weight.hdf5'
model.save_weights(weightPath, overwrite=True)
print 'training finished'
K.clear_session()
def clear_session_after_test():
"""Test wrapper to clean up after TensorFlow and CNTK tests.
This wrapper runs for all the tests in the keras test suite.
"""
yield
if K.backend() == 'tensorflow' or K.backend() == 'cntk':
K.clear_session()
def __init__(self, path=DEFAULT_MODEL_PATH):
logger.info('Loading model from: {}...'.format(path))
clear_session()
self.model = models.load_model(path)
# this seems to be required to make Keras models play nicely with threads
self.model._make_predict_function()
logger.info('Loaded model: {}'.format(self.model.name))
def new_session():
if K.backend() == 'tensorflow': # pragma: no cover
import tensorflow as tf
K.clear_session()
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth = True
session = tf.Session(config=config)
K.set_session(session)
def clear_session():
try:
K.clear_session()
K.get_session().close()
cfg = K.tf.ConfigProto()
cfg.gpu_options.allow_growth = True
K.set_session(K.tf.Session(config=cfg))
except AttributeError as e:
print('Could not clear session. Maybe you are using Theano backend?')
def model_eval(self, model, X, y):
'''
学習モデルを評価する
:param numpy.array X: 評価用の画像データのリスト
:param numpy.array y: 評価用の画像に対する分類ラベルのリスト
'''
score = model.evaluate(X, y)
print('loss=', score[0])
print('accuracy=', score[1])
backend.clear_session()
def onExit():
global dlib_detectors
global keras_model
if keras_model is not None:
del keras_model
K.clear_session()
for detector in dlib_detectors:
del detector
def build_part1_RNN(window_size, lstm_size=5):
# Clear session before creating a new model
K.clear_session()
# inputs = Input(shape=(window_size, 1))
# _lstm = LSTM(lstm_size)(inputs)
# outputs = Dense(1)(_lstm)
# return Model(inputs, outputs)
model = Sequential()
model.add(LSTM(lstm_size, input_shape=(window_size, 1)))
model.add(Dense(1))
return model
def set_keras_backend(backend):
if K.backend() != backend:
os.environ["KERAS_BACKEND"] = backend
importlib.reload(K)
assert K.backend() == backend
if backend == "tensorflow":
K.get_session().close()
cfg = K.tf.ConfigProto()
cfg.gpu_options.allow_growth = True
K.set_session(K.tf.Session(config=cfg))
K.clear_session()
def keras_get_model(self):
# keras.backend.tensorflow_backend.clear_session()
backendK.clear_session()
# if settings.GPU_FLAG == True:
# gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.333)
# sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))
# backendK.set_session(sess)
try:
self.model = keras.models.load_model(self.last_chk_path)
logging.info("Train Restored checkpoint from:" + self.last_chk_path)
except Exception as e:
logging.info("None to restore checkpoint. Initializing variables instead." + self.last_chk_path)
logging.info(e)
if self.optimizer == 'sgd':
self.optimizer = optimizers.SGD(lr=0.001, decay=1e-6, momentum=0.9, nesterov=True)
elif self.optimizer == 'rmsprop':
self.optimizer = optimizers.RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=1e-6)
elif self.optimizer == 'adagrad':
self.optimizer = optimizers.Adagrad(lr=0.01, epsilon=1e-08, decay=1e-6)
elif self.optimizer == 'adadelta':
self.optimizer = optimizers.Adadelta(lr=1.0, rho=0.95, epsilon=1e-08, decay=1e-6)
elif self.optimizer == 'adam':
self.optimizer = optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=1e-6)
# self.optimizer = optimizers.Adam(lr=self.lr_schedule(0))
elif self.optimizer == 'adamax':
self.optimizer = optimizers.Adamax(lr=0.002, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=1e-6)
elif self.optimizer == 'nadam':
self.optimizer = optimizers.Nadam(lr=0.002, beta_1=0.9, beta_2=0.999, epsilon=1e-08, schedule_decay=0.004)
if self.net_type == 'inceptionv4':
# self.labels_cnt = 1001
self.model = inception_v4_model(self.labels_cnt, 0.2, self.pretrain_model_path)
# elif self.net_type == 'nasnet':
# self.model = NASNetLarge(input_shape=(331, 331, 3))
elif self.net_type == 'resnet':
numoutputs = self.netconf["config"]["layeroutputs"]
if numoutputs == 18:
self.model = resnet.ResnetBuilder.build_resnet_18((self.channel, self.x_size, self.y_size), self.labels_cnt)
elif numoutputs == 34:
self.model = resnet.ResnetBuilder.build_resnet_34((self.channel, self.x_size, self.y_size), self.labels_cnt)
elif numoutputs == 50:
self.model = resnet.ResnetBuilder.build_resnet_50((self.channel, self.x_size, self.y_size), self.labels_cnt)
elif numoutputs == 101:
self.model = resnet.ResnetBuilder.build_resnet_101((self.channel, self.x_size, self.y_size), self.labels_cnt)
elif numoutputs == 152:
self.model = resnet.ResnetBuilder.build_resnet_152((self.channel, self.x_size, self.y_size), self.labels_cnt)
# if settings.GPU_FLAG == True:
# self.model = multi_gpu_model(self.model, gpus=1)
self.model.compile(loss='categorical_crossentropy', optimizer=self.optimizer, metrics=['accuracy'])
def multi_gpu_application_np_array_benchmark():
print('####### Xception benchmark - np i/o')
model_cls = keras.applications.Xception
num_samples = 1000
height = 224
width = 224
num_classes = 1000
epochs = 4
batch_size = 40
x = np.random.random((num_samples, height, width, 3))
y = np.random.random((num_samples, num_classes))
# Baseline
model = model_cls(weights=None,
input_shape=(height, width, 3),
classes=num_classes)
model.compile(loss='categorical_crossentropy',
optimizer='rmsprop')
# Training
start_time = time.time()
model.fit(x, y, epochs=epochs)
total_time = time.time() - start_time
print('baseline training:', total_time)
# Inference
start_time = time.time()
model.predict(x)
total_time = time.time() - start_time
print('baseline inference:', total_time)
for i in range(8, 9):
K.clear_session()
with tf.device('/cpu:0'):
model = model_cls(weights=None,
input_shape=(height, width, 3),
classes=num_classes)
parallel_model = multi_gpu_model(model, gpus=i)
parallel_model.compile(loss='categorical_crossentropy',
optimizer='rmsprop')
start_time = time.time()
parallel_model.fit(x, y, epochs=epochs, batch_size=batch_size)
total_time = time.time() - start_time
print('%d gpus training:' % i, total_time)
# Inference
start_time = time.time()
parallel_model.predict(x, batch_size=batch_size)
total_time = time.time() - start_time
print('%d gpus inference:' % i, total_time)
def gen_IR(self):
for layer in self.keras_graph.topological_sort:
current_node = self.keras_graph.get_node(layer)
node_type = current_node.type
if hasattr(self, "rename_" + node_type):
func = getattr(self, "rename_" + node_type)
func(current_node)
else:
print("KerasParser has not supported operator [%s]." % (node_type))
self.rename_UNKNOWN(current_node)
_K.clear_session()
def test_deserialization(self):
model = sequential()
model.compile(optimizer='sgd', loss='categorical_crossentropy')
ser_mod = to_dict_w_opt(model)
custom_objects = {'test_loss': [1, 2]}
custom_objects = {k: serialize(custom_objects[k])
for k in custom_objects}
model_from_dict_w_opt(ser_mod, custom_objects=custom_objects)
if K.backend() == 'tensorflow':
K.clear_session()
print(self)
def build_part2_RNN(window_size, num_chars):
# Clear session before creating a new model
K.clear_session()
# inputs = Input(shape=(window_size, num_chars))
# _lstm = LSTM(200)(inputs)
# _lstm = Dense(num_chars)(_lstm)
# outputs = Activation('softmax')(_lstm)
# return Model(inputs, outputs)
model = Sequential()
model.add(LSTM(200, input_shape=(window_size, num_chars)))
model.add(Dense(num_chars))
model.add(Activation('softmax'))
return model
def test_predict(self, get_model):
"""Test to predict using the backend"""
data, data_val = make_data(train_samples, test_samples)
model = get_model()
model.compile(optimizer='sgd', loss='categorical_crossentropy')
expe = Experiment(model)
expe.fit([data], [data_val])
KTB.predict(expe.model_dict, [data['X']])
if K.backend() == 'tensorflow':
K.clear_session()
print(self)
def predict(args):
makedirs("ans")
with open("conf/fea.yaml") as fin:
cfg = yaml.load(fin)[args.p]
begin, end = map(int, cfg["test"].split("-"))
with TimeLog("load data"):
data = read_data(cfg["data"], begin, end)
if not args.m:
args.m = args.p
model = load_model("model/" + args.m) #, model_file="e3.hdf5")
pred = model.predict(data.fea, batch_size=1024, verbose=1)
gen_ans(pred, data, "ans/" + args.m)
K.clear_session()
def create_model(input_shape, anchors, num_classes, load_pretrained=True, freeze_body=2,
weights_path='model_data/yolo_weights.h5'):
'''create the training model'''
K.clear_session() # get a new session
image_input = Input(shape=(None, None, 3))
h, w = input_shape
num_anchors = len(anchors)
y_true = [Input(shape=(h//{0:32, 1:16, 2:8}[l], w//{0:32, 1:16, 2:8}[l], \
num_anchors//3, num_classes+5)) for l in range(3)]
model_body = yolo_body(image_input, num_anchors//3, num_classes)
print('Create YOLOv3 model with {} anchors and {} classes.'.format(num_anchors, num_classes))
if load_pretrained:
model_body.load_weights(weights_path, by_name=True, skip_mismatch=True)
print('Load weights {}.'.format(weights_path))
if freeze_body in [1, 2]:
# Freeze darknet53 body or freeze all but 3 output layers.
num = (185, len(model_body.layers)-3)[freeze_body-1]
for i in range(num): model_body.layers[i].trainable = False
print('Freeze the first {} layers of total {} layers.'.format(num, len(model_body.layers)))
# get output of second last layers and create bottleneck model of it
out1=model_body.layers[246].output
out2=model_body.layers[247].output
out3=model_body.layers[248].output
bottleneck_model = Model([model_body.input, *y_true], [out1, out2, out3])
# create last layer model of last layers from yolo model
in0 = Input(shape=bottleneck_model.output[0].shape[1:].as_list())
in1 = Input(shape=bottleneck_model.output[1].shape[1:].as_list())
in2 = Input(shape=bottleneck_model.output[2].shape[1:].as_list())
last_out0=model_body.layers[249](in0)
last_out1=model_body.layers[250](in1)
last_out2=model_body.layers[251](in2)
model_last=Model(inputs=[in0, in1, in2], outputs=[last_out0, last_out1, last_out2])
model_loss_last =Lambda(yolo_loss, output_shape=(1,), name='yolo_loss',
arguments={'anchors': anchors, 'num_classes': num_classes, 'ignore_thresh': 0.5})(
[*model_last.output, *y_true])
last_layer_model = Model([in0,in1,in2, *y_true], model_loss_last)
model_loss = Lambda(yolo_loss, output_shape=(1,), name='yolo_loss',
arguments={'anchors': anchors, 'num_classes': num_classes, 'ignore_thresh': 0.5})(
[*model_body.output, *y_true])
model = Model([model_body.input, *y_true], model_loss)
return model, bottleneck_model, last_layer_model
def inference(cls, architecture, path, image_path):
if cls.sanity_check(architecture):
model = cls.architecture_map[architecture]()
import numpy as np
func = TestKit.preprocess_func['keras'][architecture]
img = func(image_path)
img = np.expand_dims(img, axis=0)
predict = model.predict(img)
predict = np.squeeze(predict)
K.clear_session()
del model
return predict
else:
return None
def download(cls, architecture, path="./"):
if cls.sanity_check(architecture):
output_filename = path + 'imagenet_{}.h5'.format(architecture)
if os.path.exists(output_filename) == False:
model = cls.architecture_map[architecture]()
model.save(output_filename)
print("Keras model {} is saved in [{}]".format(architecture, output_filename))
K.clear_session()
del model
return output_filename
else:
print("File [{}] existed, skip download.".format(output_filename))
return output_filename
else:
return None
def _run_for_all(catalog, entity, threshold, name_rule, upload, sandbox,
dir_io, join_method):
"""
Runs the `linking` procedure using all available classifiers. Joins the results using
`join_method`
"""
assert join_method[0] in constants.SC_AVAILABLE_JOIN, (
'The provided join method needs to be one of: ' +
str(constants.SC_AVAILABLE_JOIN))
assert join_method[1] in constants.SC_AVAILABLE_COMBINE, (
'The provided combine method needs to be one of: ' +
str(constants.SC_AVAILABLE_COMBINE))
# ensure that models for all classifiers exist, and directly get the model
# and results path
available_classifiers = []
for classifier_name in list(set(constants.CLASSIFIERS.values())):
model_path, result_path = _handle_io(classifier_name, catalog, entity,
dir_io)
# Exit if the model file doesn't exist
if model_path is None:
sys.exit(1)
LOGGER.debug('Loading %s classifier ..', classifier_name)
available_classifiers.append(
(classifier_name, joblib.load(model_path), result_path))
rl.set_option(*constants.CLASSIFICATION_RETURN_SERIES)
for (
wd_chunk,
target_chunk,
feature_vectors,
) in _classification_set_generator(catalog, entity, dir_io):
# predict the current chunk with all classifiers
for classifier_name, classifier, result_path in available_classifiers:
LOGGER.info('Classifying chunk with classifier: %s',
classifier_name)
# The classification set must have the same feature space
# as the training one
_add_missing_feature_columns(classifier, feature_vectors)
predictions = (
# LSVM doesn't support probability scores
classifier.predict(feature_vectors) if isinstance(
classifier, rl.SVMClassifier) else
classifier.prob(feature_vectors))
predictions = _apply_linking_rules(name_rule, predictions,
target_chunk, wd_chunk)
# Threshold will be applied later, after joining
(_get_unique_predictions_above_threshold(predictions,
0.0).to_csv(result_path,
mode='a',
header=False))
# Once we have all the classification sets we can proceed to mix them
# as desired
all_results = []
for _, _, result_path in available_classifiers:
all_results.append(
(pd.read_csv(result_path,
header=None,
names=['qid', 'tid',
'prediction']).set_index(['qid', 'tid'])))
LOGGER.info(
"Joining the results of the classifications using the '%s' method",
join_method,
)
how_to_join, how_to_rem_duplicates = join_method
# Now we use join the dataframes using the correct method
merged_results: pd.DataFrame
if how_to_join == constants.SC_UNION:
merged_results = ensembles.join_dataframes_by_union(all_results)
elif how_to_join == constants.SC_INTERSECTION:
merged_results = ensembles.join_dataframes_by_intersection(all_results)
# and then proceed to deal with duplicates. This step also removes entries under the
# specified threshold
if how_to_rem_duplicates == constants.SC_AVERAGE:
merged_results = ensembles.remove_duplicates_by_averaging(
merged_results, threshold)
elif how_to_rem_duplicates == constants.SC_VOTING:
merged_results = ensembles.remove_duplicates_by_majority_vote(
merged_results, threshold)
merged_results = merged_results['prediction'] # get a pd.Series
result_path = os.path.join(
dir_io,
constants.LINKER_RESULT_JOINED.format(catalog, entity, how_to_join,
how_to_rem_duplicates),
)
# Delete existing result file,
# otherwise the current output would be appended to it
if os.path.isfile(result_path):
LOGGER.warning("Will delete old output file found at '%s' ...",
result_path)
os.remove(result_path)
merged_results.to_csv(result_path, mode='a', header=False)
if upload:
_upload(merged_results, 0, catalog, entity, sandbox)
K.clear_session() # Clear the TensorFlow graph
def tearDown(self):
k.clear_session()
def create_model(input_shape,
anchors,
num_classes,
load_pretrained=True,
freeze_body=2,
weights_path='model_data/yolo_weights.h5'):
'''create the training model'''
K.clear_session() # get a new session
image_input = Input(shape=(None, None, 3))
h, w = input_shape
num_anchors = len(anchors)
y_true = [Input(shape=(h//{0:32, 1:16, 2:8}[l], w//{0:32, 1:16, 2:8}[l], \
num_anchors//3, num_classes+5)) for l in range(3)]
model_body = yolo_body(image_input, num_anchors // 3, num_classes)
print('Create YOLOv3 model with {} anchors and {} classes.'.format(
num_anchors, num_classes))
if load_pretrained:
model_body.load_weights(weights_path, by_name=True, skip_mismatch=True)
print('Load weights {}.'.format(weights_path))
if freeze_body in [1, 2]:
# Freeze darknet53 body or freeze all but 3 output layers.
num = (160, len(model_body.layers) - 3)[freeze_body - 1]
for i in range(num):
model_body.layers[i].trainable = False
print('Freeze the first {} layers of total {} layers.'.format(
num, len(model_body.layers)))
# get output of second last layers and create bottleneck model of it
out1 = model_body.layers[246].output
out2 = model_body.layers[247].output
out3 = model_body.layers[248].output
bottleneck_model = Model([model_body.input, *y_true], [out1, out2, out3])
# create last layer model of last layers from yolo model
in0 = Input(shape=bottleneck_model.output[0].shape[1:].as_list())
in1 = Input(shape=bottleneck_model.output[1].shape[1:].as_list())
in2 = Input(shape=bottleneck_model.output[2].shape[1:].as_list())
last_out0 = model_body.layers[249](in0)
last_out1 = model_body.layers[250](in1)
last_out2 = model_body.layers[251](in2)
model_last = Model(inputs=[in0, in1, in2],
outputs=[last_out0, last_out1, last_out2])
model_loss_last = Lambda(yolo_loss,
output_shape=(1, ),
name='yolo_loss',
arguments={
'anchors': anchors,
'num_classes': num_classes,
'ignore_thresh': 0.5
})([*model_last.output, *y_true])
last_layer_model = Model([in0, in1, in2, *y_true], model_loss_last)
model_loss = Lambda(yolo_loss,
output_shape=(1, ),
name='yolo_loss',
arguments={
'anchors': anchors,
'num_classes': num_classes,
'ignore_thresh': 0.5
})([*model_body.output, *y_true])
model = Model([model_body.input, *y_true], model_loss)
return model, bottleneck_model, last_layer_model
def TSTR_mnist(identifier,
epoch,
generate=True,
duplicate_synth=1,
vali=True,
CNN=False,
reverse=False):
"""
Either load or generate synthetic training, real test data...
Load synthetic training, real test data, do multi-class SVM
(basically just this: http://scikit-learn.org/stable/auto_examples/classification/plot_digits_classification.html)
If reverse = True: do TRTS
"""
print('Running TSTR on', identifier, 'at epoch', epoch)
if vali:
test_set = 'vali'
else:
test_set = 'test'
if generate:
data = np.load('./experiments/data/' + identifier + '.data.npy').item()
samples = data['samples']
train_X = samples['train']
test_X = samples[test_set]
labels = data['labels']
train_Y = labels['train']
test_Y = labels[test_set]
# now sample from the model
synth_Y = np.tile(train_Y, [duplicate_synth, 1])
synth_X = model.sample_trained_model(identifier,
epoch,
num_samples=synth_Y.shape[0],
C_samples=synth_Y)
# for use in TRTS
synth_testX = model.sample_trained_model(identifier,
epoch,
num_samples=test_Y.shape[0],
C_samples=test_Y)
synth_data = {
'samples': synth_X,
'labels': synth_Y,
'test_samples': synth_testX,
'test_labels': test_Y
}
np.save(
'./experiments/tstr/' + identifier + '_' + str(epoch) +
'.data.npy', synth_data)
else:
print('Loading synthetic data from pre-sampled model')
exp_data = np.load('./experiments/tstr/' + identifier + '_' +
str(epoch) + '.data.npy').item()
test_X, test_Y = exp_data['test_data'], exp_data['test_labels']
train_X, train_Y = exp_data['train_data'], exp_data['train_labels']
synth_X, synth_Y = exp_data['synth_data'], exp_data['synth_labels']
if reverse:
which_setting = 'trts'
print('Swapping synthetic test set in for real, to do TRTS!')
test_X = synth_testX
else:
print('Doing normal TSTR')
which_setting = 'tstr'
# make classifier
if not CNN:
model_choice = 'RF'
# if multivariate, reshape
if len(test_X.shape) == 3:
test_X = test_X.reshape(test_X.shape[0], -1)
if len(train_X.shape) == 3:
train_X = train_X.reshape(train_X.shape[0], -1)
if len(synth_X.shape) == 3:
synth_X = synth_X.reshape(synth_X.shape[0], -1)
# if one hot, fix
if len(synth_Y.shape) > 1 and not synth_Y.shape[1] == 1:
synth_Y = np.argmax(synth_Y, axis=1)
train_Y = np.argmax(train_Y, axis=1)
test_Y = np.argmax(test_Y, axis=1)
# random forest
#synth_classifier = SVC(gamma=0.001)
#real_classifier = SVC(gamma=0.001)
synth_classifier = RandomForestClassifier(n_estimators=500)
real_classifier = RandomForestClassifier(n_estimators=500)
# fit
real_classifier.fit(train_X, train_Y)
synth_classifier.fit(synth_X, synth_Y)
# test on real
synth_predY = synth_classifier.predict(test_X)
real_predY = real_classifier.predict(test_X)
else:
model_choice = 'CNN'
synth_predY = train_CNN(synth_X, synth_Y, samples['vali'],
labels['vali'], test_X)
clear_session()
real_predY = train_CNN(train_X, train_Y, samples['vali'],
labels['vali'], test_X)
clear_session()
# CNN setting is all 'one-hot'
test_Y = np.argmax(test_Y, axis=1)
synth_predY = np.argmax(synth_predY, axis=1)
real_predY = np.argmax(real_predY, axis=1)
# report on results
synth_prec, synth_recall, synth_f1, synth_support = precision_recall_fscore_support(
test_Y, synth_predY, average='weighted')
synth_accuracy = accuracy_score(test_Y, synth_predY)
synth_auprc = 'NaN'
synth_auroc = 'NaN'
synth_scores = [
synth_prec, synth_recall, synth_f1, synth_accuracy, synth_auprc,
synth_auroc
]
real_prec, real_recall, real_f1, real_support = precision_recall_fscore_support(
test_Y, real_predY, average='weighted')
real_accuracy = accuracy_score(test_Y, real_predY)
real_auprc = 'NaN'
real_auroc = 'NaN'
real_scores = [
real_prec, real_recall, real_f1, real_accuracy, real_auprc, real_auroc
]
all_scores = synth_scores + real_scores
if vali:
report_file = open(
'./experiments/tstr/vali.' + which_setting + '_report.v3.csv', 'a')
report_file.write('mnist,' + identifier + ',' + model_choice + ',' +
str(epoch) + ',' + ','.join(map(str, all_scores)) +
'\n')
report_file.close()
else:
report_file = open(
'./experiments/tstr/' + which_setting + '_report.v3.csv', 'a')
report_file.write('mnist,' + identifier + ',' + model_choice + ',' +
str(epoch) + ',' + ','.join(map(str, all_scores)) +
'\n')
report_file.close()
# visualise results
try:
plotting.view_mnist_eval(identifier + '_' + str(epoch), train_X,
train_Y, synth_X, synth_Y, test_X, test_Y,
synth_predY, real_predY)
except ValueError:
print('PLOTTING ERROR')
pdb.set_trace()
print(classification_report(test_Y, synth_predY))
print(classification_report(test_Y, real_predY))
return synth_f1, real_f1
def main():
global NUM_PARENT_NETWORKS
global CHILDREN_PER_PARENT
global NUM_MUTATION_WEIGHTS
global MUTATION_FACTOR
env = gym.make('CartPole-v1')
env.reset()
# time.sleep(0.5)
gym_img = env.render(mode='rgb_array') #current_window_img(WINDOW_OFFSET)
gym_img = rgb2gray(gym_img)
gym_img = gym_img[150:350, 200:400]
gym_img = resize(gym_img, (25, 25))
# exit()
gym_img = gym_img.astype('float32') / 255.0
img_tensor = np.array(gym_img, dtype='float')
img_tensor = img_tensor.reshape(
(img_tensor.shape[0], img_tensor.shape[1], 1))
for _ in range(NUM_PREVIOUS_USING_STATES):
img_tensor = np.append(img_tensor, img_tensor[:, :, 0:1], axis=2)
for i in range(NUM_PARENT_NETWORKS):
nn = generate_model(img_tensor.shape)
nn.save('nn' + str(i) + '.h5')
K.clear_session()
gc.collect()
K.clear_session()
gc.collect()
# nnetworks = [generate_model(img_tensor.shape)
# for i in range(NUM_PARENT_NETWORKS)]
nn = models.load_model('nn0.h5')
layers_info = []
for i in range(len(nn.layers)):
layers_info.append(Weights(nn.layers[i]))
max_reward = 0
for gen_idx in range(NUM_GENERATION):
print('Generation {}'.format(gen_idx))
with open('GAConfig.txt') as cfg:
NUM_PARENT_NETWORKS = int(cfg.readline())
CHILDREN_PER_PARENT = int(cfg.readline())
NUM_MUTATION_WEIGHTS = int(cfg.readline())
MUTATION_FACTOR = np.float32(float(cfg.readline()))
print(NUM_PARENT_NETWORKS, CHILDREN_PER_PARENT,
NUM_MUTATION_WEIGHTS, MUTATION_FACTOR)
for net_idx in range(NUM_PARENT_NETWORKS):
for child_idx in range(CHILDREN_PER_PARENT):
partner_idx = get_partner_idx(net_idx, NUM_PARENT_NETWORKS)
nn_parent1 = models.load_model('nn' + str(net_idx) + '.h5')
nn_parent2 = models.load_model('nn' + str(partner_idx) + '.h5')
child_model = generate_child(nn_parent1, nn_parent2,
img_tensor.shape, layers_info)
safe_idx = NUM_PARENT_NETWORKS + net_idx * CHILDREN_PER_PARENT + child_idx
child_model.save('nn' + str(safe_idx) + '.h5')
K.clear_session()
gc.collect()
K.clear_session()
gc.collect()
# nnetworks.append(child_model)
num_networks = NUM_PARENT_NETWORKS + CHILDREN_PER_PARENT * NUM_PARENT_NETWORKS
rewards = [0 for i in range(num_networks)]
for network_idx in range(num_networks):
current_nn = models.load_model('nn' + str(network_idx) + '.h5')
run_results = np.array([])
for start_id in range(NUM_STARTS_FOR_AVRG):
reward = 0
env.reset()
prev_states = np.zeros(
(img_tensor.shape[0], img_tensor.shape[1],
img_tensor.shape[2] - 1))
# for i in range(img_tensor.shape[2] - 1):
# prev_states[:,:,i:i+1] = img_tensor[:,:,0:1]
while reward < MAX_REWARD:
env.render()
gym_img = env.render(
mode='rgb_array') #current_window_img(WINDOW_OFFSET)
gym_img = rgb2gray(gym_img)
gym_img = gym_img[150:350, 200:400]
gym_img = resize(gym_img, (25, 25))
gym_img = gym_img.astype('float32') / 255.0
gym_tensor = np.array(gym_img, dtype='float')
gym_tensor = gym_tensor.reshape(
(gym_tensor.shape[0], gym_tensor.shape[1], 1))
for i in range(NUM_PREVIOUS_USING_STATES):
gym_tensor = np.append(gym_tensor,
prev_states[:, :, i:i + 1],
axis=2)
gym_tensor = np.expand_dims(gym_tensor, axis=0)
predict = current_nn.predict(gym_tensor)
action = 0 if predict[0][0] < 0.5 else 1
_, _, done, _ = env.step(action)
reward += 1
if done:
run_results = np.append(run_results, reward)
break
else:
# if reward % 2 == 0:
update_prev_states(prev_states, gym_tensor[:, :, :,
0:1])
rewards[network_idx] = int(np.mean(run_results))
if max_reward < max(rewards):
max_reward = max(rewards)
with open("max_reward.txt", "w") as f:
f.writelines(['MAX REWARD COMMON: {}'.format(max_reward)])
current_nn.save('best_network.h5')
print('Network {}: {}'.format(network_idx, rewards[network_idx]))
print('-' * 40)
print('MAX REWARD CURRENT: {}'.format(max(rewards)))
print('MAX REWARD COMMON: {}'.format(max_reward))
print('-' * 40)
nnetworks = selection(num_networks, rewards, NUM_PARENT_NETWORKS,
RANDOM_SELECTED_NETWORKS)
# for i in range(len(nnetworks)):
# nnetworks[i].save('tmp'+str(i) + '.h5')
# nnetworks.clear()
K.clear_session()
gc.collect()
def main(args):
try:
if not args.model_load:
raise ValueError()
audio_dir = args.audio_dir
print "\nReading test data: "
_, df = combine_all_wavs_and_trans_from_csvs(audio_dir)
batch_size = args.batch_size
batch_index = args.batch_index
mfcc_features = args.mfccs
n_mels = args.mels
frequency = 16 # Sampling rate of data in khz (LibriSpeech is 16khz)
# Training data_params:
model_load = args.model_load
load_multi = args.load_multi
# Sets the full dataset in audio_dir to be available through data_generator
# The data_generator doesn't actually load the audio files until they are requested through __get_item__()
epoch_length = 0
# Load trained model
# When loading custom objects, Keras needs to know where to find them.
# The CTC lambda is a dummy function
custom_objects = {'clipped_relu': models.clipped_relu,
'<lambda>': lambda y_true, y_pred: y_pred}
# When loading a parallel model saved *while* running on GPU, use load_multi
if load_multi:
model = models.load_model(model_load, custom_objects=custom_objects)
model = model.layers[-2]
print "\nLoaded existing model: ", model_load
# Load single GPU/CPU model or model saved *after* finished training
else:
model = models.load_model(model_load, custom_objects=custom_objects)
print "\nLoaded existing model: ", model_load
# Dummy loss-function to compile model, actual CTC loss-function defined as a lambda layer in model
loss = {'ctc': lambda y_true, y_pred: y_pred}
model.compile(loss=loss, optimizer='Adam')
feature_shape = model.input_shape[0][2]
# Model feature type
if not args.feature_type:
if feature_shape == 26:
feature_type = 'mfcc'
else:
feature_type = 'spectrogram'
else:
feature_type = args.feature_type
print "Feature type: ", feature_type
# Data generation parameters
data_params = {'feature_type': feature_type,
'batch_size': batch_size,
'frame_length': 20 * frequency,
'hop_length': 10 * frequency,
'mfcc_features': mfcc_features,
'n_mels': n_mels,
'epoch_length': epoch_length,
'shuffle': False
}
# Data generators for training, validation and testing data
data_generator = DataGenerator(df, **data_params)
# Print model summary
model.summary()
# Creates a test function that takes preprocessed sound input and outputs predictions
# Used to calculate WER while training the network
input_data = model.get_layer('the_input').input
y_pred = model.get_layer('ctc').input[0]
test_func = K.function([input_data], [y_pred])
if args.calc_wer:
print "\n - Calculation WER on ", audio_dir
wer = calc_wer(test_func, data_generator)
print "Average WER: ", wer[1]
predictions = predict_on_batch(data_generator, test_func, batch_index)
print "\n - Predictions from batch index: ", batch_index, "\nFrom: ", audio_dir, "\n"
for i in predictions:
print "Original: ", i[0]
print "Predicted: ", i[1], "\n"
except (Exception, StandardError, GeneratorExit, SystemExit) as e:
template = "An exception of type {0} occurred. Arguments:\n{1!r}"
message = template.format(type(e).__name__, e.args)
print "e.args: ", e.args
print message
finally:
# Clear memory
K.clear_session()
def main(argv=None):
'''
'''
main.__doc__ = __doc__
argv = sys.argv if argv is None else sys.argv.extend(argv)
desc = main.__doc__ # .format(os.path.basename(__file__))
# CLI parser
args = parser_(desc)
mgpu = 0 if getattr(args, 'mgpu', None) is None else args.mgpu
checkpt = getattr(args, 'checkpt', None)
checkpt_flag = False if checkpt is None else True
filepath = checkpt
# print('CHECKPT:', checkpt)
gdev_list = get_available_gpus(mgpu or 1)
ngpus = len(gdev_list)
batch_size_1gpu = 32
batch_size = batch_size_1gpu * ngpus
num_classes = 1000
epochs = args.epochs
data_augmentation = args.aug
logdevp = args.logdevp
datadir = getattr(args, 'datadir', None)
# The data, shuffled and split between train and test sets:
(x_train, y_train), (x_test,
y_test) = synthesize_imagenet_dataset(num_classes)
train_samples = x_train.shape[0]
test_samples = y_test.shape[0]
steps_per_epoch = train_samples // batch_size
print('train_samples:', train_samples)
print('batch_size:', batch_size)
print('steps_per_epoch:', steps_per_epoch)
# validations_steps = test_samples // batch_size
print(x_train.shape[0], 'train samples')
print(x_test.shape[0], 'test samples')
# The capacity variable controls the maximum queue size
# allowed when prefetching data for training.
capacity = 10000
# min_after_dequeue is the minimum number elements in the queue
# after a dequeue, which ensures sufficient mixing of elements.
# min_after_dequeue = 3000
# If `enqueue_many` is `False`, `tensors` is assumed to represent a
# single example. An input tensor with shape `[x, y, z]` will be output
# as a tensor with shape `[batch_size, x, y, z]`.
#
# If `enqueue_many` is `True`, `tensors` is assumed to represent a
# batch of examples, where the first dimension is indexed by example,
# and all members of `tensors` should have the same size in the
# first dimension. If an input tensor has shape `[*, x, y, z]`, the
# output will have shape `[batch_size, x, y, z]`.
# enqueue_many = True
# Force input pipeline to CPU:0 to avoid data operations ending up on GPU
# and resulting in a slow down for multigpu case due to comm overhead.
with tf.device('/cpu:0'):
# if no augmentation can go directly from numpy arrays
# x_train_batch, y_train_batch = tf.train.shuffle_batch(
# tensors=[x_train, y_train],
# # tensors=[x_train, y_train.astype(np.int32)],
# batch_size=batch_size,
# capacity=capacity,
# min_after_dequeue=min_after_dequeue,
# enqueue_many=enqueue_many,
# num_threads=8)
# NOTE: This bakes the whole dataset into the TF graph and for larger
# datasets it fails on "ValueError: GraphDef cannot be larger than 2GB".
# TODO: Load the a large dataset via queue from RAM/disk.
input_images = tf.constant(x_train.reshape(train_samples, -1))
print('train_samples', train_samples)
print('input_images', input_images.shape)
image, label = tf.train.slice_input_producer([input_images, y_train],
shuffle=True)
# If using num_epochs=epochs have to:
# sess.run(tf.local_variables_initializer())
# and maybe also: sess.run(tf.global_variables_initializer())
image = tf.reshape(image, x_train.shape[1:])
print('image', image.shape)
test_images = tf.constant(x_test.reshape(test_samples, -1))
test_image, test_label = tf.train.slice_input_producer(
[test_images, y_test], shuffle=False)
test_image = tf.reshape(test_image, x_train.shape[1:])
if data_augmentation:
print('Using real-time data augmentation.')
# Randomly flip the image horizontally.
distorted_image = tf.image.random_flip_left_right(image)
# Because these operations are not commutative, consider
# randomizing the order their operation.
# NOTE: since per_image_standardization zeros the mean and
# makes the stddev unit, this likely has no effect see
# tensorflow#1458.
distorted_image = tf.image.random_brightness(distorted_image,
max_delta=63)
distorted_image = tf.image.random_contrast(distorted_image,
lower=0.2,
upper=1.8)
# Subtract off the mean and divide by the variance of the
# pixels.
image = tf.image.per_image_standardization(distorted_image)
# Do this for testing as well if standardizing
test_image = tf.image.per_image_standardization(test_image)
# Use tf.train.batch if slice_input_producer shuffle=True,
# otherwise use tf.train.shuffle_batch. Not sure which way is faster.
x_train_batch, y_train_batch = tf.train.batch([image, label],
batch_size=batch_size,
capacity=capacity,
num_threads=8)
print('x_train_batch:', x_train_batch.shape)
# x_train_batch, y_train_batch = tf.train.shuffle_batch(
# tensors=[image, label],
# batch_size=batch_size,
# capacity=capacity,
# min_after_dequeue=min_after_dequeue,
# num_threads=8)
x_test_batch, y_test_batch = tf.train.batch(
[test_image, test_label],
# TODO: shouldn't it be: batch_size=batch_size???
batch_size=train_samples,
capacity=capacity,
num_threads=8,
name='test_batch',
shared_name='test_batch')
x_train_input = KL.Input(tensor=x_train_batch)
print('x_train_input', x_train_input)
gauge = SamplesPerSec(batch_size)
callbacks = [gauge]
if _DEVPROF or logdevp: # or True:
# Setup Keras session using Tensorflow
config = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=True)
# config.gpu_options.allow_growth = True
tfsess = tf.Session(config=config)
KB.set_session(tfsess)
model_init = make_model(x_train_input, num_classes)
x_train_out = model_init.output
# model_init.summary()
lr = 0.0001 * ngpus
if ngpus > 1:
model = make_parallel(model_init, gdev_list)
else:
# Must re-instantiate model per API below otherwise doesn't work.
model_init = Model(inputs=[x_train_input], outputs=[x_train_out])
model = model_init
opt = RMSprop(lr=lr, decay=1e-6)
# Let's train the model using RMSprop
model.compile(loss='categorical_crossentropy',
optimizer=opt,
metrics=['accuracy'],
target_tensors=[y_train_batch])
print_mgpu_modelsummary(model) # will print non-mgpu model as well
if checkpt_flag:
checkpoint = ModelCheckpoint(filepath,
monitor='acc',
verbose=1,
save_best_only=True)
callbacks = [checkpoint]
# Start the queue runners.
sess = KB.get_session()
# sess.run([tf.local_variables_initializer(),
# tf.global_variables_initializer()])
tf.train.start_queue_runners(sess=sess)
# Fit the model using data from the TFRecord data tensors.
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess, coord)
val_in_train = False # not sure how the validation part works during fit.
start_time = time.time()
model.fit(
# validation_data=(x_test_batch, y_test_batch)
# if val_in_train else None, # validation data is not used???
# validation_steps=validations_steps if val_in_train else None,
validation_steps=val_in_train,
steps_per_epoch=steps_per_epoch,
epochs=epochs,
callbacks=callbacks)
elapsed_time = time.time() - start_time
print('[{}] finished in {} ms'.format('TRAINING',
int(elapsed_time * 1000)))
gauge.print_results()
weights_file = checkptfile # './saved_cifar10_wt.h5'
if not checkpt_flag: # empty list
model.save_weights(checkptfile)
# Clean up the TF session.
coord.request_stop()
coord.join(threads)
KB.clear_session()
# Second Session. Demonstrate that the model works
# test_model = make_model(x_test.shape[1:], num_classes,
# weights_file=weights_file)
test_model = make_model(x_test.shape[1:], num_classes)
test_model.load_weights(weights_file)
test_model.compile(loss='categorical_crossentropy',
optimizer=opt,
metrics=['accuracy'])
if data_augmentation:
x_proccessed = sess.run(x_test_batch)
y_proccessed = sess.run(y_test_batch)
loss, acc = test_model.evaluate(x_proccessed, y_proccessed)
else:
loss, acc = test_model.evaluate(x_test, y_test)
print('\nTest loss: {0}'.format(loss))
print('\nTest accuracy: {0}'.format(acc))
def profit(code):
from keras.models import load_model
# import tensorflow as tf
# global graph, model
# graph = tf.get_default_graph()
K.clear_session()
UP_FOLDER = os.path.dirname(os.path.abspath(os.path.abspath(__file__)))
codelist = os.path.join(UP_FOLDER,
'static\\stock\\Data\\codelist-1.xlsx')
con = os.path.join(UP_FOLDER,
'static\\stock\\Data\\stock_notna_ffill.db')
conn = os.path.join(UP_FOLDER,
'static\\stock\\Data\\stock_external_최종.db')
codelist1 = pd.read_excel(codelist)
codelist = codelist1[['종목코드', '종목명']]
name = codelist[codelist['종목코드'] == code]['종목명'].values[0]
# codelist1 = codelist1['종목코드']
# codelist1 = codelist1.values.tolist()
con = sqlite3.connect(con)
conn = sqlite3.connect(conn)
df = Post.stock_chart(code)
Xtest, Ytest, scaler = dataset1W(con, conn, code, look_back=25)
model7 = os.path.join(UP_FOLDER,
'static\\stock\\model1W\\%s_1W.h5' % code)
model7 = load_model(model7)
x = Xtest[len(Xtest) - 7:]
pre = model7.predict(x)
del model7
y = Post.stock_chart(code)
y = y.values.tolist()
Pre = scaler.inverse_transform(pre)
past7 = y[7]
cur7 = y[-1]
gap = Pre[-1] - Pre[0]
future7 = cur7 + gap
future7 = round(future7[0], 0)
futurerate7 = float(future7 / cur7)
futurerate7 = round(futurerate7, 3)
list7 = []
list7 = [float(past7), float(cur7), float(future7)]
Xtest, Ytest, scaler = dataset15D(con, conn, code, look_back=30)
model15 = os.path.join(UP_FOLDER,
'static\\stock\\model15D\\%s_15D.h5' % code)
model15 = load_model(model15)
x = Xtest[len(Xtest) - 15:]
pre = model15.predict(x)
del model15
# Ytest = Ytest.reshape(-1,1)
# Y = scaler.inverse_transform(Ytest)
# y = y.values.tolist()
Pre = scaler.inverse_transform(pre)
past15 = y[15]
cur15 = y[-1]
gap = Pre[-1] - Pre[0]
future15 = cur15 + gap
future15 = round(future15[0], 0)
futurerate15 = float(future15 / cur15)
futurerate15 = round(futurerate15, 3)
list15 = []
list15 = [float(past15), float(cur15), float(future15)]
Xtest, Ytest, scaler = dataset20D(con, conn, code, look_back=60)
model20 = os.path.join(UP_FOLDER,
'static\\stock\\model20D\\%s_20D.h5' % code)
model20 = load_model(model20)
x = Xtest[len(Xtest) - 20:]
pre = model20.predict(x)
del model20
# Ytest = Ytest.reshape(-1, 1)
# Y = scaler.inverse_transform(Ytest)
# y = y.values.tolist()
Pre = scaler.inverse_transform(pre)
past20 = y[20]
cur20 = y[-1]
gap = Pre[-1] - Pre[0]
future20 = cur20 + gap
future20 = round(future20[0], 0)
futurerate20 = float(future20 / cur20)
futurerate20 = round(futurerate20, 3)
list20 = []
list20 = [float(past20), float(cur20), float(future20)]
# Xtest, Ytest, scaler = dataset15D(con, conn, code, look_back=30)
# model15 = os.path.join(UP_FOLDER, 'static\\stock\\model15D\\%s_15D.h5' % code)
# img1W = os.path.join(UP_FOLDER, 'static\\stock\\preimg\\%s_15D_pre.png' % code)
# model15 = load_model(model15)
# x = Xtest[len(Xtest) - 15:]
# pre = model15.predict(x)
# Pre = scaler.inverse_transform(pre)
# gap = Pre[-1] - Pre[0]
# pre = [df[-1], df[-1] + gap]
# profit = Pre[-1] - Pre[0]
# profit_list.append(profit)
# profit_df = pd.DataFrame(profit_list)
# profit_df.to_excel("D://lec403//로보플젝데이터//%s_profit.xlsx" % i)
return futurerate7, list7, futurerate15, list15, futurerate20, list20, name
callbacks,
batch_size=32,
nbr_type=GP['nbr_type'],
save_path=GP['save_path'],
len_molecular_hidden_layers=len_molecular_hidden_layers,
molecular_nbrs=molecular_nbrs,
conv_bool=conv_bool,
full_conv_bool=full_conv_bool,
type_bool=GP['type_bool'],
sampling_density=GP['sampling_density'])
frame_loss, frame_mse = ct.train_ac()
else:
frame_mse = []
frame_loss = []
return frame_loss, frame_mse
def main():
gParameters = initialize_parameters()
run(gParameters)
if __name__ == '__main__':
main()
try:
K.clear_session()
except AttributeError: # theano does not have this function
pass
def mlp(train_data, test_data, train_outcomes, test_outcomes, fold_num,
results_dir):
model = Sequential()
model.add(
Dense(32,
activation='relu',
kernel_constrain=max_norm(),
input_shape=(train_data.shape[1], )))
model.add(BatchNormalization())
model.add(Dropout(0.5))
model.add(Dense(32, kernel_constraint=max_norm(), activation='relu'))
# model.add(BatchNormalization())
model.add(Dropout(0.5))
model.add(Dense(2, activation='softmax'))
model_checkpoint = ModelCheckpoint(results_dir + "fold_" + str(fold_num) +
"_best_weights.hdf5",
monitor="categorical_accuracy",
save_best_only=True)
adam = Adam(lr=0.0002,
beta_1=0.9,
beta_2=0.999,
epsilon=None,
decay=1e-5,
amsgrad=False)
model.compile(optimizer=adam,
loss='categorical_crossentropy',
metrics=['categorical_accuracy'])
# model.summary()
train_labels = to_categorical(train_outcomes, num_classes=2)
test_labels = to_categorical(test_outcomes, num_classes=2)
class_weights = class_weight.compute_class_weight(
'balanced', np.unique(train_outcomes), train_outcomes)
hist = model.fit(train_data,
train_labels,
batch_size=128,
epochs=1200,
validation_data=(test_data, test_labels),
callbacks=[model_checkpoint],
verbose=False,
class_weight=class_weights)
# print(model.metrics_names)
plt.figure(figsize=(6, 6))
plt.plot(hist['categorical_accuracy'], label='Training')
plt.plot(hist['val_categorical_accuracy'], label='Validation')
plt.legend(loc='lower right', textsize=20)
plt.xlabel('Epoch #', fontsize=20)
plt.ylabel('% accuracy', fontsize=20)
plt.savefig(results_dir + 'fold_' + str(fold_num) + '_mlp_accuracy.png',
dpi=500)
model.load_weights(results_dir + "fold_" + str(fold_num) +
"_best_weights.hdf5")
model.save(results_dir + 'best_bol_model' + str(fold_num) + '.hdf5')
deep_probabilities = model.predict_proba(test_data)
train_labels = to_categorical(train_outcomes, num_classes=2)
explainer = lime.lime_tabular.LimeTabularExplainer(
train_data,
training_labels=train_labels,
discretize_continuous=True,
discretizer='quartile',
class_names=['Inactive', 'Active'])
lime_type_importance = np.zeros((train_data.shape[1]))
for i in range(test_data.shape[0]):
prediction = model.predict(test_data[i, ...][np.newaxis, ...])
if prediction[0][1] > 0.5:
prediction = 1
label = ['Active']
else:
prediction = 0
label = ['Inactive']
if test_outcomes[i] == prediction:
exp = explainer.explain_instance(test_data[i, ...],
model.predict_proba,
num_features=10,
labels=[prediction])
exp.save_to_file(results_dir + 'explanation' + str(fold_num) +
'-' + str(i) + '.html')
important_types = exp.as_map()
# print('types', important_types)
fig = exp.as_pyplot_figure(label=prediction)
plt.tight_layout()
fig.savefig(results_dir + str(fold_num) + '_' + str(i) +
'_explained.png')
for lesion_type in important_types[prediction]:
lime_type_importance[lesion_type[0]] += lesion_type[1]
K.clear_session()
return deep_probabilities, model, lime_type_importance
def deep_learning_experiment_vector(param, train, test, label_info):
nb_class = label_info[0]
nb_people = label_info[1]
param.nb_modal = 3
if param.method == method_select['people']:
nb_repeat = nb_people
elif param.method in method_select['repeat']:
nb_repeat = 20
elif param.method in method_select["CrossValidation"]:
nb_repeat = param.collect["CrossValidation"] * 5
# config = tf.ConfigProto()
# config.gpu_options.allow_growth = True
for repeat in range(nb_repeat):
print(f"{dt()} :: {repeat+1}/{nb_repeat} experiment progress")
tartr = train[repeat]
tarte = test[repeat]
tr_data = [tartr["data_0"], tartr["data_1"], tartr["data_2"]]
te_data = [tarte["data_0"], tarte["data_1"], tarte["data_2"]]
if param.datatype == "type":
tr_label = tartr["tag"] - 1
te_label = tarte["tag"] - 1
nb_class = label_info[0]
elif param.datatype == "disease":
tr_label = tartr["tag"]
te_label = tarte["tag"]
nb_class = label_info[0]
cat_tr = preprocessing.to_categorical(tr_label, nb_class)
cat_te = preprocessing.to_categorical(te_label, nb_class)
model = model_compactor.model_setting(param, train[repeat],
test[repeat],
[nb_class, nb_people])
print(f"{dt()} :: MODEL={param.model_name}, METHOD={param.method}")
log_dir = f"../Log/{param.model_name}_{param.method}"
# log_dir = f"/home/blackcow/mlpa/workspace/gait-rework/gait-rework/Log/{param.model_name}_{param.method}"
# tb_hist = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=0, write_graph=True, write_images=True)
# # model.summary()
# model_result = model.fit(x=tr_data, y=cat_tr, epochs=param.epochs, batch_size=param.batch_size
# , validation_data=(te_data, cat_te), verbose=2, callbacks=[tb_hist])
# model_score = model.evaluate(x=te_data, y=cat_te, verbose=0)
while True:
x_train1 = list()
x_train2 = list()
x_train3 = list()
y_train = list()
print(f"total batch : {len(tr_data[0]) // param.batch_size}")
for i in range(len(tr_data[0]) // param.batch_size):
x_batch1 = tr_data[0][i * param.batch_size:(i + 1) *
param.batch_size]
x_batch2 = tr_data[1][i * param.batch_size:(i + 1) *
param.batch_size]
x_batch3 = tr_data[2][i * param.batch_size:(i + 1) *
param.batch_size]
x_train1.append(x_batch1)
x_train2.append(x_batch2)
x_train3.append(x_batch3)
y_train.append(cat_tr[i * param.batch_size:(i + 1) *
param.batch_size])
model.summary()
optimizer = tf.optimizers.Adam(lr=0.0001)
loss_object = tf.keras.losses.CategoricalCrossentropy()
fin_loss_object = tf.keras.losses.CategoricalCrossentropy()
train_loss = tf.keras.metrics.Mean(name='train_loss')
train_accuracy = tf.keras.metrics.CategoricalAccuracy(
name='train_accuracy')
test_loss = tf.keras.metrics.Mean(name='test_loss')
test_accuracy = tf.keras.metrics.CategoricalAccuracy(
name='test_accuracy')
for epoch in range(param.epochs):
for step, (x_batch1, x_batch2, x_batch3, y_batch) in enumerate(
zip(x_train1, x_train2, x_train3, y_train)):
# predicted = model.predict([x_batch1, x_batch2, x_batch3])
with tf.GradientTape() as tape:
logits = model([x_batch1, x_batch2, x_batch3])
loss_val1 = loss_object(y_batch, logits[0])
loss_val2 = loss_object(y_batch, logits[1])
loss_val3 = loss_object(y_batch, logits[2])
true_loss = tf.math.add(logits[0] * 0.3,
logits[1] * 0.3,
logits[2] * 0.3)
true_loss = fin_loss_object(y_batch, logits[6])
# gen = model.train_on_batch(, [y_batch, y_batch, y_batch])
# print(f'train_loss : {gen}')
grads = tape.gradient(true_loss, model.trainable_variables)
optimizer.apply_gradients(
(grads, var)
for (grads,
var) in zip(grads, model.trainable_variables)
if grads is not None)
tr_loss = train_loss(true_loss)
tr_acc1 = train_accuracy(y_batch, logits[0])
tr_acc2 = train_accuracy(y_batch, logits[1])
tr_acc3 = train_accuracy(y_batch, logits[2])
tr_acc4 = train_accuracy(y_batch, logits[6])
sim_images = np.reshape(logits[3], (-1, 128, 128, 1))
logdir = f"../Log/similarity_matrix/{datetime.datetime.now().strftime('%Y%m%d-%H%M%S')}"
file_writer = tf.summary.create_file_writer(logdir)
with file_writer.as_default():
tf.summary.scalar("train_loss", tr_loss, step=epoch)
tf.summary.scalar("train_acc", tr_acc4, step=epoch)
tf.summary.image("Similarity Matrix",
sim_images,
step=epoch,
max_outputs=12)
print(
f'[step : {step}/{len(x_train1)}] [epochs : {epoch}/{param.epochs}]'
f'train loss : {tr_loss}, domain 1-3_accuracy : {tr_acc1*100}, {tr_acc2*100}, {tr_acc3*100}'
)
print(
f'train merge acc : {tr_acc4*100} test loss : not implemented...'
)
model.evaluate([te_data[0], te_data[1], te_data[2]])
if repeat == 0:
tracking = [
dt(), param.method, param.model_name, param.nb_combine, repeat,
model_score[0], model_score[1]
]
ds.stock_result(tracking)
else:
tracking = [dt(), repeat, model_score[0], model_score[1]]
ds.stock_result(tracking)
ds.save_result_obo(param, tracking)
model_result = None
model_score = None
tracking = None
tr_data = None
te_date = None
K.clear_session()
tf.keras.backend.clear_session()
sess.close()
def full_lstmnet(i):
print('Full Processing')
data_1_train = data_1
# data_2_train = data_2
train_data_reg_train = train_data_reg
train_data_city_train = train_data_city
train_data_cat1_train = train_data_cat1
train_data_cat2_train = train_data_cat2
train_data_prm1_train = train_data_prm1
train_data_prm2_train = train_data_prm2
# train_data_prm3_train = train_data_prm3
# train_data_sqnm_train = train_data_sqnm
train_data_usr_train = train_data_usr
train_data_itype_train = train_data_itype
train_user_features_train = train_user_features
labels_train = labels
pred_test = np.zeros(test_df.shape[0])
for j in range(1, nbags + 1):
print('bag ', j, ' Processing')
model = gru_Bidirectional_selfEmbedding_model()
# model = gru_attention_model()
# callbacks = [
# EarlyStopping(monitor='val_loss', patience=patience, verbose=VERBOSEFLAG),
# ModelCheckpoint(MODEL_WEIGHTS_FILE, monitor='val_loss', save_best_only=True, verbose=VERBOSEFLAG),
# ]
# reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=patience, min_lr=1e-9, epsilon = 0.00001, verbose=VERBOSEFLAG)
fullepochs = 12
learning_rate = 1e-3
#sgd = SGD(lr=1e-3, decay=1e-6, momentum=0.9, nesterov=True)
if optim_type == 'SGD':
optim = SGD(lr=learning_rate,
decay=1e-6,
momentum=0.9,
nesterov=True)
else:
optim = Adam(lr=learning_rate)
model.compile(optimizer=optim,
loss='mean_squared_error',
metrics=[root_mean_squared_error])
model.fit(
[
data_1_train, train_data_reg_train, train_data_city_train,
train_data_cat1_train, train_data_cat2_train,
train_data_prm1_train, train_data_prm2_train,
train_data_usr_train, train_data_itype_train,
train_user_features_train
],
labels_train,
batch_size=batch_size,
nb_epoch=fullepochs,
#callbacks = callbacks, #[callbacks, reduce_lr],
verbose=VERBOSEFLAG)
pred_test += model.predict([
test_data_1, test_data_reg, test_data_city, test_data_cat1,
test_data_cat2, test_data_prm1, test_data_prm2, test_data_usr,
test_data_itype, test_user_features
],
batch_size=batch_size,
verbose=VERBOSEFLAG)[:, 0]
del model
K.clear_session()
gc.collect()
pred_test /= nbags
pred_test = pd.DataFrame(pred_test)
pred_test.columns = ["deal_probability"]
pred_test["item_id"] = test_df.item_id.values
pred_test = pred_test[['item_id', 'deal_probability']]
pred_test['deal_probability'] = pred_test['deal_probability'].clip(
0.0, 1.0)
sub_file = inDir + '/submissions/Prav.nn10.full' + '.csv'
pred_test.to_csv(sub_file, index=False)
def main():
my_date_time = '_'.join(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S").split())
parameters = {
'dataset':dataset,
'n_folds':n_folds,
'n_repeats':n_repeats,
'batch_size':batch_size,
'nb_epochs':nb_epochs,
'my_patience':my_patience,
'drop_rate':drop_rate,
'my_optimizer':my_optimizer,
'my_loss_function': my_loss_function,
'dense_units':dense_units,
'layer_numer':layer_numer,
}
path_root = sys.path[0]
name_save = path_root + '/results/' + dataset + '_augmentation_' + my_date_time
print ('========== loading samples ==========')
samples = np.load("sample_all_Train10000_200.npy")
entry_model = Entry()
tf = samples[:,0,-1]
altitute = (samples[:,1,0] - entry_model.constant['R0'])/1000
downrange = samples[:,2,-1]*entry_model.constant['R0']/1000 - 100
velocity = samples[:,3,0]
gamma = samples[:,4,0]*180/np.pi
print ('Tf range: max:%.2f,min:%.2f, [s],'%(np.amax(tf),np.amin(tf)))
print ('Downrange range: max:%.2f,min:%.2f [km],'%(np.amax(downrange),np.amin(downrange)))
print ('Initial Altitute range: max:%.2f,min:%.2f,[km]'%(np.amax(altitute),np.amin(altitute)))
print ('Initial Velocity range:max:%d,min:%d, [m/s]'%(np.amax(velocity),np.amin(velocity)))
print ('Initial Flight Path Angle range: max:%.2f,min:%.2f,'%(np.amax(gamma),np.amin(gamma)))
alpha = samples[:,-1,:]
ys = alpha.flatten() #output row style
r = (samples[:,1,:].flatten() - entry_model.constant['R0'])/entry_model.constant['h0']
theta = samples[:,2,:].flatten()
v = samples[:,3,:].flatten()/ entry_model.constant['v0']
g = samples[:,4,:].flatten()
tensors = np.column_stack((r,theta,v,g))
tensors = tensors.astype(np.float32)
print ('input shape:', tensors.shape)
print ('========== shuffling data ==========')
shuffled_idxs = random.sample(range(tensors.shape[0]), int(tensors.shape[0])) # sample w/o replct
tensors = tensors[shuffled_idxs]
ys = ys[shuffled_idxs]
shuffled_idxs = np.array(shuffled_idxs)
print ('========== conducting', n_folds ,'fold cross validation ==========');
print ('repeating each fold:', n_repeats, 'times')
folds = np.array_split(tensors,n_folds,axis=0)
print ('fold sizes:', [fold.shape[0] for fold in folds])
folds_labels = np.array_split(ys,n_folds,axis=0)
outputs = []
histories = []
for i in range(n_folds):
t = time.time()
x_train = np.concatenate([fold for j,fold in enumerate(folds) if j!=i],axis=0)
x_test = [fold for j,fold in enumerate(folds) if j==i]
y_train = np.concatenate([y for j,y in enumerate(folds_labels) if j!=i],axis=0)
y_test = [y for j,y in enumerate(folds_labels) if j==i]
for repeating in range(n_repeats):
print ('clearing Keras session')
K.clear_session()
# instantiate model
model = Sequential()
model.add(Dense(dense_units,input_dim = 4))
for i in range(layer_numer):
model.add(Dense(dense_units,activation=activation_hidden))
model.add(Dropout(drop_rate))
model.add(Dense(1))
# configure model for training
model.compile(loss=my_loss_function,
optimizer=my_optimizer,
metrics=['mse', 'mae'])
print ('model compiled')
#early_stopping = EarlyStopping(monitor='val_acc', # go through epochs as long as acc on validation set increases
# patience=my_patience,
# mode='max')
history = model.fit(x_train,
y_train,
batch_size=batch_size,
nb_epoch=nb_epochs,
validation_data=(x_test, y_test),)
# callbacks=[early_stopping])
# save [min loss,max acc] on test set
max_acc = max(history.history['val_mean_absolute_error'])
max_idx = history.history['val_mean_absolute_error'].index(max_acc)
output = [history.history['val_loss'][max_idx],max_acc]
outputs.append(output)
# also save full history for sanity checking
histories.append(history.history)
print ('**** fold', i+1 ,'done in ' + str(math.ceil(time.time() - t)) + ' second(s) ****')
# save results to disk
with open(name_save + '_parameters.json', 'w') as my_file:
json.dump(parameters, my_file, sort_keys=True, indent=4)
print ('========== parameters defined and saved to disk ==========')
with open(name_save + '_results.json', 'w') as my_file:
json.dump({'outputs':outputs,'histories':histories}, my_file, sort_keys=False, indent=4)
print( '========== results saved to disk ==========')
def main(config):
use_data_augmentation = True
# This is the main configuration object
training_params = {
# Name of the split created
'split': 'name_of_split',
# Label: 'verb' or 'object'
'label': 'verb',
# Execute a quick run: 1 batch for training and 2 videos for test
'toy_execution': False,
# If the evaluation is already done and saved, whether to repeat it
'redo_evaluation': False,
# Oversample minority classes
'oversampling': False,
# Warm up: do 'warmup_epochs' epochs with learnign rate 'warmup_lr'
'use_warmup': False,
'warmup_epochs': 3,
'warmup_lr': 0.01,
# From 0 to 1, percentage of offset at the beginning and end to sample
'frame_sampling_offset': 0.,
# Number of times to run the experiment (averaged at the end)
'runs': 3,
'epochs': 100,
# Skip connection from last layer before ConvLSTM to output hidden
# states of the ConvLSTM. It requires a 1x1 conv with a number of
# channels equal to the number of hidden units of the ConvLSTM
'skip_connect': False,
# Number of timesteps
'sequence_length': 25,
'learning_rate': 0.0001,
'batch_size': 16,
# Number of layers to freeze, starting from 0
# 142 to freeze everything except for the last conv block
# 0 would set all layers as trainable
# -1 to freeze all
'last_layer_to_freeze_conv': 142,
'optimizer': 'adam',
# Maximum value to clip the gradient (to avoid large changes)
'gradient_clipvalue': 0.,
# Criterion to use for early stopping and also to choose the best model
'stop_criterion': 'val_f1_metric',
# Patience for early stopping
'patience': 10,
# Number of hidden states used in the ConvLSTM, i.e., number of
# output channels
'num_convlstms': 1,
'convlstm_hidden': 256,
'convlstm_add_initial_state': False,
'apply_conv_betweenconvlstm': False,
'apply_conv_afterconvlstm': False,
'last_layer_to_freeze': 0,
'non_uniform_sampling': False,
# Normalise input to the ConvLSTM with L2 Normalisation
'convlstm_normalise_input': False,
'dropout_rate': 0.,
'convlstm_dropout_rate': 0.,
'convlstm_recurrent_dropout_rate': 0.,
'spatial_dropout_rate': 0.,
'use_average_pool': True,
'use_data_augmentation': use_data_augmentation,
'random_horizontal_flipping': True,
'random_corner_cropping': True,
'random_lighting': False,
# Apply class weighting in the loss function using the training set
# class distribution as a prior
'apply_class_weighting': True,
# Regularisation L1/L2 in the loss. If both are used then L1_L2
# regularisation is used (keras)
'l1_reg_beta': 0.,
'l2_reg_beta': 0.,
# Add a 1x1 conv after the last conv block but before the non local
# block in order to reduce the number of channels (ch)
'add_1x1conv': True,
'add_1x1conv_ch': 256,
'min_frames': -1,
# Activates debug mode: inputs to the network are saved in the folder
# pointed out by 'debug_folder' below
'debug_mode': False,
'debug_folder': 'debug_folder',
'visualisation_mode': False
}
# Name of the experiment (e.g. split_R_verb_detector)
exp_name = 'split_{}_{}_detector'.format(
training_params['split'], training_params['label']
)
root_path = config['split_path'] + training_params['split'] + '/'
if training_params['label'] == 'verb':
training_params['num_classes'] = (
len(open(root_path + config['train_verbs_file'],
'r').readlines()))
training_params['train_classes_file'] = (
root_path + config['train_verbs_file']
)
training_params['val_classes_file'] = (
root_path + config['val_verbs_file']
)
training_params['test_classes_file'] = (
root_path + config['test_verbs_file']
)
elif training_params['label'] == 'object':
training_params['num_classes'] = (
len(open(root_path + config['train_objects_file'],
'r').readlines()))
training_params['train_classes_file'] = (
root_path + config['train_objects_file']
)
training_params['val_classes_file'] = (
root_path + config['val_objects_file']
)
training_params['test_classes_file'] = (
root_path + config['test_objects_file']
)
init_time = time.time()
# For reproducibility
tf.set_random_seed(1)
os.environ['PYTHONHASHSEED'] = '1'
seed(1)
rn.seed(1)
# Path to folders to save plots and checkpoints
checkpoints_folder = (config['project_folder'] +
config['checkpoints_folder'] +
'{}/{}/'.format(training_params['split'], exp_name)
)
plots_folder = (config['project_folder'] + config['plots_folder'] +
'{}/{}/'.format(training_params['split'], exp_name)
)
# Create any necessary folder
if not os.path.exists(plots_folder):
os.makedirs(plots_folder)
if not os.path.exists(checkpoints_folder):
os.makedirs(checkpoints_folder)
# Save training parameters
with open(plots_folder + 'training_params.json', 'w') as fp:
json.dump(training_params, fp, indent=4)
# ===============================================================
# LOAD THE DATA
# ===============================================================
# Compute number of videos from each set: train, validation and test
train_file = root_path + config['train_file']
val_file = root_path + config['val_file']
test_file = root_path + config['test_file']
nb_videos_train = num_sequences(config, training_params, 'train',
'training', train_file)
nb_videos_val = num_sequences(config, training_params, 'val',
'training', val_file)
nb_videos_test = num_sequences(config, training_params, 'test',
'training', test_file)
# Compute number of mini-batches for each set
# Add an extra mini-batch in case that the number of samples
# is not divisible by the mini-batch size
nb_batches_train = nb_videos_train // training_params['batch_size']
if nb_videos_train % training_params['batch_size'] > 0:
nb_batches_train += 1
nb_batches_val = nb_videos_val // training_params['batch_size']
if nb_videos_val % training_params['batch_size'] > 0:
nb_batches_val += 1
nb_batches_test = nb_videos_test // training_params['batch_size']
if nb_videos_test % training_params['batch_size'] > 0:
nb_batches_test += 1
# Necessary to load the model
custom_objects = {'f1_metric': f1_metric}
if training_params['use_data_augmentation']:
print('train: using data augmentation')
# Instantiate the generators of batches for training and validation
train_generator = BatchGenerator(config, 'train', train_file,
training_params, nb_batches_train)
val_generator = BatchGenerator(config, 'val', val_file,
training_params, nb_batches_val)
total_videos = float(nb_videos_train+nb_videos_val+nb_videos_test)
if training_params['apply_class_weighting']:
print('Class weighting applied')
print('Number of videos to => train: {}, val: {}, test: {}'.format(
nb_videos_train, nb_videos_val, nb_videos_test)
)
print('% of videos to => train: {}, val: {}, test: {}'.format(
nb_videos_train/total_videos*100, nb_videos_val/total_videos*100,
nb_videos_test/total_videos*100)
)
if not os.path.exists(plots_folder + 'results.json'):
all_run_results = dict()
# Vectors to accumulate the results of each run
accuracy_by_input, accuracy_by_video = dict(), dict()
f1_by_input, f1_by_video = dict(), dict()
# ===============================================================
# EXECUTE THE N RUNS OF THE EXPERIMENT
# ===============================================================
verbose = 1
if training_params['runs'] > 1:
verbose = 0
classes_val, indices_val = get_classes_ordered(
training_params['val_classes_file']
)
# Compute train labels to obtain class weights (for the loss function)
labels_by_video, _ = load_labels(
config, training_params, 'val', val_file,
training_params['val_classes_file']
)
plot_class_distribution(plots_folder, labels_by_video, classes_val, 'val')
del labels_by_video
gc.collect()
classes_test, indices_test = get_classes_ordered(
training_params['test_classes_file']
)
labels_by_video, _ = load_labels(
config, training_params, 'test', test_file,
training_params['test_classes_file']
)
plot_class_distribution(plots_folder, labels_by_video, classes_test, 'test')
del labels_by_video
gc.collect()
classes_train, indices_train = get_classes_ordered(
training_params['train_classes_file']
)
labels_by_video, indices_by_video = load_labels(
config, training_params, 'train',
train_file, training_params['train_classes_file']
)
plot_class_distribution(plots_folder, labels_by_video,
classes_train, 'train')
if training_params['apply_class_weighting']:
class_weights = compute_class_weight('balanced',
np.unique(indices_by_video),
indices_by_video)
plot_weights_distribution(plots_folder, class_weights,
classes_train, 'train')
histories = []
for run in range(training_params['runs']):
print('EXECUTING RUN {} ----------------------------------'.format(run))
run_folder = plots_folder + 'run_{}/'.format(run)
if not os.path.exists(run_folder):
os.makedirs(run_folder)
save_best_weights_file = (checkpoints_folder +
'best_weights_{}.h5'.format(run)
)
if os.path.exists(plots_folder + 'results.json'):
with open(plots_folder + 'results.json', 'r') as json_file:
all_run_results = json.load(json_file)
if not 'run_{}'.format(run) in all_run_results:
all_run_results['run_{}'.format(run)] = dict()
training_skipped = False
# If this run has already been computed, skip training
if not 'training_result' in all_run_results['run_{}'.format(run)]:
model = deploy_network(config, training_params)
if training_params['epochs'] > 0:
clipvalue = None
if training_params['gradient_clipvalue'] > 0.:
clipvalue = training_params['gradient_clipvalue']
print('Clipping gradient to {}'.format(
training_params['gradient_clipvalue']
))
if training_params['optimizer'] == 'adam':
print('Using Adam optimizer')
optimizer = Adam(lr=training_params['learning_rate'],
beta_1=0.9, beta_2=0.999, epsilon=1e-08,
clipvalue=clipvalue)
elif training_params['optimizer'] == 'sgd':
print('Using SGD optimizer')
optimizer = SGD(lr=training_params['learning_rate'],
momentum=0.9, decay=1e-6, nesterov=True,
clipvalue=clipvalue)
elif training_params['optimizer'] == 'rmsprop':
print('Using RMSprop optimizer')
optimizer = RMSprop(lr=training_params['learning_rate'])
metric_list = config['metrics'][1:] + [f1_metric]
model.compile(optimizer=optimizer,
loss='categorical_crossentropy',
metrics=metric_list)
model.summary()
print('Exp {}, run {}'.format(exp_name, run))
apply_cw = training_params['apply_class_weighting']
# Optional warmup training
if training_params['use_warmup']:
warmup_epochs = training_params['warmup_epochs']
history = model.fit_generator(generator=train_generator,
validation_data=val_generator,
epochs=warmup_epochs,
max_queue_size=10,
workers=2,
verbose=1,
class_weight=(class_weights
if apply_cw
else None),
shuffle=False,
use_multiprocessing=False)
# Type of criterion for the ModelCheckpoint and EarlyStopping
# depending on the metric used for the EarlStopping
if training_params['stop_criterion'] == 'val_loss':
mode = 'min'
else:
mode = 'max'
c = ModelCheckpoint(str(save_best_weights_file),
monitor=training_params['stop_criterion'],
save_best_only=True,
save_weights_only=False,
mode=mode,
period=1)
e = EarlyStopping(monitor=training_params['stop_criterion'],
min_delta=0,
patience=training_params['patience'],
verbose=0,
mode=mode,
baseline=None)
callbacks = [c, e]
train_time = time.time()
steps = None
if training_params['toy_execution']:
steps = 1
history = model.fit_generator(generator=train_generator,
steps_per_epoch=steps,
validation_data=val_generator,
validation_steps=steps,
epochs=training_params['epochs'],
max_queue_size=10,
workers=2,
verbose=1,
class_weight=(class_weights
if apply_cw
else None),
shuffle=False,
use_multiprocessing=False,
callbacks=callbacks)
# Save the history of training
histories.append(history.history)
print('TRAINING PHASE ENDED')
metric = training_params['stop_criterion']
# Depending on the metric to stop the training, choose whether
# the minimum or maximum value must be chosen
if metric == 'val_loss':
func = np.argmin
else:
func = np.argmax
best_epoch = func(history.history[metric])
# Save results to the dictionary
k1, k2 = 'run_{}'.format(run), 'training_result'
all_run_results[k1][k2] = dict()
all_run_results[k1][k2]['best_epoch'] = best_epoch
all_run_results[k1][k2][
'best_epoch_val_loss'] = history.history[
'val_loss'][best_epoch]
all_run_results[k1][k2][
'best_epoch_val_acc'] = history.history[
'val_acc'][best_epoch]
all_run_results[k1][k2][
'best_epoch_val_f1'] = history.history[
'val_f1_metric'][best_epoch]
# Save intermediate result
with open(plots_folder + 'results.json', 'w') as f:
json.dump(all_run_results, f, indent=4)
print('Time to train for {} epochs: {}s'.format(
training_params['epochs'], time.time()-train_time))
else:
training_skipped = True
# TEST ========================
print('='*20)
print('TEST PHASE')
print('training_skipped', training_skipped)
# If training was not skipped, save the histories (loss, accuracy and
# f1 per epoch, per run)
if not training_skipped:
save_history(run_folder, history.history)
del model
# If training was skipped, load the history of this run
else:
histories.append(load_history(run_folder, run))
# Load the best model for evaluation
model = load_model(str(save_best_weights_file),
custom_objects=custom_objects)
print('Loaded the checkpoint at {}'.format(save_best_weights_file))
class_list = classes_train
print('Exp {}, run {}'.format(exp_name, run))
res_dict = dict()
for mode in ['train', 'val', 'test']:
# If the evaluation was already saved and 'redo_evaluation' is not
# set to True
if (
('evaluation_{}'.format(mode) in
all_run_results['run_{}'.format(run)]) and
not training_params['redo_evaluation']
):
if not mode in accuracy_by_video:
accuracy_by_video[mode] = []
if not mode in f1_by_video:
f1_by_video[mode] = []
k1, k2 = 'run_{}'.format(run), 'evaluation_{}'.format(mode)
_f1_by_video = all_run_results[k1][k2]['f1']
_accuracy_by_video = all_run_results[k1][k2]['accuracy']
accuracy_by_video[mode].append(_accuracy_by_video/100.)
#f1_by_input[mode].append(_f1_by_input)
f1_by_video[mode].append(_f1_by_video/100.)
print('{}: Accuracy per video: {}, '.format(
mode, _accuracy_by_video) +
'Macro-F1 per video: {}'.format( _f1_by_video)
)
continue
if training_params['use_data_augmentation']:
print('{}: using data augmentation'.format(mode))
#if not results_loaded:
if mode == 'train':
if training_params['oversampling']:
nb_videos_train = num_sequences(config, training_params,
'train', 'evaluation',
train_file)
nb_videos = nb_videos_train
generator = load_gaze_plus_sequences(config, 'train',
train_file,
training_params)
classes_train, indices_train = get_classes_ordered(
training_params['train_classes_file']
)
elif mode == 'val':
nb_videos = nb_videos_val
generator = load_gaze_plus_sequences(config, 'val',
val_file,
training_params)
classes_train, indices_train = get_classes_ordered(
training_params['train_classes_file']
)
elif mode == 'test':
nb_videos = nb_videos_test
generator = load_gaze_plus_sequences(config, 'test',
test_file,
training_params)
classes_test, indices_test = get_classes_ordered(
training_params['train_classes_file']
)
if training_params['toy_execution']:
nb_videos = 2
predictions_by_video, ground_truth_by_video = [], []
length_of_videos = dict()
predictions_by_class = []
for _ in range(training_params['num_classes']):
predictions_by_class.append([])
info = dict()
# Process video by video
print('Processing {}, {} videos'.format(mode, nb_videos))
for i in tqdm(range(nb_videos)):
batch_x, batch_y, video_name, length = generator.next()
predictions = model.predict(batch_x)[0]
# Dictionary to save results by length of video
if not length in length_of_videos:
length_of_videos[length] = []
# Save class predicted by the model and ground truth
predicted = np.argmax(predictions,0)
predictions_by_video.append(predicted)
ground_truth_by_video.append(batch_y)
# Save prediction by class
predictions_by_class[batch_y].append(predicted)
# Save results by video
info[video_name] = dict()
info[video_name]['ground_truth_index'] = batch_y
info[video_name]['ground_truth_class'] = class_list[batch_y]
info[video_name]['prediction_index'] = predicted
info[video_name]['prediction_softmax'] = predictions[0]
info[video_name]['prediction_class'] = class_list[predicted]
info[video_name]['length'] = length
info[video_name]['classes'] = class_list
ground_truth_by_video = np.squeeze(np.stack(ground_truth_by_video))
predictions_by_video = np.squeeze(np.stack(predictions_by_video))
cm_by_video = confusion_matrix(ground_truth_by_video,
predictions_by_video,
labels=range(
training_params['num_classes']
))
_accuracy_by_video = accuracy_score(ground_truth_by_video,
predictions_by_video)
_f1_by_video = f1_score(ground_truth_by_video, predictions_by_video,
average='macro')
k1, k2 = 'run_{}'.format(run), 'evaluation_{}'.format(mode)
all_run_results[k1][k2] = dict()
all_run_results[k1][k2]['num_videos'] = nb_videos
all_run_results[k1][k2]['accuracy'] = _accuracy_by_video*100.
all_run_results[k1][k2]['f1'] = _f1_by_video*100.
print('{}: Accuracy per video: {}, Macro-F1 per video: {}'.format(
mode, _accuracy_by_video, _f1_by_video)
)
plot_confusion_matrix(
cm_by_video, class_list,
run_folder + '_normalized_by_video_{}_{}_{}.pdf'.format(
mode, exp_name, run),
normalize=True,
title='Normalized confusion matrix for {} set'.format(mode),
cmap='coolwarm'
)
plot_confusion_matrix(
cm_by_video, class_list,
run_folder + '_by_video_{}_{}_{}.pdf'.format(
mode, exp_name, run),
normalize=False,
title='Confusion matrix for {} set'.format(mode),
cmap='coolwarm'
)
# Compute and save results by class
for i in range(len(predictions_by_class)):
if len(predictions_by_class[i]) > 0:
pred = predictions_by_class[i]
acc = accuracy_score([i]*len(pred),pred)
f1 = f1_score([i]*len(pred),pred, average='macro')
predictions_by_class[i] = [acc, f1,
len(predictions_by_class[i])]
else:
predictions_by_class[i] = [0., 0.,
len(predictions_by_class[i])]
save_results(run_folder, mode, predictions_by_class,
class_list, run)
# Save general info
save_in_csv(run_folder + '{}_run{}_evaluation_info.csv'.format(
mode,run), info)
if not mode in accuracy_by_video:
accuracy_by_video[mode] = []
f1_by_video[mode] = []
accuracy_by_video[mode].append(_accuracy_by_video)
f1_by_video[mode].append(_f1_by_video)
del generator
gc.collect()
with open(plots_folder + 'results.json', 'w') as f:
json.dump(all_run_results, f, indent=4)
# END OF THE EVALUATION ===========================================
del model
gc.collect()
K.clear_session()
tf.set_random_seed(1)
# END OF ALL THE RUNS ===========================================
if not training_skipped:
del val_generator, train_generator
gc.collect()
plot_training_info(plots_folder,
exp_name,
config['metrics'] + ['f1_metric'],
True,
histories)
# ===============================================================
# SHOW THE AVERAGED RESULTS
# ===============================================================
results_dict = dict()
print('='*20)
results_file = open(plots_folder + 'results.txt', 'w')
for mode in ['train', 'val', 'test']:
res_msg = '='*20 + '\n'
res_msg += '{}: AVERAGE RESULTS OF {} RUNS\n'.format(
mode, training_params['runs'])
res_msg += '='*20 + '\n'
results_dict[mode] = dict()
results_dict[mode]['accuracy_by_video'] = accuracy_by_video[mode]
results_dict[mode]['f1_by_video'] = f1_by_video[mode]
res_msg += 'ACCURACY: {:.2f} (-+{:.2f}), MACRO F1: {:.2f} (-+{:.2f})\n'.format(
np.mean(accuracy_by_video[mode])*100, np.std(accuracy_by_video[mode])*100,
np.mean(f1_by_video[mode])*100, np.std(f1_by_video[mode])*100
)
res_msg += 'RESULTS PER RUN\n'
res_msg += '----------------\n'
res_msg += '\nAccuracy by video:\n'
res_msg += ', '.join(str(x) for x in accuracy_by_video[mode])
res_msg += '\nMacro F1 by video:\n'
res_msg += ', '.join(str(x) for x in f1_by_video[mode])
res_msg += '\n'
print(res_msg)
results_file.write(res_msg)
results_file.close()
res_msg = '\n\nTime for training and evaluation of every run: {}s'.format(time.time()-init_time)
final_results = dict()
for run in range(training_params['runs']):
k1 = 'run_{}'.format(run)
final_results[k1] = dict()
for mode in ['train', 'val', 'test']:
final_results[k1][mode] = dict()
final_results[k1][mode]['accuracies'] = [x*100 for x in accuracy_by_video[mode]]
final_results[k1][mode]['f1s'] = [x*100 for x in f1_by_video[mode]]
with open(plots_folder + 'overall_results.json', 'w') as f:
json.dump(final_results, f, indent=4)
def load_model(self, name):
backend.clear_session()
self.model = load_model(name)
return
def predict(self):
'''
Everyday prediction
'''
if not os.path.isdir(self.folder):
os.makedirs(self.folder)
data_predict = {}
data_predict['test_id'] = []
data_predict['PM2.5'] = []
data_predict['PM10'] = []
data_predict['O3'] = []
tz = pytz.timezone('utc')
now = datetime.datetime.now(tz)
now = datetime.datetime(now.year, now.month, now.day, now.hour)
data_folder = request_data(self.request_paras,
now,
use_caiyun=self.use_caiyun)
for station_id in self.station_infos:
print('*' * 5 + station_id + '*' * 5)
station_info = self.station_infos[station_id]
for i in range(48):
data_predict['test_id'].append('{}#{}'.format(station_id, i))
for p in station_info['pollutions']:
print('-' * 5 + p + '-' * 5)
x = prepare_data(data_folder,
station_id,
station_info['model_id'][p],
station_info['norm'][p],
p,
use_caiyun=self.use_caiyun)
#return x
#print(datetime.datetime.now())
model = keras.models.load_model(station_info['model_file'][p],
custom_objects={
'loss_smape_rmse':
my_loss.loss_smape_rmse
})
#print(datetime.datetime.now())
predicted = model.predict(x)
#return predicted
norm_y = utils.normalization()
norm_y.load(station_info['norm'][p]['norm_y'])
predicted = norm_y(predicted, forward=False)
for y in predicted[0]:
data_predict[p].append(y)
K.clear_session()
length = max(len(data_predict['O3']), len(data_predict['PM2.5']),
len(data_predict['PM10']))
if len(data_predict['O3']) < length:
for i in range(length - len(data_predict['O3'])):
data_predict['O3'].append(0)
predict_file = os.path.join(
self.folder, self.prefix_predicted + now.strftime("%Y-%m-%d-%H") +
self.postfix_predicted + '.csv')
pd_predict = pd.DataFrame.from_dict(data_predict)
pd_predict.to_csv(predict_file,
columns=['test_id', 'PM2.5', 'PM10', 'O3'],
index=False)
if self.auto_submit:
utils.submit(predict_file, 'submit_' + now.strftime("%Y-%m-%d-%H"),
'auto_submit')
return predict_file
import matplotlib.pyplot as plt
from cv2 import imread, imwrite
# from imageio import imread, imwrite
from LAST_BUILD.gan_settings import _IMG_ROWS, _IMG_COLS, _CHANNEL, \
_TRAIN_IMG_PATH, _TRUE_PHOTOS_DIR, _BLENDER_PHOTOS_DIR, \
_ROTATION, _LIGHTNING, \
_TRAIN_STEPS, _BATCH_SIZE, _SAVE_INTERVAL, \
_OUTPUT_IMAGES_X, _OUTPUT_IMAGES_Y, \
_MOBILENET_INPUT_SHAPE, \
_GENERATED_FACES_PATH, _INPUT_TENSOR_SHAPE, _PERSONS
clear_session()
"""
########################################################################################################################
########################################################################################################################
########################################################################################################################
"""
__COLS = 4016
__ROWS = 6016
def _get_labeled_true_photos(directory):
for path, _, file_names in os.walk(directory):
for file_name in file_names:
for rotation_label in _get_rotation_labels():
for light_label in _get_lightning_labels():
def main():
train = pd.read_csv(args.folds_csv)
MODEL_PATH = os.path.join(args.models_dir, args.network + args.alias)
folds = [int(f) for f in args.fold.split(',')]
print('Training Model:', args.network + args.alias)
for fold in folds:
K.clear_session()
print(
'***************************** FOLD {} *****************************'
.format(fold))
if fold == 0:
if os.path.isdir(MODEL_PATH):
raise ValueError('Such Model already exists')
os.system("mkdir {}".format(MODEL_PATH))
# Train/Validation sampling
df_train = train[train.fold != fold].copy().reset_index(drop=True)
df_valid = train[train.fold == fold].copy().reset_index(drop=True)
# Train on pseudolabels only
if args.pseudolabels_dir != '':
pseudolabels = pd.read_csv(args.pseudolabels_csv)
df_train = pseudolabels.sample(
frac=1, random_state=13).reset_index(drop=True)
# Keep only non-black images
ids_train, ids_valid = df_train[
df_train.unique_pixels > 1].id.values, df_valid[
df_valid.unique_pixels > 1].id.values
print('Training on {} samples'.format(ids_train.shape[0]))
print('Validating on {} samples'.format(ids_valid.shape[0]))
# Initialize model
weights_path = os.path.join(MODEL_PATH,
'fold_{fold}.hdf5'.format(fold=fold))
# Get the model
model, preprocess = get_model(args.network,
input_shape=(args.input_size,
args.input_size, 3),
freeze_encoder=args.freeze_encoder)
# LB metric threshold
def lb_metric(y_true, y_pred):
return Kaggle_IoU_Precision(
y_true,
y_pred,
threshold=0 if args.loss_function == 'lovasz' else 0.5)
model.compile(optimizer=RMSprop(lr=args.learning_rate),
loss=make_loss(args.loss_function),
metrics=[lb_metric])
if args.pretrain_weights is None:
print('No weights passed, training from scratch')
else:
wp = args.pretrain_weights.format(fold)
print('Loading weights from {}'.format(wp))
model.load_weights(wp, by_name=True)
# Get augmentations
augs = get_augmentations(args.augmentation_name,
p=args.augmentation_prob)
# Data generator
dg = SegmentationDataGenerator(input_shape=(args.input_size,
args.input_size),
batch_size=args.batch_size,
augs=augs,
preprocess=preprocess)
train_generator = dg.train_batch_generator(ids_train)
validation_generator = dg.evaluation_batch_generator(ids_valid)
# Get callbacks
callbacks = get_callback(args.callback,
weights_path=weights_path,
fold=fold)
# Fit the model with Generators:
model.fit_generator(
generator=ThreadsafeIter(train_generator),
steps_per_epoch=ids_train.shape[0] // args.batch_size * 2,
epochs=args.epochs,
callbacks=callbacks,
validation_data=ThreadsafeIter(validation_generator),
validation_steps=np.ceil(ids_valid.shape[0] / args.batch_size),
workers=args.num_workers)
gc.collect()
def run_cross_validation(self, output_dir=None):
"""
The main function that calls most of the other interface functions.
"""
# create the output directory for storing stuff
if output_dir is not None:
os.makedirs(output_dir, exist_ok=False)
else:
output_dir = "./cv_output"
# be sure that the context has its data ready
self.cv_context.data_handler.read_data_if_necessary()
# allow the cv context to write additional information
# about the data
self.cv_context.write_data_info(output_dir)
# make training/test split
train_ids, test_ids = self.make_train_test_split()
if test_ids is not None:
# check if training and testset are similar wrt survival information
assert not set(train_ids).intersection(set(test_ids))
check_cohort_differences(
train_ids, test_ids, self.cv_context.data_handler.outcome_dict)
else:
print("No separate testset available!")
print("\nStarting {}-fold cross-validation with {} repetitions.".format(
self.folds, self.reps))
# for each of the training repetitions create stratified cross validation splits
# of the training data
self.create_cv_splits(train_ids)
# store the cv splits and create folders for each rep and fold
self.store_cv_splits(output_dir)
perf_dfs = [] # dataframes of performance indices for each fold
pred_dfs = dict()
for r in range(self.reps):
rep = "rep_" + str(r)
rep_dir = os.path.join(output_dir, rep)
# os.makedirs(rep_dir, exist_ok=False)
for k in range(self.folds):
fold = "fold_" + str(k)
fold_dir = os.path.join(rep_dir, fold)
# os.makedirs(fold_dir, exist_ok=False)
ids_train, ids_valid = self.cv_splits[r][k]
pred_dict, perf_df = self.cv_context.run_single_fold(
ids_train, ids_valid, test_ids,
output_dir=fold_dir, rep=r, fold=k,
print_model_summary=((r==0) and (k==0)))
perf_dfs.append(perf_df)
# update the pred_dfs dictionary
# with the dictionary pred_dict that contains
# the subevaluation results for the current fold
for res_descr in pred_dict:
if res_descr not in pred_dfs:
pred_dfs[res_descr] = {}
if r not in pred_dfs[res_descr]:
pred_dfs[res_descr][r] = {}
pred_dfs[res_descr][r][k] = pred_dict[res_descr]
# release GPU memory after every training
K.clear_session()
# all model performances for different folds and reps
full_perf_df = pd.concat(perf_dfs, ignore_index=True, sort=False)
# store the evaluation perf_df
full_perf_df.to_csv(
os.path.join(output_dir, "model_performances.csv"),
na_rep="NAN", index=False)
# store the test set
pd.DataFrame(test_ids).to_csv(
os.path.join(output_dir, "ids_test.csv"), index=False,
header=False)
return pred_dfs, perf_dfs
# 300x300의 grayscale 이미지로 리사이즈
train_224 = np.zeros([2048, 100, 100, 3], dtype=np.float32)
for i, s in enumerate(x_train):
converted = cv2.cvtColor(s, cv2.COLOR_GRAY2RGB)
# converted = 변환 , 컬러색으로 변환(특성 강조)
resized = cv2.resize(converted, (100, 100), interpolation=cv2.INTER_CUBIC)
# 원본이미지, 결과 이미지 크기, 보간법(cv2.INTER_CUBIC, cv2.INTER_LINEAR 이미지 확대할 때 사용/cv2.INTER_AREA는 사이즈 줄일 때 사용)
# 보간법(interpolation)이란 통계적 혹은 실험적으로 구해진 데이터들(xi)로부터,
# 주어진 데이터를 만족하는 근사 함수(f(x))를 구하고, 이 식을 이용하여 주어진 변수에 대한 함수 값을 구하는 일련의 과정을 의미
del converted # 변수 초기화 (삭제 x)
train_224[i] = resized
del resized
bek.clear_session()
gc.collect()
# plt.imshow(train_224[i])
# plt.show()
from keras.callbacks import ModelCheckpoint, EarlyStopping, ReduceLROnPlateau, LearningRateScheduler
# ReduceLROnPlateau에서 lr는 이전 epoch가 끝날 때 변경되고, LearningRateScheduler에서 lr는 현재 epoch가 시작될 때 변경된다.
# 둘 다 써보기 reduc : ,learning:
from keras.preprocessing.image import ImageDataGenerator
from sklearn import metrics
datagen = ImageDataGenerator(width_shift_range=(-1, 1),
height_shift_range=(-1, 1),
zoom_range=0.15,
validation_split=0.2)
from data_generator.object_detection_2d_data_generator import DataGenerator
from eval_utils.average_precision_evaluator import Evaluator
#%matplotlib inline
# Set a few configuration parameters.
img_height = 300
img_width = 300
n_classes = 20
# 1: Build the Keras model
model_mode = 'inference'
model_mode = 'training'
K.clear_session() # Clear previous models from memory.
model = ssd_300(image_size=(img_height, img_width, 3),
n_classes=n_classes,
mode=model_mode,
l2_regularization=0.0005,
scales=[0.1, 0.2, 0.37, 0.54, 0.71, 0.88, 1.05], # The scales for MS COCO are [0.07, 0.15, 0.33, 0.51, 0.69, 0.87, 1.05]
aspect_ratios_per_layer=[[1.0, 2.0, 0.5],
[1.0, 2.0, 0.5, 3.0, 1.0/3.0],
[1.0, 2.0, 0.5, 3.0, 1.0/3.0],
[1.0, 2.0, 0.5, 3.0, 1.0/3.0],
[1.0, 2.0, 0.5],
[1.0, 2.0, 0.5]],
two_boxes_for_ar1=True,
steps=[8, 16, 32, 64, 100, 300],
offsets=[0.5, 0.5, 0.5, 0.5, 0.5, 0.5],
def clear_model(self):
self.model = None
K.clear_session()
gc.collect()
validation_data=([val_his, val_mask, val_target,
val_feats], np.eye(output_dim)[val_y]),
verbose=1,
callbacks=[early_stopping])
oof_df.loc[val_idx, 'nn_prob'] = clf.predict(
[val_his, val_mask, val_target, val_feats], batch_size=2048)[:, 1]
print('val auc:', roc_auc_score(val_y, oof_df['nn_prob'].values[val_idx]))
test_pred_df['nn_prob'] += clf.predict(
[test_his, test_mask, test_target, test_feats],
batch_size=2048)[:, 1] / skf.n_splits
emb_layer = keras.models.Model(inputs=clf.input,
outputs=clf.get_layer(name='emb').output)
oof_emb[val_idx] = emb_layer.predict(
[val_his, val_mask, val_target, val_feats], batch_size=2048)
test_emb += emb_layer.predict(
[test_his, test_mask, test_target, test_feats],
batch_size=2048) / skf.n_splits
del emb_layer, clf
BKD.clear_session()
tf.reset_default_graph()
gc.collect()
print('runtime: {}\n'.format(time.time() - t))
for i in tqdm(range(emb_size)):
oof_df['nn_emb_{}'.format(i)] = oof_emb[:, i]
test_pred_df['nn_emb_{}'.format(i)] = test_emb[:, i]
oof_df.to_pickle('../user_data/data/nn/nn_trn.pkl')
test_pred_df.to_pickle('../user_data/data/nn/nn_test.pkl')
def predict_liver_and_tumor(model, image, batch, input_size, input_cols,
thresh_liver=0.5, thresh_tumor=0.9, num=3,
latent_model=None):
'''
Prediction of segmentation of both liver and tumor
'''
print('Prediction segmentations...')
window_cols = (input_cols/4)
count = 0
box_test = np.zeros((batch, input_size, input_size, input_cols, 1),
dtype="float32")
x = image.shape[0]
y = image.shape[1]
z = image.shape[2]
right_cols = int(min(z, y+10)-input_cols)
left_cols = max(0, min(0-5, right_cols))
score = np.zeros((x, y, z, num), dtype='float32')
score_num = np.zeros((x, y, z, num), dtype='int16')
print(x, y, z)
if latent_model:
# Initialize 3D latent space features array
n_filters = 504
# Not sure where this comes from. Start is nb_filters
# = 96. Then through a combination of compression and growth rate
# for a number of dense blocks it becomes 504
x_ls = x / 2**5 # Number of dense blocks + 1
y_ls = y / 2**5
latent_features = np.zeros((x_ls, y_ls, z, n_filters), dtype='float32')
latent_features_num = np.zeros((x_ls, y_ls, z, n_filters), dtype='int16')
iterator = xrange(left_cols, right_cols+window_cols, window_cols)
cnum = 0
for cols in iterator:
cnum += 1
# print ('and', z-input_cols,z)
print(('\t Running column {} / {}.').format(cnum, len(iterator)))
if cols > z - input_cols:
patch_test = image[0:input_size, 0:input_size, z-input_cols:z]
box_test[count, :, :, :, 0] = patch_test
# print ('final', input_cols-window_cols, input_cols)
patch_test_mask = model.predict(box_test, batch_size=batch,
verbose=0)
patch_test_mask = K.softmax(patch_test_mask)
patch_test_mask = K.eval(patch_test_mask)
patch_test_mask = patch_test_mask[:, :, :, 1:-1, :]
for i in xrange(batch):
score[0:input_size, 0:input_size, z-input_cols+1:z-1, :] += patch_test_mask[i]
score_num[0:input_size, 0:input_size, z-input_cols+1:z-1, :] += 1
if latent_model:
latent_features_test = latent_model.predict(box_test,
batch_size=batch,
verbose=0)
for i in xrange(batch):
# FIXME: adopted size to default settings, should be adaptive
latent_features[:, :, z-3:z-1, :] += latent_features_test[i]
latent_features_num[:, :, z-3:z-1, :] += 1
else:
patch_test = image[0:input_size, 0:input_size, cols:cols + input_cols]
box_test[count, :, :, :, 0] = patch_test
patch_test_mask = model.predict(box_test, batch_size=batch,
verbose=0)
patch_test_mask = K.softmax(patch_test_mask)
patch_test_mask = K.eval(patch_test_mask)
patch_test_mask = patch_test_mask[:, :, :, 1:-1, :]
for i in xrange(batch):
score[0:input_size, 0:input_size, cols+1:cols+input_cols-1, :] += patch_test_mask[i]
score_num[0:input_size, 0:input_size, cols+1:cols+input_cols-1, :] += 1
if latent_model:
latent_features_test = latent_model.predict(box_test,
batch_size=batch,
verbose=0)
for i in xrange(batch):
# FIXME: adopted size to default settings, should be adaptive
latent_features[:, :, cols+1:cols+3, :] += latent_features_test[i]
latent_features_num[:, :, cols+1:cols+3, :] += 1
score = score/(score_num + 1e-4)
if latent_model:
latent_features = latent_features/(latent_features_num + 1e-4)
# score[:, :, 0] == background
result1 = score[:, :, :, num-2] # Liver
result2 = score[:, :, :, num-1] # Lesions
K.clear_session()
print('\t Applying postprocessing.')
result1[result1 >= thresh_liver] = 1
result1[result1 < thresh_liver] = 0
result2[result2 >= thresh_tumor] = 1
result2[result2 < thresh_tumor] = 0
result1[result2 == 1] = 1
# preserve the largest liver
lesions = result2
box = list()
[liver_res, num] = measure.label(result1, return_num=True)
region = measure.regionprops(liver_res)
for i in xrange(num):
box.append(region[i].area)
label_num = box.index(max(box)) + 1
liver_res[liver_res != label_num] = 0
liver_res[liver_res == label_num] = 1
# preserve the largest liver
liver_res = ndimage.binary_dilation(liver_res, iterations=1).astype(liver_res.dtype)
box = []
[liver_labels, num] = measure.label(liver_res, return_num=True)
region = measure.regionprops(liver_labels)
for i in xrange(num):
box.append(region[i].area)
label_num = box.index(max(box)) + 1
liver_labels[liver_labels != label_num] = 0
liver_labels[liver_labels == label_num] = 1
liver_labels = ndimage.binary_fill_holes(liver_labels).astype(int)
# preserve tumor within ' largest liver' only
lesions = lesions * liver_labels
lesions = ndimage.binary_fill_holes(lesions).astype(int)
liver_res = np.array(liver_res, dtype='uint8')
liver_res = ndimage.binary_fill_holes(liver_res).astype(int)
# liver_res[lesions == 1] = 2
# transpose back and convert to float 64 for uint8 saving
lesions = np.transpose(lesions, [2, 1, 0])
liver_res = np.transpose(liver_res, [2, 1, 0])
lesions = np.array(lesions, dtype='uint8')
liver_res = np.array(liver_res, dtype='uint8')
if latent_model:
return liver_res, lesions, latent_features
else:
return liver_res, lesions
def process(spath, t_path, conll_path, text_seed, k, prf_file):
step_1(spath, t_path, text_seed, k)
# step_2
# 法律文档*k+conll2012 语料训练
step_2(t_path, conll_path, text_seed, k)
# step_3
# text_seed=2222
train_file = 'train_%d.data' % (text_seed)
test_file = 'test_%d.data' % (text_seed)
train_documents = create_documents(train_file)
print(len(train_documents))
# print(train_documents[1].chars)
# print(train_documents[1].labels)
for doc in train_documents[:20]:
print(doc.index, doc.chars, doc.labels)
test_documents = create_documents(test_file)
print(len(test_documents))
# 生成词典
lexicon, lexicon_reverse = get_lexicon(train_documents + test_documents)
print(len(lexicon), len(lexicon_reverse))
embedding_model = gensim.models.Word2Vec.load(r'model_conll_law.m')
embedding_size = embedding_model.vector_size
print(embedding_size)
# 预训练词向量
embedding_weights = create_embedding(embedding_model, embedding_size,
lexicon_reverse)
print(embedding_weights.shape)
print(lexicon_reverse[1])
print(embedding_weights[1])
# 0 为padding的label
label_2_index = {'B': 1, 'M': 2, 'E': 3, 'S': 4}
index_2_label = {0: 'Pad', 1: 'B', 2: 'M', 3: 'E', 4: 'S'}
train_data_list, train_label_list, train_index_list = create_matrix(
train_documents, lexicon, label_2_index)
test_data_list, test_label_list, test_index_list = create_matrix(
test_documents, lexicon, label_2_index)
print(len(train_data_list), len(train_label_list), len(train_label_list))
print(len(test_data_list), len(test_label_list), len(test_index_list))
print(train_data_list[1])
print(train_label_list[1])
# print(train_index_list[:20])
print('-' * 15)
max_len = max(map(len, train_data_list + test_data_list))
print('max_len:', max_len) # 128
min_len = min(map(len, train_data_list + test_data_list))
print('min_len:', min_len)
# 前面补0 padding
print(train_data_list[0])
train_data_array, train_label_list_padding = padding_sentences(
train_data_list, train_label_list, max_len)
print(train_data_array.shape)
print(train_data_array[0])
print(test_data_list[0])
test_data_array, test_label_list_padding = padding_sentences(
test_data_list, test_label_list, max_len)
print(test_data_array.shape)
print(test_data_array[0])
# label
# print(train_label_list_padding[0])
train_label_array=np_utils.to_categorical(train_label_list_padding,len(label_2_index)+1).\
reshape((len(train_label_list_padding),len(train_label_list_padding[0]),-1))
print(train_label_array.shape)
# label
# print(test_label_list_padding[0])
test_label_array=np_utils.to_categorical(test_label_list_padding,len(label_2_index)+1).\
reshape((len(test_label_list_padding),len(test_label_list_padding[0]),-1))
print(test_label_array.shape)
# model
model = Bilstm_CNN_Crf(max_len, len(lexicon),
len(label_2_index) + 1, embedding_weights)
model.summary()
print(model.input_shape)
print(model.output_shape)
plot_model(model,
to_file='bilstm_cnn_crf_model.png',
show_shapes=True,
show_layer_names=True)
train_nums = len(train_data_array)
train_array, val_array = train_data_array[:int(train_nums *
0.9)], train_data_array[
int(train_nums * 0.9):]
train_label, val_label = train_label_array[:int(train_nums *
0.9)], train_label_array[
int(train_nums * 0.9):]
print(train_array.shape, train_label.shape)
print(val_array.shape, val_label.shape)
print(test_data_array.shape, test_label_array.shape)
checkpointer=ModelCheckpoint(filepath='best_val_model.hdf5',verbose=1,\
save_best_only=True,monitor='val_loss',mode='auto')
# train model
hist=model.fit(train_array,train_label,batch_size=256,epochs=20,verbose=1,\
validation_data=(val_array,val_label),callbacks=[checkpointer])
print(hist.history['val_loss'])
best_model_epoch = np.argmin(hist.history['val_loss'])
print('best_model_epoch:', best_model_epoch)
# 可视化loss acc
# plot_acc(hist)
# print(hist.history)
model.load_weights('best_val_model.hdf5')
test_y_pred = model.predict(test_data_array, batch_size=512, verbose=1)
# print(test_y_pred)
# 预测标签 [0,0,....,1,2,3,1]
pred_label = np.argmax(test_y_pred, axis=2)
print(pred_label[0])
print(test_label_list_padding[0])
K.clear_session()
print(pred_label.shape, test_label_list_padding.shape)
# 生成输出文档
real_text_list,pred_text_list,real_label_list,pred_label_list=create_pred_text(\
lexicon_reverse,test_data_array,pred_label,test_label_list_padding,test_index_list)
'''
for r_text,p_text,r_label,p_label in zip(real_text_list,pred_text_list,real_label_list,pred_label_list):
print(r_text)
print([index_2_label[r] for r in r_label])
print('-'*10)
print(p_text)
print([index_2_label[p] for p in p_label])
print('='*20)
'''
# 写文件
write_2_file(real_text_list, pred_text_list)
# score
F = score.prf_score('real_text.txt', 'pred_text.txt', prf_file, text_seed,
best_model_epoch)
# F_list.append([text_seed,F])
return F
val_writer_path = os.path.join(logdir, 'validation')
val_writer = tf.summary.FileWriter(val_writer_path)
predictions = model.predict(X,batch_size=1)
for i in [0, 1, 10, 20]: #camera indices
img = _plot_y_vs_pred_on_BytesIO(y[:, i], predictions[:, i])
summary_pb = tf.Summary(value=[tf.Summary.Value(tag="camera_{}_prediction_vs_actual".format(i), image=img)])
val_writer.add_summary(summary_pb)
text_tensor = tf.make_tensor_proto([(k,str(v)) for k,v in params.items()], dtype=tf.string)
meta = tf.SummaryMetadata()
meta.plugin_data.plugin_name = "text"
summary_pb = tf.Summary()
summary_pb.value.add(tag="parameters", metadata=meta, tensor=text_tensor)
val_writer.add_summary(summary_pb)
val_writer.close()
params["_ignore"] = True
del model
del history
del prepare_data
K.clear_session() #to hopefully prevent slow down after a few models..
#writing the log file back with potential info about the run (including _ignore flag which is set after it is run
# once).
with open(paramss_path,'w') as f:
json.dump(paramss,f)
def DetectLines(self):
# os.environ['CUDA_VISIBLE_DEVICES'] = str(self.gpu)
os.environ['CUDA_VISIBLE_DEVICES'] = str(self.gpu)
config = tf.ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = 0.5
set_session(tf.Session(config=config))
# os.environ['CUDA_VISIBLE_DEVICES'] = str(self.gpu)
BigImg = self.folder + 'sat_original.png'
BigLane = self.folder + 'final_marks.png'
bigImg = imread(BigImg, -1)
bigLane = imread(BigLane, -1) + self.lane_v2
bigLane_gray = imread(BigLane, 0) + self.lane_v2[:, :, 0]
bigLine = np.zeros_like((bigImg))
checkpoint_name = "GAN_v4_label_normal_lanenet2/"
model_name = 'lanenet2'
checkpoint_dir_root = "/home/beemap/Documents/cesar-workspace/Segmentation/Keras_Code/keras3/checkpoints/"
checkpoint_dir = checkpoint_dir_root + checkpoint_name
print("Loading Model in .... ", checkpoint_dir)
json_file = open(checkpoint_dir + model_name + ".json", 'r')
loaded_model_json = json_file.read()
json_file.close()
model = model_from_json(loaded_model_json)
model.load_weights("%s/%s_weights_90.h5" %
(checkpoint_dir, model_name))
for r in range(0, bigImg.shape[0], 256):
for c in range(0, bigImg.shape[1], 256):
img = bigImg[r:r + 256, c:c + 256, :3]
# h, w, _ = img.shape
tmp = img[:]
lane = bigLane[r:r + 256, c:c + 256, :3]
img = np.float32(img) / 255.0
lane = np.float32(lane) / 255.0
# input_image_gray = data_util.get_image(image,do_aug=[],gray=True, change=False)
input_image_gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
w1, w2, w3, w4 = lanenet_wavelet(input_image_gray)
w1 = np.expand_dims(w1, axis=0)
w2 = np.expand_dims(w2, axis=0)
w3 = np.expand_dims(w3, axis=0)
w4 = np.expand_dims(w4, axis=0)
lane_gray = bigLane_gray[r:r + 256, c:c + 256]
hough = hough_lines(lane_gray,
rho=1,
min_line_len=10,
threshold=40,
max_line_gap=300) / 255.0
mask = model.predict([
np.expand_dims(img, axis=0),
np.expand_dims(lane, axis=0), w1, w2, w3, w4
],
batch_size=None,
verbose=0,
steps=None)
mask = np.round(mask[0, :, :, 0]).astype(int)
seg = np.zeros((256, 256, 3))
seg[:, :, 0] += ((mask[:, :] == 1) * (255)).astype('uint8')
seg[:, :, 1] += ((mask[:, :] == 1) * (255)).astype('uint8')
seg[:, :, 2] += ((mask[:, :] == 1) * (255)).astype('uint8')
bigLine[r:r + 256, c:c + 256, :] = seg
K.clear_session()
name = 'BigLine.png'
new_dir = self.folder
dest = new_dir + name
cv2.imwrite(dest, bigLine)
def model(X_train,
Y_train,
X_test,
Y_test,
learning_rate=0.0001,
num_epochs=500,
minibatch_size=32,
print_cost=True,
keep_prob=0.75,
units=64):
num_epochs = num_epochs + 1
ops.reset_default_graph()
K.clear_session()
(m, n_x) = X_train.shape
n_y = Y_train.shape[1]
costs = []
L2penalty = 0.7 / 2
X, Y = create_placeholders(n_x, n_y)
# sets the number of nodes in each hidden layer:
parameters = initialize_parameters(units)
out = forward_propagation(X, parameters, units, keep_prob)
cost = compute_cost(out, Y, parameters, (L2penalty / m))
starter_learning_rate = learning_rate
global_step = tf.Variable(0, trainable=False)
end_learning_rate = learning_rate / 1000
decay_steps = 1000
learn_rate = tf.train.polynomial_decay(starter_learning_rate,
global_step,
decay_steps,
end_learning_rate,
power=0.5)
optimizer = tf.train.AdamOptimizer(learning_rate=learn_rate).minimize(cost)
init = tf.global_variables_initializer()
with tf.Session() as sess:
K.set_session(sess)
sess.run(init)
for epoch in range(num_epochs):
epoch_cost = 0. # Defines a cost related to an epoch
num_minibatches = int(
m / minibatch_size
) # number of minibatches of size minibatch_size in the train set
minibatches = random_mini_batches(X_train.T, Y_train.T,
minibatch_size)
for minibatch in minibatches:
(minibatch_X, minibatch_Y) = minibatch
minibatch_X = minibatch_X.T
minibatch_Y = minibatch_Y.T
_, minibatch_cost = sess.run([optimizer, cost],
feed_dict={
X: minibatch_X,
Y: minibatch_Y
})
epoch_cost += minibatch_cost / num_minibatches
# Print the cost every epoch
if print_cost == True and epoch % 1 == 0:
print("Cost after epoch %i: %f" % (epoch, epoch_cost))
if print_cost == True and epoch == (num_epochs - 1):
print("Final Cost after epoch %i: %f" % (epoch, epoch_cost))
if print_cost == True and epoch % 1 == 0:
costs.append(epoch_cost)
# plot the cost
plt.plot(np.squeeze(costs))
plt.ylabel('cost')
plt.xlabel('iterations (per tens)')
plt.title("Learning rate =" + str(learning_rate))
plt.show()
#save the parameters in a variable
parameters = sess.run(parameters)
print("Parameters have been trained!")
# Calculate the correct predictions
correct_prediction = tf.equal(tf.round(tf.sigmoid(out)), Y)
# Calculate accuracy on the test set
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
keep_prob = 1.0
print("Train Accuracy:%", 100 * accuracy.eval({
X: X_train,
Y: Y_train
}))
print("Test Accuracy:%", 100 * accuracy.eval({X: X_test, Y: Y_test}))
print(" ")
return parameters
def lstmnet(i):
print('Fold ', i, ' Processing')
X_build = train_df[train_df['CVindices'] != i] # 636112
#X_build = X_build.groupby('id').first().reset_index() # 331085
X_val = train_df[train_df['CVindices'] == i]
trainindex = train_df[train_df['CVindices'] != i].index.tolist()
valindex = train_df[train_df['CVindices'] == i].index.tolist()
data_1_train = data_1[trainindex]
# data_2_train = data_2[trainindex]
train_data_reg_train = train_data_reg[trainindex]
train_data_city_train = train_data_city[trainindex]
train_data_cat1_train = train_data_cat1[trainindex]
train_data_cat2_train = train_data_cat2[trainindex]
train_data_prm1_train = train_data_prm1[trainindex]
train_data_prm2_train = train_data_prm2[trainindex]
# train_data_prm3_train = train_data_prm3[trainindex]
# train_data_sqnm_train = train_data_sqnm[trainindex]
train_data_usr_train = train_data_usr[trainindex]
train_data_itype_train = train_data_itype[trainindex]
train_user_features_train = train_user_features[trainindex]
labels_train = labels[trainindex]
data_1_val = data_1[valindex]
# data_2_val = data_2[valindex]
train_data_reg_val = train_data_reg[valindex]
train_data_city_val = train_data_city[valindex]
train_data_cat1_val = train_data_cat1[valindex]
train_data_cat2_val = train_data_cat2[valindex]
train_data_prm1_val = train_data_prm1[valindex]
train_data_prm2_val = train_data_prm2[valindex]
# train_data_prm3_val = train_data_prm3[valindex]
# train_data_sqnm_val = train_data_sqnm[valindex]
train_data_usr_val = train_data_usr[valindex]
train_data_itype_val = train_data_itype[valindex]
train_user_features_val = train_user_features[valindex]
labels_val = labels[valindex]
pred_cv = np.zeros(X_val.shape[0])
pred_test = np.zeros(test_df.shape[0])
for j in range(1, nbags + 1):
print('bag ', j, ' Processing')
bag_cv = np.zeros(X_val.shape[0])
model = gru_Bidirectional_selfEmbedding_model()
# model = gru_attention_model()
callbacks = [
EarlyStopping(monitor='val_loss',
patience=patience,
verbose=VERBOSEFLAG),
ModelCheckpoint(MODEL_WEIGHTS_FILE,
monitor='val_loss',
save_best_only=True,
verbose=VERBOSEFLAG),
]
# reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=patience, min_lr=1e-9, epsilon = 0.00001, verbose=VERBOSEFLAG)
folds_nb_epoch = 10
learning_rate = 1e-3
#sgd = SGD(lr=1e-3, decay=1e-6, momentum=0.9, nesterov=True)
if optim_type == 'SGD':
optim = SGD(lr=learning_rate,
decay=1e-6,
momentum=0.9,
nesterov=True)
else:
optim = Adam(lr=learning_rate)
model.compile(optimizer=optim,
loss='mean_squared_error',
metrics=[root_mean_squared_error])
model.fit(
[
data_1_train, train_data_reg_train, train_data_city_train,
train_data_cat1_train, train_data_cat2_train,
train_data_prm1_train, train_data_prm2_train,
train_data_usr_train, train_data_itype_train,
train_user_features_train
],
labels_train,
validation_data=([
data_1_val, train_data_reg_val, train_data_city_val,
train_data_cat1_val, train_data_cat2_val, train_data_prm1_val,
train_data_prm2_val, train_data_usr_val, train_data_itype_val,
train_user_features_val
], labels_val),
batch_size=batch_size,
nb_epoch=folds_nb_epoch,
callbacks=callbacks, #[callbacks, reduce_lr],
verbose=VERBOSEFLAG)
bag_cv = model.predict([
data_1_val, train_data_reg_val, train_data_city_val,
train_data_cat1_val, train_data_cat2_val, train_data_prm1_val,
train_data_prm2_val, train_data_usr_val, train_data_itype_val,
train_user_features_val
],
batch_size=batch_size,
verbose=VERBOSEFLAG)[:, 0]
pred_cv += model.predict([
data_1_val, train_data_reg_val, train_data_city_val,
train_data_cat1_val, train_data_cat2_val, train_data_prm1_val,
train_data_prm2_val, train_data_usr_val, train_data_itype_val,
train_user_features_val
],
batch_size=batch_size,
verbose=VERBOSEFLAG)[:, 0]
pred_test += model.predict([
test_data_1, test_data_reg, test_data_city, test_data_cat1,
test_data_cat2, test_data_prm1, test_data_prm2, test_data_usr,
test_data_itype, test_user_features
],
batch_size=batch_size,
verbose=VERBOSEFLAG)[:, 0]
bag_score = np.sqrt(metrics.mean_squared_error(labels_val, bag_cv))
print('bag ', j, '- rmse:', bag_score)
del model
K.clear_session()
gc.collect()
pred_cv /= nbags
pred_test /= nbags
fold_score = np.sqrt(metrics.mean_squared_error(labels_val, pred_cv))
print('Fold ', i, '- rmse:', fold_score)
pred_cv = pd.DataFrame(pred_cv)
pred_cv.columns = ["deal_probability"]
pred_cv["item_id"] = X_val.item_id.values
pred_cv = pred_cv[['item_id', 'deal_probability']]
pred_cv['deal_probability'] = pred_cv['deal_probability'].clip(0.0, 1.0)
sub_valfile = inDir + '/submissions/Prav.nn10.fold' + str(i) + '.csv'
pred_cv.to_csv(sub_valfile, index=False)
pred_test = pd.DataFrame(pred_test)
pred_test.columns = ["deal_probability"]
pred_test["item_id"] = test_df.item_id.values
pred_test = pred_test[['item_id', 'deal_probability']]
pred_test['deal_probability'] = pred_test['deal_probability'].clip(
0.0, 1.0)
sub_file = inDir + '/submissions/Prav.nn10.fold' + str(
i) + '-test' + '.csv'
pred_test.to_csv(sub_file, index=False)
del pred_cv
del pred_test
def clear_session():
K.clear_session()
gc.collect()
