def try_train_model_nn(model_name, fold):
with utils.timeit_context('load data'):
X, y, video_ids = load_train_data(model_name, fold)
print(X.shape, y.shape)
model = model_nn(input_size=X.shape[1])
model.compile(optimizer=Adam(lr=1e-3),
loss='binary_crossentropy',
metrics=['accuracy'])
model.summary()
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=0.25,
random_state=42)
batch_size = 64
model.fit(
X_train,
y_train,
batch_size=batch_size,
epochs=128,
verbose=1,
validation_data=[X_test, y_test],
callbacks=[ReduceLROnPlateau(factor=0.2, verbose=True, min_lr=1e-6)])
prediction = model.predict(X_test)
print(y_test.shape, prediction.shape)
print(metrics.pri_matrix_loss(y_test, prediction))
print(metrics.pri_matrix_loss(y_test, np.clip(prediction, 0.001, 0.999)))
delta = prediction - y_test
print(np.min(delta), np.max(delta), np.mean(np.abs(delta)),
np.sum(np.abs(delta) > 0.5))
def try_train_all_models_nn_combined(models_with_folds):
X_all_combined = []
y_all_combined = []
for model_with_folds in models_with_folds:
X_combined = []
y_combined = []
for model_name, fold in model_with_folds:
with utils.timeit_context('load data'):
X, y, video_ids = load_train_data(model_name, fold)
X_combined.append(X)
y_combined.append(y)
X_all_combined.append(np.row_stack(X_combined))
y_all_combined.append(np.row_stack(y_combined))
X = np.column_stack(X_all_combined)
y = y_all_combined[0]
print(X.shape, y.shape)
model = model_nn_combined(input_size=X.shape[1])
model.compile(optimizer=Adam(lr=1e-4),
loss='binary_crossentropy',
metrics=['accuracy'])
model.summary()
batch_size = 64
def cheduler(epoch):
if epoch < 32:
return 1e-3
if epoch < 48:
return 4e-4
if epoch < 80:
return 1e-4
return 1e-5
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=0.25,
random_state=42)
model.fit(X_train,
y_train,
batch_size=batch_size,
epochs=128,
verbose=1,
validation_data=[X_test, y_test],
callbacks=[LearningRateScheduler(schedule=cheduler)])
prediction = model.predict(X_test)
print(y_test.shape, prediction.shape)
print(metrics.pri_matrix_loss(y_test, prediction))
print(metrics.pri_matrix_loss(y_test, np.clip(prediction, 0.001, 0.999)))
print(metrics.pri_matrix_loss(y_test, np.clip(prediction, 0.0001, 0.9999)))
delta = prediction - y_test
print(np.min(delta), np.max(delta), np.mean(np.abs(delta)),
np.sum(np.abs(delta) > 0.5))
def try_train_model_xgboost(model_name, fold):
with utils.timeit_context('load data'):
X, y, video_ids = load_train_data(model_name, fold)
y_cat = np.argmax(y, axis=1)
print(X.shape, y.shape)
print(np.unique(y_cat))
X_train, X_test, y_train, y_test = train_test_split(X,
y_cat,
test_size=0.25,
random_state=42)
model = XGBClassifier(n_estimators=500,
objective='multi:softprob',
silent=True)
model.fit(X,
y_cat,
eval_set=[(X_test, y_test)],
early_stopping_rounds=20,
verbose=True)
# model = ExtraTreesClassifier(n_estimators=500, max_features=32)
# pickle.dump(model, open(f"../output/et_{model_name}.pkl", "wb"))
# model = pickle.load(open(f"../output/et_{model_name}.pkl", "rb"))
#
# print(model)
# model.fit(X_train, y_train)
prediction = model.predict_proba(X_test)
if prediction.shape[1] == 23: # insert mission lion col
prediction = np.insert(prediction, obj=12, values=0.0, axis=1)
print(model.score(X_test, y_test))
y_test_one_hot = np.eye(24)[y_test]
print(y_test.shape, prediction.shape, y_test_one_hot.shape)
print(metrics.pri_matrix_loss(y_test_one_hot, prediction))
print(
metrics.pri_matrix_loss(y_test_one_hot,
np.clip(prediction, 0.001, 0.999)))
delta = prediction - y_test_one_hot
print(np.min(delta), np.max(delta), np.mean(np.abs(delta)),
np.sum(np.abs(delta) > 0.5))
avg_prob = avg_probabilities()
# print(avg_prob)
avg_pred = np.repeat([avg_prob], y_test_one_hot.shape[0], axis=0)
print(metrics.pri_matrix_loss(y_test_one_hot, avg_pred))
print(
metrics.pri_matrix_loss(y_test_one_hot,
avg_pred * 0.1 + prediction * 0.9))
def try_train_model_lgb(model_name, fold):
with utils.timeit_context('load data'):
X, y, video_ids = load_train_data(model_name, fold)
y_cat = np.argmax(y, axis=1)
print(X.shape, y.shape)
print(np.unique(y_cat))
X_train, X_test, y_train, y_test = train_test_split(X,
y_cat,
test_size=0.25,
random_state=42)
train_data = lgb.Dataset(X_train, label=y_train)
param = {
'num_leaves': 50,
'objective': 'multiclass',
'max_depth': 5,
'learning_rate': .1,
'max_bin': 200,
'num_class': NB_CAT,
'metric': ['multi_logloss']
}
model = lgb.train(param,
train_data,
valid_sets=lgb.Dataset(X_test, label=y_test),
num_boost_round=100,
verbose_eval=True)
prediction = model.predict(X_test)
# if prediction.shape[1] == 23: # insert mission lion col
# prediction = np.insert(prediction, obj=12, values=0.0, axis=1)
# print(model.score(X_test, y_test))
y_test_one_hot = np.eye(24)[y_test]
print(y_test.shape, prediction.shape, y_test_one_hot.shape)
print(metrics.pri_matrix_loss(y_test_one_hot, prediction))
print(
metrics.pri_matrix_loss(y_test_one_hot,
np.clip(prediction, 0.001, 0.999)))
delta = prediction - y_test_one_hot
print(np.min(delta), np.max(delta), np.mean(np.abs(delta)),
np.sum(np.abs(delta) > 0.5))
avg_prob = avg_probabilities()
# print(avg_prob)
avg_pred = np.repeat([avg_prob], y_test_one_hot.shape[0], axis=0)
print(metrics.pri_matrix_loss(y_test_one_hot, avg_pred))
print(
metrics.pri_matrix_loss(y_test_one_hot,
avg_pred * 0.1 + prediction * 0.9))
def check_model_score(model_name, weights, fold):
model = MODELS[model_name].factory(lock_base_model=True)
model.load_weights(weights, by_name=True)
model.compile(optimizer=RMSprop(lr=3e-4),
loss='binary_crossentropy',
metrics=['accuracy'])
dataset = SingleFrameCNNDataset(
preprocess_input_func=MODELS[model_name].preprocess_input,
batch_size=MODELS[model_name].batch_size,
validation_batch_size=4, # MODELS[model_name].batch_size,
fold=fold,
use_non_blank_frames=True)
gt = []
pred = []
steps = dataset.validation_steps() // 500
print('steps:', steps)
step = 0
for X, y in tqdm(dataset.generate_test(verbose=True)):
gt.append(y)
pred.append(model.predict_on_batch(X))
step += 1
if step >= steps:
break
# print(gt)
# print(pred)
gt = np.vstack(gt).astype(np.float64)
pred = np.vstack(pred).astype(np.float64)
print(gt.shape, pred.shape)
# print(gt)
# print(pred)
checkpoint_name = os.path.basename(weights)
out_dir = f'../output/check_model_score/gt{model_name}_{fold}_{checkpoint_name}'
os.makedirs(out_dir, exist_ok=True)
print(out_dir)
np.save(f'{out_dir}/gt.npy', gt)
np.save(f'{out_dir}/pred.npy', pred)
for clip in [0.0, 1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1]:
print(clip, metrics.pri_matrix_loss(gt,
np.clip(pred, clip, 1.0 - clip)))
def try_train_all_models_lgb_combined(models_with_folds):
X_all_combined = []
y_all_combined = []
requests = []
results = []
for model_with_folds in models_with_folds:
for model_name, fold in model_with_folds:
requests.append((model_name, fold))
# results.append(load_one_model(requests[-1]))
pool = Pool(20)
with utils.timeit_context('load all data'):
results = pool.starmap(load_train_data, requests)
for model_with_folds in models_with_folds:
X_combined = []
y_combined = []
for model_name, fold in model_with_folds:
X, y, video_ids = results[requests.index((model_name, fold))]
print(model_name, fold, X.shape)
X_combined.append(X)
y_combined.append(y)
X_all_combined.append(np.row_stack(X_combined))
y_all_combined.append(np.row_stack(y_combined))
X = np.column_stack(X_all_combined)
y = y_all_combined[0]
print(X.shape, y.shape)
y_cat = np.argmax(y, axis=1)
print(X.shape, y.shape)
print(np.unique(y_cat))
X_train, X_test, y_train, y_test = train_test_split(X,
y_cat,
test_size=0.25,
random_state=42)
train_data = lgb.Dataset(X_train, label=y_train)
with utils.timeit_context('fit 200 est'):
param = {
'num_leaves': 50,
'objective': 'multiclass',
'max_depth': 5,
'learning_rate': .05,
'max_bin': 300,
'num_class': NB_CAT,
'metric': ['multi_logloss']
}
model = lgb.train(param,
train_data,
valid_sets=lgb.Dataset(X_test, label=y_test),
num_boost_round=300)
prediction = model.predict(X_test)
# if prediction.shape[1] == 23: # insert mission lion col
# prediction = np.insert(prediction, obj=12, values=0.0, axis=1)
# print(model.score(X_test, y_test))
y_test_one_hot = np.eye(24)[y_test]
print(y_test.shape, prediction.shape, y_test_one_hot.shape)
print(metrics.pri_matrix_loss(y_test_one_hot, prediction))
print(
metrics.pri_matrix_loss(y_test_one_hot,
np.clip(prediction, 0.001, 0.999)))
delta = prediction - y_test_one_hot
print(np.min(delta), np.max(delta), np.mean(np.abs(delta)),
np.sum(np.abs(delta) > 0.5))
avg_prob = avg_probabilities()
# print(avg_prob)
avg_pred = np.repeat([avg_prob], y_test_one_hot.shape[0], axis=0)
print(metrics.pri_matrix_loss(y_test_one_hot, avg_pred))
print(
metrics.pri_matrix_loss(y_test_one_hot,
avg_pred * 0.1 + prediction * 0.9))