def training(model_name):
# Data parameters
data_dir = r'/src/'
seq_length = 40
n_videos = {'train': 1000, 'validation': 100}
image_size = (50, 88)
# Training parameters
n_epochs = 50
batch_size = 8
steps_per_epoch = n_videos['train'] // batch_size
# Load data generators
data = CNN3DDataLoader(data_dir,
seq_length=seq_length,
n_videos=n_videos,
labels=labels_want)
train_gen = data.sequence_generator('train', batch_size, image_size)
validation_gen = data.sequence_generator('validation', batch_size,
image_size)
#optimizer = keras.optimizers.SGD(lr=0.1, momentum=0.9, decay= 1e-6, nesterov=True)
#Load model
optimizer = keras.optimizers.Adadelta()
ml = ModelLoader(data.n_labels,
data.seq_length,
model_name,
image_size=image_size,
optimizer=optimizer)
model = ml.model
#Define callbacks
checkpointer = ModelCheckpoint(filepath='model_name' +
'-{epoch:03d}-{loss:.3f}.hdf5',
verbose=1,
save_best_only=True)
tb = TensorBoard(log_dir='./models/logs')
early_stopper = EarlyStopping(patience=2)
csv_logger = CSVLogger('./models/logs/' + model_name + '-' + 'training-' + \
str(time.time()) + '.log')
callbacks = [tb, early_stopper, csv_logger, checkpointer]
# Training
print('Starting training')
history = model.fit_generator(
generator=train_gen,
steps_per_epoch=20,
#sample_per_epoch= 200,
epochs=n_epochs,
verbose=1,
callbacks=callbacks,
validation_data=validation_gen,
validation_steps=10,
)
model.save('./my_model.h5')
save_history(history, "c3d")
"verbosity": 0,
"seed": 42
}
def fit(self, train, cv):
x_tr, y_tr = train
x_cv, y_cv = cv
trn_data = lgb.Dataset(x_tr, label=y_tr)
val_data = lgb.Dataset(x_cv, label=y_cv)
evals_result = {}
self.model = lgb.train(self.lgb_params,
trn_data,
100000,
valid_sets=[trn_data, val_data],
early_stopping_rounds=3000,
verbose_eval=1000,
evals_result=evals_result)
def predict(self, test):
return self.model.predict(test)
model = ModelLoader(LightGbmTrainer, model_params)
results = model.run(data_loader, roc_auc_score, {}, {}, verbose=True)
if args.save:
current_file_path = os.path.abspath(__file__) # to save this .py file
model.save(data_loader, results, current_file_path, args.preds,
args.models)
## << Create and train model
n_splits=5,
shuffle=True,
random_state=42)
## << Read and preprocess data
## >> Create and train model
from sklearn.naive_bayes import GaussianNB
from sklearn.metrics import roc_auc_score
model_params = {
'name': "gaussian",
'fit': "fit",
'predict': "predict_proba",
'pred_col': 1
}
model = ModelLoader(GaussianNB, model_params)
fit_params = {}
predict_params = {}
results = model.run(data_loader,
roc_auc_score,
fit_params,
predict_params,
verbose=True)
if args.save:
current_file_path = os.path.abspath(__file__) # to save this .py file
model.save(data_loader, results, current_file_path, args.preds,
args.models)
## << Create and train model
from sklearn.metrics import roc_auc_score
model_params = {
'name': "logreg",
'fit': "fit",
'predict': "predict_proba",
'pred_col': 1
}
logreg_params = {
'C': 0.9,
'penalty': 'l2',
'multi_class': 'ovr',
'solver': 'liblinear',
'random_state': 42
}
model = ModelLoader(LogisticRegression, model_params, **logreg_params)
fit_params = {}
predict_params = {}
results = model.run(data_loader,
roc_auc_score,
fit_params,
predict_params,
verbose=True)
if args.save:
current_file_path = os.path.abspath(__file__) # to save this .py file
model.save(data_loader, results, current_file_path, args.preds,
args.models)
## << Create and train model
def transform(self, X):
return X
class IsUniqueOnline(GenericDataPreprocessor):
def __init__(self):
self.counters = np.load(os.path.join('data/', 'value_counters.npy'))
def fit_transform(self, X, y=None):
return self.transform(X), y
def transform(self, X):
var_cols = ['var_{}'.format(x) for x in range(200)]
for i, c in enumerate(var_cols):
X['count_' + c] = (X[c].map(self.counters[i]) == 1)
return X
model = ModelLoader(LightGbmTrainer, model_params)
model.preprocess_online(UpsamplingPreprocessor, IsUniqueOnline)
results = model.run(data_loader, roc_auc_score, {}, {}, verbose=True)
if args.save:
current_file_path = os.path.abspath(__file__) # to save this .py file
model.save(data_loader, results, current_file_path, args.preds,
args.models)
## << Create and train model