def step_six(liste):
#Verify csv
path_data = "dataset/image/dataset"
path_folder = "dataset/image/dataset/{}"
path_image = "dataset/image/dataset/{}/{}"
liste_path = os.listdir(path_data)
path_label = "dataset/information_data/label.py"
for i in liste_path:
print(i)
picture_folder = os.listdir(path_folder.format(i))
if len(picture_folder) > 10 and i != "assiette":
for info_size in liste:
if info_size[2] == path_folder.format(i):
size = define_size(info_size)
number_pix = size[0] * size[1]
write_labels(path_label, "None", str(i), "None",
str(size[0]), str(size[1]), "None")
csv_name = "training/csv/in_training/" + str(i) + ".csv"
model_name = "training/models/in_training/" + str(i)
head_writting(csv_name, number_pix)
picture_writting(csv_name, path_folder.format(i), "", size[0],
size[1], "1")
negativ_training(i, csv_name, size)
train(csv_name, model_name)
def __train(args):
if not os.path.exists(args.input):
os.mkdir(args.input)
documents = data.get_documents(args.base)
print('Documents: {}'.format(len(documents)))
all_features = get_db_features().find_one()
features = {}
for feature_id in all_features:
if feature_id == '_id':
continue
if all_features[feature_id]['type']['name'] not in []:
features[feature_id] = all_features[feature_id]
features = data.nltk_feature_filter(features)
print('skipped {} entries'.format(len(all_features) - len(features)))
result = []
max_retrys = 3
if 'lda' in args.train:
retrys = 0
success = False
while not success and retrys < max_retrys:
mdl, file_prefix = tp_lda.create_model(args.lda_k1, args.alpha, args.eta)
data_list, mdl, success = training.train(mdl, documents, features, args.input, file_prefix,
args.iterations, args.burn_in)
retrys += 1
res = tp_lda.save_model(mdl, args.lda_k1, data_list, args.input, file_prefix)
result.append(res)
if 'pa' in args.train:
retrys = 0
success = False
while not success and retrys < max_retrys:
mdl, file_prefix = tp_pachinko.create_model(args.pa_k1, args.pa_k2, args.alpha, args.sub_alpha, args.eta)
data_list, mdl, success = training.train(mdl, documents, features, args.input, file_prefix,
args.iterations, args.burn_in)
retrys += 1
res = tp_pachinko.save_model(mdl, args.pa_k1, args.pa_k2, data_list, args.input, file_prefix)
result.append(res)
return result
def train(self, eta=0.4, num_clusters=40, n_iter=15):
self.eta = eta
self.num_clusters = num_clusters
self.n_iter = n_iter
try:
self.weights, self.matrix_centers = train(
num_clusters=self.num_clusters,
eta=self.eta,
n_iter=self.n_iter,
beta=self.beta,
cross_validation=True,
features=self.X,
labels=self.y)
except:
self.weights, self.matrix_centers = train(
num_clusters=self.num_clusters,
eta=self.eta,
n_iter=self.n_iter,
beta=self.beta)
optimizer = optim.Adamax(model.parameters(), lr=learning_rate, eps=1.e-7)
''' TRAINING '''
# for early stopping
best_loss = np.inf
best_bpd = np.inf
train_loss = []
val_loss = []
epoch = 0
train_times = []
for epoch in range(1, epochs + 1):
t_start = time.time()
tr_loss = train(train_loader, model, flow_name, optimizer, input_size,
epoch)
train_loss.append(tr_loss)
v_loss = evaluate(val_loader, model, input_size)
val_loss.append(v_loss)
# early-stopping
if v_loss < best_loss:
e = 0
best_loss = v_loss
print('Model saved....')
torch.save(model, 'test.model')
#for name, param in model.named_parameters():
# if param.requires_grad:
# print(name)
if __name__ == "__main__":
# Parse our command line arguments
command = argparse.ArgumentParser(
description='Utility for training a Visual Mesh network')
command.add_argument('command', choices=['train', 'test'], action='store')
command.add_argument('config',
action='store',
help='Path to the configuration file for training')
command.add_argument('output_path',
nargs='?',
action='store',
help='Output directory to store the logs and models')
args = command.parse_args()
# Load our yaml file and convert it to an object
with open(args.config) as f:
config = yaml.load(f)
config = Config(**config)
output_path = 'output' if args.output_path is None else args.output_path
# Run the appropriate action
if args.command == 'train':
training.train(config, output_path)
elif args.command == 'test':
testing.test(config, output_path)
device = torch.device("cuda" if use_cuda else "cpu")
model = ResNet18().to(device)
summary(model, input_size=(3, 32, 32))
epochs = 1
optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.9)
test_losses = []
train_losses = []
test_accuracy = []
train_accuracy = []
for epoch in range(0, epochs):
print("EPOCH:", epoch)
train(model,
device,
train_loader,
optimizer,
epoch,
train_losses,
train_accuracy,
L1lambda=0.001)
test(model, device, test_loader, test_losses, test_accuracy)
showMeasurePlots.plot_accuracy_curves(train_accuracy, test_accuracy, epochs)
showMeasurePlots.plot_loss_curves(train_accuracy, test_accuracy, epochs)
misclass_data_r, misclass_targets_r, misclass_pred_r = identifyImages.misclassified(
model, test_loader, device)
correctclass_data_r, correctclass_targets_r, correctclass_pred_r = identifyImages.correct_classified(
model, test_loader, device)
from matplotlib.pyplot import figure
fig = plt.figure(num=None,
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--device',
type=str,
default='gpu',
help='For cpu: \'cpu\', for gpu: \'gpu\'')
parser.add_argument('--chunk_size',
type=int,
default=36,
help='chunk size(sequence length)')
parser.add_argument('--step_size',
type=int,
default=1,
help='sequence split step')
parser.add_argument('--lr', type=float, default=5e-4, help='learning rate')
parser.add_argument('--weight_decay',
type=argtype.check_float,
default='1e-2',
help='weight_decay')
parser.add_argument('--epoch',
type=argtype.epoch,
default='inf',
help='the number of epoch for training')
parser.add_argument('--batch_size',
type=int,
default=256,
help='size of batches for training')
parser.add_argument('--val_ratio',
type=float,
default=.3,
help='validation set ratio')
parser.add_argument('--model_name',
type=str,
default='main_model',
help='model name to save')
parser.add_argument('--transfer',
type=argtype.boolean,
default=False,
help='whether fine tuning or not')
parser.add_argument('--oversample_times',
type=int,
default=30,
help='the times oversampling times for fine tuning')
parser.add_argument('--patience',
type=int,
default=20,
help='patience for early stopping')
parser.add_argument('--c_loss',
type=argtype.boolean,
default=True,
help='whether using custom loss or not')
parser.add_argument('--predict',
type=argtype.boolean,
default=False,
help='predict and save csv file or not')
parser.add_argument('--filename',
type=str,
default='submission',
help='csv file name to save predict result')
parser.add_argument('--Y_list',
type=argtype.str_to_list,
default='Y12,Y15',
help='target Y for pre-training')
parser.add_argument('--window_size',
type=int,
default=1,
help='window size for moving average')
parser.add_argument('--attention',
type=argtype.boolean,
default=True,
help='select model using attention mechanism')
args = parser.parse_args()
data_dir = './data'
if args.device == 'gpu':
args.device = 'cuda'
device = torch.device(args.device)
chunk_size = args.chunk_size
step_size = args.step_size
lr = args.lr
weight_decay = args.weight_decay
EPOCH = args.epoch
batch_size = args.batch_size
val_ratio = args.val_ratio
model_name = args.model_name
transfer_learning = args.transfer
times = args.oversample_times
patience = args.patience
c_loss = args.c_loss
pred = args.predict
filename = args.filename
Y_list = args.Y_list
window_size = args.window_size
attention = args.attention
params = {
'chunk_size': chunk_size,
'step_size': step_size,
'learning_rate': lr,
'weight_decay': weight_decay,
'epoch size': EPOCH,
'batch_size': batch_size,
'valid_ratio': val_ratio,
'model_name': model_name,
'transfer_learning': transfer_learning,
'oversample_times': times,
'early_stopping_patience': patience,
'c_loss': c_loss,
'pred': pred,
'filename': filename,
'Y_list': Y_list,
'window_size': window_size,
'attention': attention
}
Y = ''
for y in Y_list:
Y += y
model_name = f'{model_name}/{Y}'
Dataframe = dataframe.Dataframe(data_dir=data_dir)
input_size = len(Dataframe.feature_cols)
if attention:
model = regressor.Attention_Regressor(input_size).to(device)
else:
model = regressor.BiLSTM_Regressor().to(device)
checkpoint = Checkpoint(model_name=model_name,
transfer_learning=transfer_learning)
early_stopping = Early_stopping(patience=patience)
vis = Custom_Visdom(model_name, transfer_learning)
vis.print_params(params)
if transfer_learning:
dataset_list = []
if attention:
pre_df = Dataframe.get_pretrain_df()\
.iloc[-chunk_size+1:][Dataframe.feature_cols]
df = Dataframe.get_y18_df()
df = pd.concat([pre_df, df], axis=0)
else:
df = Dataframe.get_y18_df()
train_dataset = datasets.CustomSequenceDataset(chunk_size=chunk_size,
df=df,
Y='Y18',
step_size=step_size,
noise=True,
times=times)
dataset_list.append(train_dataset)
dataset = ConcatDataset(dataset_list)
train_loader, valid_loader = datasets.split_dataset(
dataset=dataset,
batch_size=batch_size,
val_ratio=val_ratio,
shuffle=True)
checkpoint.load_model(model)
else:
dataset_list = []
for y in Y_list:
df = Dataframe.get_pretrain_df()
df[y] = df[y].rolling(window=window_size, min_periods=1).mean()
dataset = datasets.CustomSequenceDataset(chunk_size=chunk_size,
df=df,
Y=y,
step_size=step_size,
noise=False,
times=1)
dataset_list.append(dataset)
dataset = ConcatDataset(dataset_list)
train_loader, valid_loader = datasets.split_dataset(
dataset=dataset,
batch_size=batch_size,
val_ratio=val_ratio,
shuffle=True)
optimizer = Adam(model.parameters(),
lr=lr,
weight_decay=float(weight_decay))
if c_loss:
criterion = custom_loss.mse_AIFrenz_torch
else:
criterion = nn.MSELoss()
training_time = time.time()
epoch = 0
y_df = Dataframe.get_pretrain_df()[Y_list]
y18_df = Dataframe.get_y18_df()[['Y18']]
while epoch < EPOCH:
print(f'\r Y: {Y} \
chunk size: {chunk_size} \
transfer: {transfer_learning}')
epoch += 1
train_loss_per_epoch, train_loss_list_per_batch, batch_list = train(
model=model,
train_loader=train_loader,
criterion=criterion,
optimizer=optimizer,
epoch=epoch,
transfer_learning=transfer_learning,
attention=attention,
freeze_name='transfer_layer')
valid_loss = valid(model=model,
valid_loader=valid_loader,
criterion=criterion,
attention=attention)
iter_time = time.time() - training_time
print(
f'\r Epoch: {epoch:3d}/{str(EPOCH):3s}\t',
f'train time: {int(iter_time//60):2d}m {iter_time%60:5.2f}s\t'
f'avg train loss: {train_loss_per_epoch:7.3f}\t'
f'valid loss: {valid_loss:7.3f}')
checkpoint.save_log(batch_list, epoch, train_loss_list_per_batch,
train_loss_per_epoch, valid_loss)
early_stop, is_best = early_stopping(valid_loss)
checkpoint.save_checkpoint(model, optimizer, is_best)
vis.print_training(EPOCH, epoch, training_time, train_loss_per_epoch,
valid_loss, patience, early_stopping.counter)
vis.loss_plot(checkpoint)
print('-----' * 17)
y_true, y_pred, y_idx = predict.trainset_predict(
model=model,
data_dir=data_dir,
Y=Y_list[0],
chunk_size=chunk_size,
attention=attention,
window_size=window_size)
y18_true, y18_pred, y18_idx = predict.trainset_predict(
model=model,
data_dir=data_dir,
Y='Y18',
chunk_size=chunk_size,
attention=attention,
window_size=window_size)
y_df['pred'] = y_pred
y18_df['pred'] = y18_pred
vis.predict_plot(y_df, 'pre')
vis.predict_plot(y18_df, 'trans')
vis.print_error()
if early_stop:
break
if transfer_learning:
checkpoint.load_model(model, transfer_learningd=True)
else:
checkpoint.load_model(model, transfer_learningd=False)
y_true, y_pred, y_idx = predict.trainset_predict(model=model,
data_dir=data_dir,
Y=Y_list[0],
chunk_size=chunk_size,
attention=attention,
window_size=window_size)
y18_true, y18_pred, y18_idx = predict.trainset_predict(
model=model,
data_dir=data_dir,
Y='Y18',
chunk_size=chunk_size,
attention=attention,
window_size=window_size)
y_df['pred'] = y_pred
y18_df['pred'] = y18_pred
vis.predict_plot(y_df, 'pre')
vis.predict_plot(y18_df, 'trans')
vis.print_error()
if pred:
predict.test_predict(model=model,
chunk_size=chunk_size,
filename=filename,
attention=attention)
help='Print losses after every [this many] training iterations')
parser.add_argument('--save-every', type=int, default=500,
help='Save the state of the network after every [this many] training iterations')
parser.add_argument('--use-gpu', type=bool, default=torch.cuda.is_available(),
help='Should we use gpu')
parser.add_argument('--backup-saves', action='store_true')
parser.add_argument('--reset-optimizer', action='store_true')
args = parser.parse_args()
if args.mode == 'train':
train(args)
elif args.mode == 'test':
model, _, _ = load_model(args=args)
loader = image_loader(args)
for file in glob.glob('data/inputs/*.JPEG'):
inp, gl, loh, lom, lab = loader(file)
out = model(inp, gl, loh, lom)[0][0]
out = torch.cat((inp, out.double()), dim=-1).detach().numpy()
out = skimage.color.lab2rgb(out)
skimage.io.imsave(file.replace('inputs', 'outputs'), out)
elif args.mode == 'analyze':
analyze(args)
else:
raise ValueError('--mode should be one of "train" or "test" or "analyze".')
# Check dependencies
command.add_argument('output_path',
nargs='?',
action='store',
help='Output directory to store the logs and models')
args = command.parse_args()
# Load our yaml file
with open(args.config) as f:
config = yaml.load(f)
# Tensorflow configuration
tf_config = tf.ConfigProto()
tf_config.allow_soft_placement = True
tf_config.graph_options.build_cost_model = 1
tf_config.gpu_options.allow_growth = True
with tf.Session(config=tf_config) as sess:
# Select our device to run operations on
with tf.device('/device:GPU:{}'.format(args.gpu)):
output_path = 'output' if args.output_path is None else args.output_path
# Run the appropriate action
if args.command == 'train':
training.train(sess, config, output_path)
elif args.command == 'test':
test.test(sess, config, output_path)
optimizer = optim.SGD(model.parameters(), lr=best_lr, momentum=0.9,weight_decay=L2lambda)
lrscheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer,
mode='min',
factor=0.1, # learning rate will be reduced by this factor
patience=10, # wait foe epochs to reduce
verbose=True, #print the update
threshold=0.0001,
threshold_mode='rel',
cooldown=0, #wait till we can resumen the normal operation
min_lr=0,
eps=1e-08)
for epoch in range(0, epochs):
print("EPOCH:",epoch)
train(model, device, train_loader, optimizer, train_losses, train_accuracy, L1lambda=0,scheduler=None)
test(model, device, test_loader,test_losses,test_accuracy)
torch.save(model,"model_s10")
showMeasurePlots.plot_accuracy_curves(train_accuracy,test_accuracy,epochs)
showMeasurePlots.plot_loss_curves(train_losses,test_losses,epochs)
misclass_data_r, misclass_targets_r,misclass_pred_r = identifyImages.misclassified(model,test_loader,device)
from matplotlib.pyplot import figure
from utils import denormalize
fig = plt.figure(num=None, figsize=(10, 10), dpi=80, facecolor='w', edgecolor='k')
misclass_targets_r_cpu = misclass_targets_r.cpu().numpy()
misclass_pred_r_cpu = misclass_pred_r.cpu().numpy()
for num in range(0,20):
plt.subplot(5,5,num+1)
if not os.path.exists(config_path) or not os.path.isfile(config_path):
print("The configuration file {} does not exist".format(config_path))
exit(1)
# Load our yaml file and convert it to an object
with open(config_path) as f:
config = yaml.safe_load(f)
# Make all GPUs grow memory as needed
gpus = tf.config.experimental.list_physical_devices("GPU")
if gpus:
try:
for gpu in gpus:
tf.config.experimental.set_memory_growth(gpu, True)
except RuntimeError as e:
print(e)
# Run the appropriate action
if args.command == "train":
training.train(config, network_path)
elif args.command == "test":
testing.test(config, network_path)
elif args.command == "export":
export.export(config, network_path)
elif args.command == "find_lr":
find_lr.find_lr(config, network_path, args.min_lr, args.max_lr,
args.steps, args.window_size)