def train_SPOS(epoch,
lr,
frame_path,
frequency,
batch_size,
sub,
file_name="spos_training"):
'''
training function for SPOS databases, 7 cross validation
Parameters
----------
epoch : int,how many epochs used to train the network
lr : float,learning rate for the network
frame_path : str, the path that contains processed frames, the label can be read from the path,
thus no label path required
frequency : int, how many frames used per second
batch_size : int, the number of data for mini-batch size
file_name : path, the place to store the trained nn weights.
'''
for index, name in enumerate(
['tomas', 'nelly', 'riku', 'yi', 'ying', 'rui', 'xiaopeng']):
c = 0
c2 = 0
traind = dict()
testd = dict()
for i in ["posed", "spontaneous"]:
path = os.join(frame_path, i, "happy")
for j in os.listdir(path):
k = os.path.join(path, j)
if not os.path.isdir(k):
continue
if name in k:
testd[c2] = k
c2 += 1
else:
traind[c] = k
c += 1
params = {"fold": traind, "frequency": 5}
dg = SPOSDataGenerator(**params)
training_generator = torch.utils.data.DataLoader(dg,
batch_size=batch_size,
shuffle=True)
params = {"fold": testd, "test": True, "frequency": 5}
test_generator = torch.utils.data.DataLoader(
SPOSDataGenerator(**params), batch_size=32, shuffle=True)
train(epoch, lr, DeepSmileNet(re=sub), file_name, training_generator,
test_generator, name)
def train_BBC(epoch,
lr,
frame_path,
frequency,
batch_size,
sub,
file_name="bbc_training"):
'''
training function for BBC databases, 10 cross validation
Parameters
----------
epoch : int,how many epochs used to train the network
lr : float,learning rate for the network
frame_path : str, the path that contains processed frames, the label can be read from the path,
thus no label path required
frequency : int, how many frames used per second
batch_size : int, the number of data for mini-batch size
file_name : path, the place to store the trained nn weights.
'''
# subjects
a = list(range(20))
np.random.seed(12)
np.random.shuffle(a)
b = a[10:]
a = a[:10]
c = list(zip(a, b))
from copy import deepcopy
for name, file2 in enumerate(c):
file = deepcopy(c)
file.remove(file2)
params = {
"fold": file,
"frame_path": frame_path,
"frequency": frequency
}
dg = BBCDataGenerator(**params)
training_generator = torch.utils.data.DataLoader(dg,
batch_size=batch_size,
shuffle=True)
params = {
"fold": file2,
"frame_path": frame_path,
"test": True,
"frequency": frequency
}
test_generator = torch.utils.data.DataLoader(
BBCDataGenerator(**params), batch_size=32, shuffle=True)
train(epoch, lr, DeepSmileNet(re=sub), file_name, training_generator,
test_generator, name)
def train_UVANEMO(epoch,
lr,
label_path,
frame_path,
frequency,
batch_size,
sub,
file_name="uvanemo_training"):
'''
training function for UVANEMO databases
Parameters
----------
epoch : int,how many epochs used to train the network
lr : float,learning rate for the network
label_path : str, the path that contains the details of video and label in the cross validation
frame_path : str, the path that contains processed frames
frequency : int, how many frames used per second
batch_size : int, the number of data for mini-batch size
file_name : path, the place to store the trained nn weights.
'''
for file in os.listdir(label_path):
current_path = os.path.join(label_path, file)
if not os.path.isdir(current_path):
continue
train_labels = os.path.join(current_path, "train.json")
params = {
"label_path": train_labels,
"frame_path": frame_path,
"frequency": frequency
}
dg = UVANEMODataGenerator(**params)
training_generator = torch.utils.data.DataLoader(dg,
batch_size=batch_size,
shuffle=True)
test_labels = os.path.join(current_path, "test.json")
params = {
"label_path": test_labels,
"frame_path": frame_path,
"test": True,
"frequency": frequency
}
test_generator = torch.utils.data.DataLoader(
UVANEMODataGenerator(**params), batch_size=32, shuffle=True)
train(epoch, lr, DeepSmileNet(re=sub), file_name, training_generator,
test_generator, file)
def insert():
data = request.json
resp, template_recognition_lentgh = insertPerson(
getEncode(carpeta_standby + data['cliente'] + '/' + data['foto']),
data['cedula'], data['category'], data['status'], data['cliente'])
ruta_foto = carpeta_standby + data['cliente'] + '/' + data['foto']
new_nombre = ruta_foto.replace(
'.jpg', '-' + str(template_recognition_lentgh - 1) + '.jpg')
os.rename(ruta_foto, new_nombre)
moveToFotos(new_nombre, data['cedula'])
train(carpeta_fotos, model_save_path="modeloknn.clf", n_neighbors=1)
# eliminarImagen(carpeta_standby+data['cliente']+'/'+data['foto'])
return data
def post(self):
file1 = self.request.files['file1'][0]
original_fname = file1['filename']
extension = os.path.splitext(original_fname)[1]
fname = ''.join(random.choice(string.ascii_lowercase + string.digits) for x in xrange(6))
final_filename = fname+extension
with open(final_filename, 'w') as out_file:
out_file.write(file1['body'])
font = "dejavusans-alphanumeric"
fontimg = mpimg.imread('train/' + font + '.jpg')
util.train(fontimg, font)
testimg = mpimg.imread(final_filename)
self.write(util.test(testimg, font))
def client_update(self, global_model, global_init_model, comm_round):
self.elapsed_comm_rounds += 1
print(f'***** Client #{self.client_id} *****', flush=True)
self.model = copy_model(global_model, self.args.dataset,
self.args.arch)
losses = []
accuracies = []
for epoch in range(self.args.client_epoch):
train_score = train(comm_round,
self.client_id,
epoch,
self.model,
self.train_loader,
lr=self.args.lr,
verbose=self.args.train_verbosity)
losses.append(train_score['Loss'][-1].data.item())
accuracies.append(train_score['Accuracy'][-1])
num_pruned, num_params = get_prune_summary(self.model)
cur_prune_rate = num_pruned / num_params
print(
f"num_pruned {num_pruned}, num_params {num_params}, cur_prune_rate {cur_prune_rate}"
)
self.losses[comm_round:] = np.array(losses)
self.accuracies[comm_round:] = np.array(accuracies)
self.prune_rates[comm_round:] = cur_prune_rate
def post(self):
file1 = self.request.files['file1'][0]
original_fname = file1['filename']
extension = os.path.splitext(original_fname)[1]
fname = ''.join(
random.choice(string.ascii_lowercase + string.digits)
for x in xrange(6))
final_filename = fname + extension
with open(final_filename, 'w') as out_file:
out_file.write(file1['body'])
font = "dejavusans-alphanumeric"
fontimg = mpimg.imread('train/' + font + '.jpg')
util.train(fontimg, font)
testimg = mpimg.imread(final_filename)
self.write(util.test(testimg, font))
def client_update(self, global_model, global_init_model, round_index):
self.elapsed_comm_rounds += 1
print(f'***** Client #{self.client_id} *****', flush=True)
self.model = copy_model(global_model,
self.args.dataset, self.args.arch,
dict(self.model.named_buffers()))
num_pruned, num_params = get_prune_summary(self.model)
cur_prune_rate = num_pruned / num_params
#prune_step = math.floor(num_params * self.args.prune_step)
eval_score = evaluate(self.model,
self.test_loader,
verbose=self.args.test_verbosity)
if eval_score['Accuracy'][
0] > self.args.acc_thresh and cur_prune_rate < self.args.prune_percent:
# I'm adding 0.001 just to ensure we go clear the target prune_percent. This may not be needed
prune_fraction = min(
self.args.prune_step,
0.001 + self.args.prune_percent - cur_prune_rate)
prune_fixed_amount(self.model,
prune_fraction,
verbose=self.args.prune_verbosity,
glob=True)
self.model = copy_model(global_init_model, self.args.dataset,
self.args.arch,
dict(self.model.named_buffers()))
losses = []
accuracies = []
for i in range(self.args.client_epoch):
train_score = train(round_index,
self.client_id,
i,
self.model,
self.train_loader,
lr=self.args.lr,
verbose=self.args.train_verbosity)
losses.append(train_score['Loss'][-1].data.item())
accuracies.append(train_score['Accuracy'][-1])
mask_log_path = f'{self.args.log_folder}/round{round_index}/c{self.client_id}.mask'
client_mask = dict(self.model.named_buffers())
log_obj(mask_log_path, client_mask)
num_pruned, num_params = get_prune_summary(self.model)
cur_prune_rate = num_pruned / num_params
prune_step = math.floor(num_params * self.args.prune_step)
print(
f"num_pruned {num_pruned}, num_params {num_params}, cur_prune_rate {cur_prune_rate}, prune_step: {prune_step}"
)
self.losses[round_index:] = np.array(losses)
self.accuracies[round_index:] = np.array(accuracies)
self.prune_rates[round_index:] = cur_prune_rate
return copy_model(self.model, self.args.dataset, self.args.arch)
def train_model(wrapped_model, model, model_path, train_loader, test_loader, init_lr, epochs, args):
train_loss_f = refinery_loss.RefineryLoss()
val_loss_f = nn.CrossEntropyLoss()
best_model_path = '.'.join(model_path.split('.')[:-1]) + '.best.pth'
# tracking stats
if not hasattr(model, 'stats'):
model.stats = {'train_loss': [], 'test_acc': [], 'test_loss': [],
'weight': [], 'lr': [], 'macs': [], 'efficiency': []}
start_epoch = 1
best_acc = 0
else:
start_epoch = len(model.stats['test_loss'])
best_acc = max(model.stats['test_acc']).item()
curr_weights, _ = util.num_nonzeros(model)
if hasattr(model, 'packed_layer_size'):
macs = np.sum([x*y for x, y in model.packed_layer_size])
else:
macs = curr_weights
# optimizer
optimizer = optim.RMSprop(util.group_weight(model), lr=init_lr, momentum=0.9, alpha=0.9,
weight_decay=0, eps=1.0)
print("Optimizer:")
print(optimizer)
# pruning stage
for epoch in range(start_epoch, epochs + 1):
print('[Epoch {}]'.format(epoch))
for g in optimizer.param_groups:
lr = g['lr']
break
train_loss = util.train(train_loader, wrapped_model, train_loss_f, optimizer, epoch-1, args)
test_loss, test_acc = util.validate(test_loader, model, val_loss_f, epoch-1, args)
print('LR :: {}'.format(lr))
print('Train Loss:: {}'.format(train_loss))
print('Test Loss:: {}'.format(test_loss))
print('Test Acc.:: {}'.format(test_acc))
print('Nonzeros :: {}'.format(curr_weights))
print('')
print('')
model.stats['lr'].append(lr)
model.optimizer = optimizer.state_dict()
model.cpu()
torch.save(model, model_path)
if test_acc > best_acc:
print('New best model found')
torch.save(model, best_model_path)
best_acc = test_acc
model.cuda()
def client_update_method1(client_self, global_model, global_init_model):
print(f'***** Client #{client_self.client_id} *****', flush=True)
# Checking if the client object has been properly initialized
assert isinstance(client_self.model,
nn.Module), "A model must be a PyTorch module"
assert 0 <= client_self.args.prune_percent <= 1, "The prune percentage must be between 0 and 1"
assert client_self.args.client_epoch, '"args" must contain a "client_epoch" field'
assert client_self.test_loader, "test_loader field does not exist. Check if the client is initialized correctly"
assert client_self.train_loader, "train_loader field does not exist. Check if the client is initialized correctly"
assert isinstance(
client_self.train_loader,
torch.utils.data.DataLoader), "train_loader must be a DataLoader type"
assert isinstance(
client_self.test_loader,
torch.utils.data.DataLoader), "test_loader must be a DataLoader type"
client_self.model = copy_model(global_model, client_self.args.dataset,
client_self.args.arch)
num_pruned, num_params = get_prune_summary(client_self.model)
cur_prune_rate = num_pruned / num_params
prune_step = math.floor(num_params * client_self.args.prune_step)
for i in range(client_self.args.client_epoch):
print(f'Epoch {i + 1}')
train(client_self.model,
client_self.train_loader,
lr=client_self.args.lr,
verbose=client_self.args.train_verbosity)
score = evaluate(client_self.model,
client_self.test_loader,
verbose=client_self.args.test_verbosity)
if score['Accuracy'][
0] > client_self.args.acc_thresh and cur_prune_rate < client_self.args.prune_percent:
prune_fixed_amount(client_self.model,
prune_step,
verbose=client_self.args.prune_verbosity)
return copy_model(client_self.model, client_self.args.dataset,
client_self.args.arch)
def pruning_process():
util.topic_log("Before pruning")
top1_acc, top5_acc = util.val_epoch(val_loader, model, args, topk=(1, 5))
util.log(args.log_file_path,
f"before pruning accuracy\t{top1_acc} ({top5_acc})")
util.topic_log("Pruning CNN")
model.prune(args)
util.topic_log("After prune CNN")
util.print_nonzeros(model, args.log_file_path)
top1_acc, top5_acc = util.val_epoch(val_loader, model, args, topk=(1, 5))
util.log(args.log_file_path,
f"after pruning accuracy\t{top1_acc} ({top5_acc})")
util.topic_log("Start retrain after prune CNN")
util.train(model, args, train_loader, val_loader, 'prune_retrain')
util.topic_log("After retraining")
top1_acc, top5_acc = util.val_epoch(val_loader, model, args, topk=(1, 5))
util.log(args.log_file_path,
f"after pruning and retrain accuracy\t{top1_acc} ({top5_acc})")
def main():
cuda = torch.cuda.is_available() and True
embedding_size = 200
convolution_size = 3
LR= 0.01
CNN_size = 667
batch_size = 5
num_epoch = 1 # 10
print("-- cnn model details --")
print("embedding Size:", CNN_size, "convolution size:", convolution_size, "learning rate:", LR, "batch size:", batch_size, "num epochs:", num_epoch)
padding = "<padding>"
train_file = "../data/ask_ubuntu/train_random.txt"
dev_file = "../data/ask_ubuntu/dev.txt"
test_file = "../data/ask_ubuntu/test.txt"
corpus_file = "../data/ask_ubuntu/text_tokenized.txt"
embedding_path = "../data/ask_ubuntu/vectors_pruned.200.txt"
data_loader = util.data_loader(corpus_file, cut_off = 2, padding=padding)
encoder = util.Encoder(data_loader.vocab_map[padding], data_loader, embedding_path, cuda)
print("loaded encoder")
CNN = util.CNN(embedding_size, CNN_size, convolution_size)
if cuda:
encoder = encoder.cuda()
CNN = CNN.cuda()
print("loading annotations...")
dev = data_loader.read_annotations(dev_file, 20, 10)
dev_data = data_loader.create_eval_batches(dev)
test = data_loader.read_annotations(test_file, 20, 10)
test_data = data_loader.create_eval_batches(test)
train_data = data_loader.read_annotations(train_file)
print("loaded annotations")
train_losses, dev_metrics, test_metrics = \
util.train(encoder, CNN, num_epoch, data_loader, train_data, dev_data, test_data, batch_size, util.CNN_forward, True, cuda, LR=LR)
CNN = CNN.cpu()
torch.save(CNN, "cnn.model")
return train_losses, dev_metrics, test_metrics
def main():
cuda = torch.cuda.is_available() and True
num_epoch = 10
batch_size = 2
input_size = 200
output_size = 120
LR = 0.001
dev_file = "../data/ask_ubuntu/dev.txt"
test_file = "../data/ask_ubuntu/test.txt"
train_file = "../data/ask_ubuntu/train_random.txt"
corpus_file = "../data/ask_ubuntu/text_tokenized.txt"
padding = "<padding>"
embedding_path = "../data/ask_ubuntu/vectors_pruned.200.txt"
print("-- lstm model --")
print("embedding size:", output_size, "learning rate:", LR, "batch size:",
batch_size, "num epoch:", num_epoch)
# Represent each question as a word sequence (and not as a bag of words)
data_loader = util.data_loader(corpus_file, cut_off=1, padding=padding)
dev = data_loader.read_annotations(dev_file, 20, 10)
dev_data = data_loader.create_eval_batches(dev)
test = data_loader.read_annotations(test_file, 20, 10)
test_data = data_loader.create_eval_batches(test)
train_data = data_loader.read_annotations(train_file, 10, 2)
# Utilize an exisiting vector representation of the words
encoder = util.Encoder(data_loader.vocab_map[padding], data_loader,
embedding_path, cuda)
print("embeddings done")
model = util.LSTM(input_size, output_size)
if cuda:
model = model.cuda()
encoder = encoder.cuda()
train_losses, dev_metrics, test_metrics \
= util.train(encoder, model, num_epoch, data_loader, train_data, dev_data, test_data, batch_size, util.LSTM_forward, True, cuda, LR)
model = model.cpu()
torch.save(model, "lstm.model")
return train_losses, dev_metrics, test_metrics
def runGym():
env_id = "CartPole-v1"
config = Config(env_id, env_type="gym")
config.update_every = 256 * 4
config.num_learn = 20
config.win_condition = 200
config.n_steps = 2.5e5
config.hidden_size = 256
config.lr = 0.002
config.lr_annealing = True
config.epsilon_annealing = True
config.memory = Memory
config.model = ActorCritic
config.init_wandb(project="gym", entity="procgen")
scores, average_scores = train(config, config.env)
def runAtari():
env_id = "BreakoutNoFrameskip-v4"
config = Config(env_id, env_type="atari")
config.update_every = 128
config.num_learn = 4
config.win_condition = 230
config.n_steps = 7e6
config.hidden_size = 512
config.lr = 2.5e-4
config.lr_annealing = True
config.epsilon_annealing = True
config.memory = Memory
config.model = ActorCriticCnn
config.init_wandb()
scores, average_scores = train(config, config.env)
def runProcGen():
env_id = "coinrun"
config = Config(env_id, env_type="procgen", num_envs=64)
config.update_every = 256
config.num_learn = 3
config.win_condition = 230
config.n_steps = 1e8
config.hidden_size = 512
config.lr = 2.5e-4
config.lr_annealing = True
config.epsilon_annealing = True
config.memory = Memory
config.model = ActorCriticCnnProcGen
config.init_wandb()
scores, average_scores = train(config, config.env)
"--cmd",
type=str,
default='train',
help="command")
flags = parse.parse_args()
settings = {}
settings['data'] = flags.data
settings['net'] = flags.net
settings['loss'] = flags.loss
settings['step'] = flags.step
settings['dump'] = flags.dump + os.sep + flags.net
settings['batch_size'] = flags.batch_size
settings['test_batch_size'] = flags.test_batch_size
settings['width'] = flags.img_width
settings['height'] = flags.img_height
settings['channel'] = flags.img_channel
settings['pts_num'] = flags.points_num
settings['grid_num'] = flags.grid_num
settings['grid_dim'] = flags.grid_dim
settings['epoch_num'] = flags.epoch_num
if flags.gpu:
os.environ['CUDA_VISIBLE_DEVICES'] = flags.gpu
try:
if flags.cmd == 'train':
util.train(settings)
elif flags.cmd == 'test':
util.test(settings)
except Exception, e:
print(e)
else:
print('Done Runing')
# MODEL_CONFIG
INPUT_SIZE = 40
HIDDEN_SIZE = 160
OUTPUT_SIZE = 4
model = SIMPLE_RNN(INPUT_SIZE, HIDDEN_SIZE, OUTPUT_SIZE)
model.to(device)
# TRAINING_CONFIG
CRITERION = torch.nn.MSELoss()
LR = 1e-5
EPCH = 20
optim = optim.Adam(model.parameters(), lr=LR)
EXPORT_PATH = 'models/saved_weights/simple_rnn_v2.pth'
print("==============================")
print("Starting training...")
loss_hist = []
val_loss_hist = []
for i in tqdm(range(EPCH)):
avg_loss = train(model, optim, CRITERION, deap_train_loader, device)
loss_hist.append(avg_loss)
val_loss = eval(model, CRITERION, deap_test_loader, device, eval_size=99999)
export_or_not(val_loss, val_loss_hist, model, EXPORT_PATH)
val_loss_hist.append(val_loss)
if i % 1 == 0:
plt.plot(loss_hist, label="Training loss")
plt.plot(val_loss_hist, label="Validation loss")
plt.legend()
plt.savefig("loss.png")
plt.show()
def initUI(self):
"""
Main Window global parameters
"""
self.imgOriginal = np.array([])
self.mainWidth = 1280
self.mainHeight = 640
self.main = QLabel()
self.imgNpBefore = np.array([])
self.imgNpAfter = np.array([])
self.skin = mpimg.imread('res/skin.jpg')
grid = QGridLayout()
self.main.setLayout(grid)
self.mainBefore = QLabel('Before')
self.mainBefore.setAlignment(Qt.AlignHCenter | Qt.AlignVCenter)
self.mainBefore.setWordWrap(True)
self.mainBefore.setFont(QFont('Monospace', 10))
self.mainAfter = QLabel('After')
self.mainAfter.setAlignment(Qt.AlignHCenter | Qt.AlignVCenter)
self.mainAfter.setWordWrap(True)
self.mainAfter.setFont(QFont('Monospace', 10))
grid.addWidget(self.mainBefore, 0, 0)
grid.addWidget(self.mainAfter, 0, 1)
"""
Menu Bar
"""
# FILE MENU
openFile = QAction('Open', self)
openFile.setShortcut('Ctrl+O')
openFile.setStatusTip('Open new File')
openFile.triggered.connect(self.showDialog)
exitAction = QAction('Exit', self)
exitAction.setShortcut('Ctrl+Q')
exitAction.setStatusTip('Exit application')
exitAction.triggered.connect(self.close)
moveLeft = QAction('Move Left', self)
moveLeft.setShortcut('Ctrl+L')
moveLeft.triggered.connect(lambda: self.updateBefore(self.imgNpAfter))
# PROCESS MENU
equalizeMenu = QAction('Equalize', self)
equalizeMenu.triggered.connect(lambda: self.updateImgAfter(util.equalize(self.imgNpBefore)))
histogramMenu = QAction('Histogram', self)
#histogramMenu.triggered.connect(lambda: self.updateImgAfter(
grayscaleMenu = QAction('Grayscale', self)
grayscaleMenu.triggered.connect(lambda: self.updateImgAfter(util.getgrayscale(self.imgNpBefore)))
binarizeMenu = QAction('Binarize', self)
binarizeMenu.triggered.connect(lambda: self.updateImgAfter(util.otsu(self.imgNpBefore)))
gaussianMenu = QAction('Smooth', self)
gaussianMenu.triggered.connect(lambda: self.updateImgAfter(util.convolvefft(util.gaussian_filt(), util.getgrayscale(self.imgNpBefore))))
resizeMenu = QAction('Resize', self)
resizeMenu.triggered.connect(lambda: self.updateImgAfter(util.downsample(self.imgNpBefore)))
segmentMenu = QAction('Segment', self)
segmentMenu.triggered.connect(lambda: self.updateImgAfter(util.showobj(util.downsample(self.imgNpBefore, target_height=480), util.segment(util.thin(util.otsu(util.downsample(self.imgNpBefore, target_height=480), bg='light'))), box=False)))
# EDGE DETECTION MENU
averageMenu = QAction('Average', self)
averageMenu.triggered.connect(lambda: self.updateImgAfter(util.degreezero(self.imgNpBefore, type="average")))
differenceMenu = QAction('Difference', self)
differenceMenu.triggered.connect(lambda: self.updateImgAfter(util.degreezero(self.imgNpBefore, type="difference")))
homogenMenu = QAction('Homogen', self)
homogenMenu.triggered.connect(lambda: self.updateImgAfter(util.degreezero(self.imgNpBefore, type="homogen")))
sobelMenu = QAction('Sobel', self)
sobelMenu.triggered.connect(lambda: self.updateImgAfter(util.degreeone(self.imgNpBefore, type="sobel")))
prewittMenu = QAction('Prewitt', self)
prewittMenu.triggered.connect(lambda: self.updateImgAfter(util.degreeone(self.imgNpBefore, type="prewitt")))
freichenMenu = QAction('Frei-Chen', self)
freichenMenu.triggered.connect(lambda: self.updateImgAfter(util.degreeone(self.imgNpBefore, type="freichen")))
kirschMenu = QAction('Kirsch', self)
kirschMenu.triggered.connect(lambda: self.updateImgAfter(util.degreetwo(self.imgNpBefore, type="kirsch")))
# FEATURE MENU
chaincodeMenu = QAction('Chain code', self)
chaincodeMenu.triggered.connect(lambda: self.updateTxtAfter(str([util.getdirection(chain[n][0], chain[n][1]) for chain in util.segment(util.thin(self.imgNpBefore), cc=True) for n in xrange(len(chain))])))
turncodeMenu = QAction('Turn code', self)
turncodeMenu.triggered.connect(lambda: self.updateTxtAfter(str([util.getturncode(cc) for cc in util.segment(util.thin(self.imgNpBefore, bg='light'), cc=False)])))
skeletonMenu = QAction('Zhang-Suen thinning', self)
skeletonMenu.triggered.connect(lambda:self.updateImgAfter(util.zhangsuen(util.binarize(self.imgNpBefore, bg='light'))))
skinMenu = QAction('Boundary detection', self)
skinMenu.triggered.connect(lambda:self.updateImgAfter(util.thin(self.imgNpBefore, bg='light')))
freemanMenu = QAction('Contour profile', self)
# RECOGNITION MENU
freemantrainfontMenu = QAction('Train Contour Font', self)
freemantrainfontMenu.triggered.connect(lambda: util.train(self.imgNpBefore, feats='zs', order='font', setname='font'))
freemantrainplatMenu = QAction('Train ZS Plate (GNB)', self)
freemantrainplatMenu.triggered.connect(lambda: util.train(self.imgNpBefore, feats='zs', order='plat', setname='plat'))
cctctrainfontMenu = QAction('Train CC + TC Font', self)
cctctrainplatMenu = QAction('Train CC + TC Plate', self)
freemantestfontMenu = QAction('Predict Contour Font', self)
freemantestfontMenu.triggered.connect(lambda: self.updateTxtAfter(util.test(self.imgNpBefore, feats='zs', order='font', setname='font')))
freemantestplatMenu = QAction('Predict Contour Plate', self)
freemantestplatMenu.triggered.connect(lambda:self.updateTxtAfter(util.test(self.imgNpBefore, feats='zs', order='plat', setname='plat')))
cctctestfontMenu = QAction('Predict CC + TC Font', self)
cctctestplatMenu = QAction('Predict CC + TC Plate', self)
facesMenu = QAction('Show faces', self)
facesMenu.triggered.connect(lambda: self.updateImgAfter(util.getFaces(self.imgNpBefore, self.skin, range=70)))
faceMenu = QAction('Show facial features', self)
faceMenu.triggered.connect(lambda: self.updateImgAfter(util.showobj(self.imgNpBefore, util.getFaceFeats(self.imgNpBefore, self.skin, range=100), color=False)))
# MENU BAR
menubar = self.menuBar()
fileMenu = menubar.addMenu('&File')
fileMenu.addAction(openFile)
fileMenu.addAction(exitAction)
fileMenu.addAction(moveLeft)
processMenu = menubar.addMenu('&Preprocess')
#processMenu.addAction(histogramMenu)
processMenu.addAction(equalizeMenu)
processMenu.addAction(grayscaleMenu)
processMenu.addAction(binarizeMenu)
processMenu.addAction(gaussianMenu)
processMenu.addAction(resizeMenu)
processMenu.addAction(segmentMenu)
edgeMenu = menubar.addMenu('&Edge Detection')
edgeMenu.addAction(averageMenu)
edgeMenu.addAction(differenceMenu)
edgeMenu.addAction(homogenMenu)
edgeMenu.addAction(sobelMenu)
edgeMenu.addAction(prewittMenu)
edgeMenu.addAction(freichenMenu)
edgeMenu.addAction(kirschMenu)
featureMenu = menubar.addMenu('&Features')
featureMenu.addAction(chaincodeMenu)
featureMenu.addAction(turncodeMenu)
featureMenu.addAction(skeletonMenu)
featureMenu.addAction(skinMenu)
featureMenu.addAction(freemanMenu)
recogMenu = menubar.addMenu('&Recognition')
recogMenu.addAction(freemantrainfontMenu)
recogMenu.addAction(freemantrainplatMenu)
recogMenu.addAction(cctctrainfontMenu)
recogMenu.addAction(cctctrainplatMenu)
recogMenu.addAction(freemantestfontMenu)
recogMenu.addAction(freemantestplatMenu)
recogMenu.addAction(cctctestfontMenu)
recogMenu.addAction(cctctestplatMenu)
recogMenu.addAction(facesMenu)
recogMenu.addAction(faceMenu)
#recogMenu.addAction(
"""
Toolbar, Status Bar, Tooltip
"""
self.statusBar().showMessage('Ready')
QToolTip.setFont(QFont('SansSerif', 10))
#self.setToolTip('This is a <b>QWidget</b> widget')
"""
Displaying
"""
self.setGeometry(12, 30, self.mainWidth, self.mainHeight+80)
self.setWindowTitle('Pyxel')
self.setWindowIcon(QIcon('res/web.png'))
self.setCentralWidget(self.main)
self.main.setGeometry(QRect(0, 80, self.mainWidth, self.mainHeight))
#self.mainAfter.setGeometry(QRect(self.mainWidth/2, 80, self.mainWidth/2, self.mainHeight))
self.show()
activation="softmax",
name="combined_board")(dense_out_2)
if net_type == 'from':
model = Model(board_input, output_from)
elif net_type == 'to':
model = Model([board_input, from_input], output_to)
elif net_type == 'full':
model = Model(board_input, output_combined)
else:
model = Model(board_input, [output_from, output_to])
model.compile('adamax',
'categorical_crossentropy',
metrics=['accuracy', 'top_k_categorical_accuracy'])
return model
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument("network_type", help="Either from, to, or both")
args = parser.parse_args()
net_type = args.network_type
assert (net_type == 'from' or net_type == 'to' or net_type == 'both'
or net_type == 'full')
util.train(net_type,
'state_action_sl',
'data/large-ccrl_',
build_network,
featurized=True)
def mlpTest():
import util
model = createTestMlp()
itr = util.trainingIterator(model=model,patience=10000)
util.train(itr,model)
def run(stem_fn,
block_fn,
classifier_fn,
voting_strategy_fn,
boosting_strategy_fn,
training_style,
epochs,
batch_size,
block_num,
dataset_name,
classes,
metrics_options,
log_dir,
load_stem=None,
patience=12,
progressive_training_epochs=5):
"""
Args:
load_stem (str): path to the weights file for the stem
"""
metrics.setup_log_files(log_dir, block_num, metrics_options)
# load data
train_gen, validate_gen, train_data_shape, validate_data_shape, label_shape, class_num = data.load_data(
dataset_name, batch_size, classes)
data_ph, label_ph, _, weak_logits, classifier, classification_metrics = boosted_classifier.build_model(
stem_fn,
block_fn,
classifier_fn,
block_num,
voting_strategy_fn,
batch_size,
class_num,
train_data_shape,
label_shape,
load_stem=load_stem)
stem_saver = tf.train.Saver(
tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='stem'))
weighted_losses = boosting_strategy.calculate_boosted_losses(
boosting_strategy_fn, weak_logits, label_ph, batch_size, class_num)
weights_scale_ph = tf.placeholder_with_default(
tf.ones([block_num]), [block_num])
def feed_dict_fn(epoch):
data, labels = next(train_gen)
feed_dict = {data_ph: data, label_ph: labels}
if training_style == 'progressive':
val = np.zeros([block_num], dtype=np.float32)
val[epoch // 2] = 1.
val[(epoch // 2) - 1] = 0.
feed_dict[weights_scale_ph] = val
return feed_dict
def validate_feed_dict_fn():
data, labels = next(validate_gen)
feed_dict = {data_ph: data, label_ph: labels}
return feed_dict
# calculate gradients
optimizer = tf.train.AdamOptimizer()
final_grads_and_vars, grad_metrics = boosting_strategy.calculate_boosted_gradients(
optimizer, weighted_losses, weights_scale_ph)
train_op = optimizer.apply_gradients(final_grads_and_vars)
# if the voting strategy has an update fn, use it
# I, for one, welcome our new duck typing overlords
if hasattr(classifier.voting_strategy, 'update'):
voting_strategy_update_op = classifier.voting_strategy.update(
weak_logits, label_ph)
train_op = tf.group(train_op, voting_strategy_update_op)
print("Trainable Parameters: {}".format(
np.sum([
np.prod(v.get_shape().as_list()) for v in tf.trainable_variables()
])))
verbose_ops_dict = classification_metrics
if 'gradient_norms' in metrics_options:
verbose_ops_dict.update(grad_metrics)
# initialize session and train
process_metrics_fn = functools.partial(
metrics.process_metrics, log_dir=log_dir, options=metrics_options)
early_stopping_fn = util.build_early_stopping_fn(patience=patience)
full_metrics = util.train(
train_op,
epochs,
train_steps_per_epoch=train_data_shape[0] // batch_size,
validate_steps_per_epoch=validate_data_shape[0] // batch_size,
verbose_ops_dict=verbose_ops_dict,
train_feed_dict_fn=feed_dict_fn,
validate_feed_dict_fn=validate_feed_dict_fn,
process_metrics_fn=process_metrics_fn,
early_stopping_fn=early_stopping_fn,
stem_saver=stem_saver,
stem=load_stem)
return full_metrics
model = BiLSTM(vocab_size=len(word_to_ix),
tag_size=len(tag_to_ix),
embed_dim=128,
hid_dim=128,
num_layers=2,
loss_fn=loss_fn,
use_dropout=True,
dropout=0.1,
pre_word_embeds=pretrianed_embedding,
use_subembed=True,
subembed_dim=64,
subhid_dim=64,
use_init=True,
init_fns=init_fns)
#@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
optimizer = optim.Adam(model.parameters(), lr=lr, weight_decay=1e-6)
#@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
for epoch in range(epoch_num):
model.train()
model, avg_loss = util.train(X, y, trn_ixs, model, optimizer)
model.eval()
vld_loss, y_preds, y_trues = util.evaluate(X, y, vld_ixs, model)
print('Epoch {}/{} \t avg_loss {:.4f} \t vld_loss {:.4f} \t'.format(
epoch + 1, epoch_num, avg_loss, vld_loss))
util.eval_info(y_preds, y_trues, ix_to_tag)
embedding_size = 10
cin_layer_nums = [40, 30, 20]
dnn_hidden = [200, 200]
device = torch.device('cuda:0')
#device = torch.device('cpu')
model = xDeepFM(dataset.feature_size, dataset.field_size, embedding_size,
cin_layer_nums, dnn_hidden).to(device)
lr = 0.001
weight_l2 = 1e-4
epoch = 10
st = time.time()
criterion = torch.nn.BCELoss()
optimizer = torch.optim.Adam(params=model.parameters(),
lr=lr,
weight_decay=weight_l2)
for epoch_i in range(epoch):
util.train(model,
optimizer,
train_data_loader,
criterion,
device,
verbose=200)
auc = util.test(model, valid_data_loader, device)
print('epoch:', epoch_i, 'validation: auc:', auc)
print('cost total time: %d' % (time.time() - st))
auc = util.test(model, test_data_loader, device)
print('test auc:', auc)
def train(model, train_loader, val_loader, args):
criterion = nn.CrossEntropyLoss().cuda()
optimizer = torch.optim.SGD(model.parameters(),
args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, args.epochs)
prune_epoch = 0
max_prune_rate = 0.85
max_prune_rate = 0.8
final_prune_epoch = int(0.5 * args.epochs)
num_prune_epochs = 10
prune_rates = [
max_prune_rate * (1 - (1 - (i / num_prune_epochs))**3)
for i in range(num_prune_epochs)
]
prune_rates[-1] = max_prune_rate
prune_epochs = np.linspace(0, final_prune_epoch,
num_prune_epochs).astype('i').tolist()
print("Pruning Epochs: {}".format(prune_epochs))
print("Pruning Rates: {}".format(prune_rates))
curr_weights, num_weights = util.num_nonzeros(model)
macs = curr_weights
model.stats = {
'train_loss': [],
'test_acc': [],
'test_loss': [],
'weight': [],
'lr': [],
'macs': [],
'efficiency': []
}
best_path = args.save_path.split('.pth')[0] + '.best.pth'
best_test_acc = 0
for epoch in range(1, args.epochs + 1):
scheduler.step()
for g in optimizer.param_groups:
lr = g['lr']
break
# prune smallest weights up to a set prune_rate
if epoch in prune_epochs:
util.prune(model, prune_rates[prune_epoch])
curr_weights, num_weights = util.num_nonzeros(model)
packing.pack_model(model, args.gamma)
macs = np.sum([x * y for x, y in model.packed_layer_size])
curr_weights, num_weights = util.num_nonzeros(model)
prune_epoch += 1
if epoch == prune_epochs[-1]:
# disable l1 penalty, as target sparsity is reached
args.l1_penalty = 0
print(' :: [{}]\tLR {:.4f}\tNonzeros ({}/{})'.format(
epoch, lr, curr_weights, num_weights))
train_loss = util.train(train_loader, model, criterion, optimizer,
epoch, args)
test_loss, test_acc = util.validate(val_loader, model, criterion,
epoch, args)
is_best = test_acc > best_test_acc
best_test_acc = max(test_acc, best_test_acc)
model.stats['lr'].append(lr)
model.stats['macs'].append(macs)
model.stats['weight'].append(curr_weights)
model.stats['efficiency'].append(100.0 * (curr_weights / macs))
model.optimizer = optimizer.state_dict()
model.epoch = epoch
model.cpu()
torch.save(model, args.save_path)
if is_best:
torch.save(model, best_path)
model.cuda()
def main():
global args, best_er1
args = parser.parse_args()
# Check if CUDA is enabled
args.cuda = not args.no_cuda and torch.cuda.is_available()
for tgt_idx, tgt in enumerate(dataset_targets[args.dataset]):
print("Training a model for {}".format(tgt))
# Load data
root = args.dataset_path if args.dataset_path else dataset_paths[
args.dataset]
task_type = args.dataset_type if args.dataset_type else dataset_types[
args.dataset]
if args.resume:
resume_dir = args.resume.format(dataset=args.dataset,
model=args.model,
layers=args.layers,
feature=tgt)
#end if
Model_Class = model_dict[args.model]
print("Preparing dataset")
node_features, edge_features, target_features, task_type, train_loader, valid_loader, test_loader = read_dataset(
args.dataset, root, args.batch_size, args.prefetch)
# Define model and optimizer
print('\tCreate model')
hidden_state_size = args.hidden
model = Model_Class(node_features=node_features,
edge_features=edge_features,
target_features=1,
hidden_features=hidden_state_size,
num_layers=args.layers,
dropout=0.5,
type=task_type,
s2s_processing_steps=args.s2s)
print("#Parameters: {param_count}".format(
param_count=count_params(model)))
print('Optimizer')
optimizer = optim.Adam(model.parameters(),
lr=args.lr,
weight_decay=args.weight_decay)
criterion, evaluation, metric_name, metric_compare, metric_best = get_metric_by_task_type(
task_type, target_features)
print('Logger')
logger = Logger(
args.log_path.format(dataset=args.dataset,
model=args.model,
layers=args.layers,
feature=tgt))
lr_step = (args.lr - args.lr * args.lr_decay) / (
args.epochs * args.schedule[1] - args.epochs * args.schedule[0])
# get the best checkpoint if available without training
if args.resume:
checkpoint_dir = resume_dir
best_model_file = os.path.join(checkpoint_dir, 'model_best.pth')
if not os.path.isdir(checkpoint_dir):
os.makedirs(checkpoint_dir)
if os.path.isfile(best_model_file):
print("=> loading best model '{}'".format(best_model_file))
checkpoint = torch.load(best_model_file)
args.start_epoch = checkpoint['epoch']
best_acc1 = checkpoint['best_er1']
model.load_state_dict(checkpoint['state_dict'])
if args.cuda:
model.cuda()
optimizer.load_state_dict(checkpoint['optimizer'])
print("=> loaded best model '{}' (epoch {})".format(
best_model_file, checkpoint['epoch']))
else:
print("=> no best model found at '{}'".format(best_model_file))
print('Check cuda')
if args.cuda:
print('\t* Cuda')
model = model.cuda()
criterion = criterion.cuda()
# Epoch for loop
for epoch in range(0, args.epochs):
try:
if epoch > args.epochs * args.schedule[
0] and epoch < args.epochs * args.schedule[1]:
args.lr -= lr_step
for param_group in optimizer.param_groups:
param_group['lr'] = args.lr
#end if
# train for one epoch
train(train_loader,
model,
criterion,
optimizer,
epoch,
evaluation,
logger,
target_range=(tgt_idx, ),
tgt_name=tgt,
metric_name=metric_name,
cuda=args.cuda,
log_interval=args.log_interval)
# evaluate on test set
er1 = validate(valid_loader,
model,
criterion,
evaluation,
logger,
target_range=(tgt_idx, ),
tgt_name=tgt,
metric_name=metric_name,
cuda=args.cuda,
log_interval=args.log_interval)
is_best = metric_compare(er1, best_er1)
best_er1 = metric_best(er1, best_er1)
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'best_er1': best_er1,
'optimizer': optimizer.state_dict(),
},
is_best=is_best,
directory=resume_dir)
# Logger step
logger.log_value('learning_rate', args.lr).step()
except KeyboardInterrupt:
break
#end try
#end for
# get the best checkpoint and test it with test set
if args.resume:
checkpoint_dir = resume_dir
best_model_file = os.path.join(checkpoint_dir, 'model_best.pth')
if not os.path.isdir(checkpoint_dir):
os.makedirs(checkpoint_dir)
if os.path.isfile(best_model_file):
print("=> loading best model '{}'".format(best_model_file))
checkpoint = torch.load(best_model_file)
args.start_epoch = checkpoint['epoch']
best_acc1 = checkpoint['best_er1']
model.load_state_dict(checkpoint['state_dict'])
if args.cuda:
model.cuda()
optimizer.load_state_dict(checkpoint['optimizer'])
print("=> loaded best model '{}' (epoch {})".format(
best_model_file, checkpoint['epoch']))
else:
print("=> no best model found at '{}'".format(best_model_file))
#end if
#end if
# (For testing)
validate(test_loader,
model,
criterion,
evaluation,
target_range=(tgt_idx, ),
tgt_name=tgt,
metric_name=metric_name,
cuda=args.cuda,
log_interval=args.log_interval)
# data
train_data, val_data = load_data(data_config, exp_config['batch_size'])
eval_length = data_config['eval_length']
# logger
# model
model = Model(**model_config).to(0)
# optimizer
optimizer = AdamOptimizer(params=model.parameters(), lr=exp_config['lr'],
grad_clip_value=exp_config['grad_clip_value'],
grad_clip_norm=exp_config['grad_clip_norm'])
logger_on = True
if logger_on:
logger = Logger(exp_config, model_config, data_config)
# train / val loop
for epoch in range(exp_config['n_epochs']):
print('Epoch:', epoch)
if logger_on:
logger.log(train(train_data, model, optimizer, eval_length), 'train')
logger.log(validation(val_data, model, eval_length), 'val')
logger.save(model)
else:
train(train_data, model, optimizer, eval_length)
validation(val_data, model, eval_length)
def train_model(wrapped_model, model, train_loss_f, model_path, train_loader,
test_loader, init_lr, epochs, args):
val_loss_f = nn.CrossEntropyLoss()
best_model_path = '.'.join(model_path.split('.')[:-1]) + '.best.pth'
# tracking stats
if not hasattr(model, 'stats'):
model.stats = {
'train_loss': [],
'test_acc': [],
'test_loss': [],
'weight': [],
'lr': [],
'macs': [],
'efficiency': []
}
start_epoch = 1
else:
start_epoch = len(model.stats['test_loss'])
curr_weights, _ = util.num_nonzeros(model)
if hasattr(model, 'packed_layer_size'):
macs = np.sum([x * y for x, y in model.packed_layer_size])
else:
macs = curr_weights
# optimizer
optimizer = optim.RMSprop(util.group_weight(model),
lr=init_lr,
momentum=0.9,
alpha=0.9,
weight_decay=4e-5,
eps=1.0)
print("Optimizer:")
print(optimizer)
best_acc = 0
prune_epoch = 0
max_prune_rate = 0.8
final_prune_epoch = int(0.9 * args.epochs)
num_prune_epochs = 10
prune_rates = [
max_prune_rate * (1 - (1 - (i / num_prune_epochs))**3)
for i in range(num_prune_epochs)
]
prune_rates[-1] = max_prune_rate
prune_epochs = np.linspace(0, final_prune_epoch,
num_prune_epochs).astype('i').tolist()
prune_rate = 0.1
prune_total = 0.0
prune_cycle = 8
max_prune = 0.7
# pruning stage
for epoch in range(start_epoch, epochs + 1):
print('[Epoch {}]'.format(epoch))
for g in optimizer.param_groups:
lr = g['lr']
break
if epoch % prune_cycle == 0 and prune_total < max_prune:
prune_total += prune_rate
print('Prune Total: {:2.2f}'.format(100. * prune_total))
util.prune(model, prune_total)
packing.pack_model(model, args.gamma)
macs = np.sum([x * y for x, y in model.packed_layer_size])
curr_weights, num_weights = util.num_nonzeros(model)
train_loss = util.train(train_loader, wrapped_model, train_loss_f,
optimizer, epoch - 1, args)
test_loss, test_acc = util.validate(test_loader, model, val_loss_f,
epoch - 1, args)
print('LR :: {}'.format(lr))
print('Train Loss:: {}'.format(train_loss))
print('Test Loss:: {}'.format(test_loss))
print('Test Acc.:: {}'.format(test_acc))
print('Nonzeros :: {}'.format(curr_weights))
print('')
print('')
model.stats['lr'].append(lr)
model.optimizer = optimizer.state_dict()
model.cpu()
torch.save(model, model_path)
if test_acc > best_acc and prune_total >= max_prune:
print('New best model found')
torch.save(model, best_model_path)
best_acc = test_acc
model.cuda()
def train_vae(args, trial=None, test_ratio=1):
"""
"""
model = VAE(args["num_layers"], args["z_dim"]).to(device)
if args["model_path"] != "":
model.load_state_dict(torch.load(args["model_path"]))
optimizer = optim.Adam(model.parameters(), lr=2e-3) # 1e-3
train_loader, test_loader = create_loader()
train_num = int(args["train_ratio"] * len(train_loader)) # 1
test_num = int(test_ratio * len(test_loader))
train_loader = train_loader[:train_num]
test_loader = test_loader[:test_num]
loss_list = []
test_loss_list = []
f0_loss_list = []
f0_loss_trainlist = []
start = time.time()
for epoch in range(1, args["num_epoch"] + 1):
loss, f0_loss_train = train(epoch,
model,
train_loader,
vae_loss,
optimizer,
f0=True)
test_loss, f0_loss = test(epoch, model, test_loader, vae_loss)
print("epoch [{}/{}], loss: {:.4f} test_loss: {:.4f}".format(
epoch + 1, args["num_epoch"], loss, test_loss))
# logging
loss_list.append(loss)
f0_loss_trainlist.append(f0_loss_train)
test_loss_list.append(test_loss)
f0_loss_list.append(f0_loss)
if trial is not None:
trial.report(test_loss, epoch - 1)
if trial is not None:
if trial.should_prune():
return optuna.TrialPruned()
print(time.time() - start)
if epoch % 5 == 0:
torch.save(
model.state_dict(),
args["output_dir"] + "/vae_model_{}.pth".format(epoch),
)
np.save(args["output_dir"] + "/loss_list.npy", np.array(loss_list))
np.save(args["output_dir"] + "/f0loss_list.npy",
np.array(f0_loss_trainlist))
np.save(args["output_dir"] + "/test_loss_list.npy",
np.array(test_loss_list))
np.save(args["output_dir"] + "/test_f0loss_list.npy",
np.array(f0_loss_list))
return f0_loss
def main(args):
network = importlib.import_module(args.model_def)
subdir = datetime.strftime(datetime.now(), '%Y%m%d-%H%M%S')
# 设置log日志
log_dir = os.path.join(os.path.expanduser(args.logs_base_dir), subdir)
if not os.path.isdir(
log_dir): # Create the log directory if it doesn't exist
os.makedirs(log_dir)
model_dir = os.path.join(os.path.expanduser(args.models_base_dir), subdir)
if not os.path.isdir(
model_dir): # Create the model directory if it doesn't exist
os.makedirs(model_dir)
# Store some git revision info in a text file in the log directory
src_path, _ = os.path.split(os.path.realpath(__file__))
# facenet.store_revision_info(src_path, log_dir, ' '.join(sys.argv))
np.random.seed(seed=args.seed)
random.seed(args.seed)
# 获取数据集,通过get_dataset获取的train_set是包含文件路径与标签的集合
train_set = deal_data.get_dataset(args.data_dir)
test_set = deal_data.get_dataset(args.test_dir)
nrof_classes = len(train_set)
print('Model directory: %s' % model_dir)
print('Log directory: %s' % log_dir)
with tf.Graph().as_default():
tf.set_random_seed(args.seed)
global_step = tf.Variable(0, trainable=False)
# Get a list of image paths and their labels
# 获取图片地址和标签
image_list, label_list = deal_data.get_image_paths_and_labels(
train_set)
test_image_list, test_label_list = deal_data.get_image_paths_and_labels(
test_set)
assert len(image_list) > 0, 'The dataset should not be empty'
# Create a queue that produces indices into the image_list and label_list
labels = ops.convert_to_tensor(label_list, dtype=tf.int32)
range_size = array_ops.shape(labels)[0]
index_queue = tf.train.range_input_producer(range_size,
num_epochs=None,
shuffle=True,
seed=None,
capacity=32)
index_dequeue_op = index_queue.dequeue_many(
args.batch_size * args.epoch_size, 'index_dequeue')
# 学习率
learning_rate_placeholder = tf.placeholder(tf.float32,
name='learning_rate')
# 批大小
batch_size_placeholder = tf.placeholder(tf.int32, name='batch_size')
# 用于判断是训练还是测试
phase_train_placeholder = tf.placeholder(tf.bool, name='phase_train')
# 图像路径
image_paths_placeholder = tf.placeholder(tf.string,
shape=(None, 1),
name='image_paths')
# 图像标签
labels_placeholder = tf.placeholder(tf.int64,
shape=(None, 1),
name='labels')
# labels_placeholder = tf.reshape(labels_placeholder, shape=(-1, 1))
# 新建一个队列,数据流操作,fifo,先入先出
input_queue = data_flow_ops.FIFOQueue(capacity=100000,
dtypes=[tf.string, tf.int64],
shapes=[(1, ), (1, )],
shared_name=None,
name=None)
# enqueue_many返回的是一个操作
enqueue_op = input_queue.enqueue_many(
[image_paths_placeholder, labels_placeholder], name='enqueue_op')
nrof_preprocess_threads = 4
images_and_labels = []
for _ in range(nrof_preprocess_threads):
filenames, label = input_queue.dequeue()
images = []
for filename in tf.unstack(filenames):
file_contents = tf.read_file(filename)
# 读图像
image = tf.image.decode_image(file_contents, channels=3)
if args.random_rotate:
image = tf.py_func(random_rotate_image, [image], tf.uint8)
if args.random_crop:
image = tf.random_crop(
image, [args.image_size, args.image_size, 3])
else:
image = tf.image.resize_image_with_crop_or_pad(
image, args.image_size, args.image_size)
if args.random_flip:
image = tf.image.random_flip_left_right(image)
# pylint: disable=no-member
image.set_shape((args.image_size, args.image_size, 3))
images.append(tf.image.per_image_standardization(image))
# 读取后的图片和标签list
images_and_labels.append([images, label])
image_batch, label_batch = tf.train.batch_join(
images_and_labels,
batch_size=batch_size_placeholder,
shapes=[(args.image_size, args.image_size, 3), ()],
enqueue_many=True,
capacity=4 * nrof_preprocess_threads * args.batch_size,
allow_smaller_final_batch=True)
image_batch = tf.identity(image_batch, 'image_batch')
image_batch = tf.identity(image_batch, 'input')
label_batch = tf.identity(label_batch, 'label_batch')
print('Total number of classes: %d' % nrof_classes)
print('Total number of examples: %d' % len(image_list))
print('Building training graph')
# Build the inference graph
# 创建网络图:除了全连接层和损失层
prelogits, _ = network.inference(
image_batch,
args.keep_probability,
phase_train=phase_train_placeholder,
bottleneck_layer_size=args.embedding_size,
weight_decay=args.weight_decay)
# 全联接
logits = slim.fully_connected(
prelogits,
4,
activation_fn=None,
weights_initializer=tf.truncated_normal_initializer(stddev=0.1),
weights_regularizer=slim.l2_regularizer(args.weight_decay),
scope='Logits',
reuse=False)
predicts_soft = tf.nn.softmax(logits=logits, dim=-1)
# 对维度dim进行L2范式标准化 output = x / sqrt(max(sum(x**2), epsilon))
embeddings = tf.nn.l2_normalize(prelogits, 1, 1e-10, name='embeddings')
# train_prediction = tf.nn.softmax(prelogits)
# Add center loss
# if args.center_loss_factor > 0.0:
# prelogits_center_loss, _ = util.center_loss(prelogits, label_batch, args.center_loss_alfa, nrof_classes)
# tf.add_to_collection(tf.GraphKeys.REGULARIZATION_LOSSES, prelogits_center_loss * args.center_loss_factor)
# 将指数衰减应用到学习率上
learning_rate = tf.train.exponential_decay(
learning_rate_placeholder,
global_step,
args.learning_rate_decay_epochs * args.epoch_size,
args.learning_rate_decay_factor,
staircase=True)
tf.summary.scalar('learning_rate', learning_rate)
# Calculate the average cross entropy loss across the batch
cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(
labels=label_batch,
logits=logits,
name='cross_entropy_per_example')
cross_entropy_mean = tf.reduce_mean(cross_entropy,
name='cross_entropy')
tf.add_to_collection('losses', cross_entropy_mean)
# Calculate the total losses
# 根据REGULARIZATION_LOSSES返回一个收集器中所收集的值的列表
regularization_losses = tf.get_collection(
tf.GraphKeys.REGULARIZATION_LOSSES)
# 我们选择L2-正则化来实现这一点，L2正则化将网络中所有权重的平方和加到损失函数。如果模型使用大权重，则对应重罚分，并且如果模型使用小权重，则小罚分。
# 这就是为什么我们在定义权重时使用了regularizer参数，并为它分配了一个l2_regularizer。这告诉了TensorFlow要跟踪
# l2_regularizer这个变量的L2正则化项（并通过参数reg_constant对它们进行加权）。
# 所有正则化项被添加到一个损失函数可以访问的集合——tf.GraphKeys.REGULARIZATION_LOSSES。
# 将所有正则化损失的总和与先前计算的triplet_loss相加，以得到我们的模型的总损失。
total_loss = tf.add_n([cross_entropy_mean] + regularization_losses,
name='total_loss')
# Build a Graph that trains the model with one batch of examples and updates the model parameters
# 确定优化方法并求根据损失函数求梯度,在这里面,每更新一次参数,global_step会加1
train_op = util.train(total_loss, global_step, args.optimizer,
learning_rate, args.moving_average_decay,
tf.global_variables(), args.log_histograms)
# top_k_op = tf.nn.in_top_k(predicts_soft, labels_placeholder, 1)
correct_predict = tf.equal(tf.argmax(predicts_soft, 1),
tf.argmax(labels_placeholder, 1))
# Create a saver
# 创建一个saver用于保存或从内存中恢复一个模型参数
saver = tf.train.Saver(tf.trainable_variables(), max_to_keep=3)
# Build the summary operation based on the TF collection of Summaries.
summary_op = tf.summary.merge_all()
# Start running operations on the Graph.
# 能够在gpu上分配的最大内存
# gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=args.gpu_memory_fraction)
# sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options, log_device_placement=False))
sess = tf.Session()
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
summary_writer = tf.summary.FileWriter(log_dir, sess.graph)
# 获取线程坐标
coord = tf.train.Coordinator()
# 将队列中的所有sunner开始执行
tf.train.start_queue_runners(coord=coord, sess=sess)
with sess.as_default():
# Training and validation loop
print('Running training')
epoch = 0
# 将所有数据过一遍的次数
while epoch < args.max_nrof_epochs:
# 这里是返回当前的global_step值吗,step可以看做是全局的批处理个数
step = sess.run(global_step, feed_dict=None)
# epoch_size是一个epoch中批的个数
# 这个epoch是全局的批处理个数除以一个epoch中批的个数得到epoch,这个epoch将用于求学习率
epoch = step // args.epoch_size
# Train for one epoch
train(args, sess, epoch, image_list, label_list,
index_dequeue_op, enqueue_op, image_paths_placeholder,
labels_placeholder, learning_rate_placeholder,
phase_train_placeholder, batch_size_placeholder,
global_step, total_loss, train_op, summary_op,
summary_writer, regularization_losses)
# Save variables and the metagraph if it doesn't exist already
save_variables_and_metagraph(sess, saver, summary_writer,
model_dir, subdir, step)
if args.lfw_dir:
evaluate(sess, correct_predict, image_paths_placeholder,
labels_placeholder, phase_train_placeholder,
batch_size_placeholder, logits, label_batch,
test_image_list, args.test_batch_size, log_dir,
step, summary_writer)
return model_dir
def main(args,
train_file,
dev_file,
embeddings_file,
vocab_file,
target_dir,
max_length=50,
epochs=50,
batch_size=128,
lr=0.0005,
patience=5,
max_grad_norm=10.0,
gpu_index=0,
checkpoint=None):
device = torch.device(
"cuda:{}".format(gpu_index) if torch.cuda.is_available() else "cpu")
print(20 * "=", " Preparing for training ", 20 * "=")
# 保存模型的路径
if not os.path.exists(target_dir):
os.makedirs(target_dir)
# -------------------- Data loading ------------------- #
print("\t* Loading training data...")
train_data = LCQMC_Dataset(train_file, vocab_file, max_length)
train_loader = DataLoader(train_data, shuffle=True, batch_size=batch_size)
print("\t* Loading validation data...")
dev_data = LCQMC_Dataset(dev_file, vocab_file, max_length)
dev_loader = DataLoader(dev_data, shuffle=True, batch_size=batch_size)
# -------------------- Model definition ------------------- #
print("\t* Building model...")
embeddings = load_embeddings(embeddings_file)
model = RE2(args, embeddings, device=device).to(device)
# -------------------- Preparation for training ------------------- #
criterion = nn.CrossEntropyLoss()
# 过滤出需要梯度更新的参数
parameters = filter(lambda p: p.requires_grad, model.parameters())
# optimizer = optim.Adadelta(parameters, params["LEARNING_RATE"])
optimizer = torch.optim.Adam(parameters, lr=lr)
# optimizer = torch.optim.Adam(model.parameters(), lr=lr)
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer,
mode="max",
factor=0.85,
patience=0)
best_score = 0.0
start_epoch = 1
# Data for loss curves plot
epochs_count = []
train_losses = []
valid_losses = []
# Continuing training from a checkpoint if one was given as argument
if checkpoint:
checkpoint = torch.load(checkpoint)
start_epoch = checkpoint["epoch"] + 1
best_score = checkpoint["best_score"]
print("\t* Training will continue on existing model from epoch {}...".
format(start_epoch))
model.load_state_dict(checkpoint["model"])
optimizer.load_state_dict(checkpoint["optimizer"])
epochs_count = checkpoint["epochs_count"]
train_losses = checkpoint["train_losses"]
valid_losses = checkpoint["valid_losses"]
# Compute loss and accuracy before starting (or resuming) training.
_, valid_loss, valid_accuracy, auc = validate(model, dev_loader, criterion)
print(
"\t* Validation loss before training: {:.4f}, accuracy: {:.4f}%, auc: {:.4f}"
.format(valid_loss, (valid_accuracy * 100), auc))
# -------------------- Training epochs ------------------- #
print("\n", 20 * "=", "Training RE2 model on device: {}".format(device),
20 * "=")
patience_counter = 0
for epoch in range(start_epoch, epochs + 1):
epochs_count.append(epoch)
print("* Training epoch {}:".format(epoch))
epoch_time, epoch_loss, epoch_accuracy = train(model, train_loader,
optimizer, criterion,
epoch, max_grad_norm)
train_losses.append(epoch_loss)
print("-> Training time: {:.4f}s, loss = {:.4f}, accuracy: {:.4f}%".
format(epoch_time, epoch_loss, (epoch_accuracy * 100)))
print("* Validation for epoch {}:".format(epoch))
epoch_time, epoch_loss, epoch_accuracy, epoch_auc = validate(
model, dev_loader, criterion)
valid_losses.append(epoch_loss)
print(
"-> Valid. time: {:.4f}s, loss: {:.4f}, accuracy: {:.4f}%, auc: {:.4f}\n"
.format(epoch_time, epoch_loss, (epoch_accuracy * 100), epoch_auc))
# Update the optimizer's learning rate with the scheduler.
scheduler.step(epoch_accuracy)
# Early stopping on validation accuracy.
if epoch_accuracy < best_score:
patience_counter += 1
else:
best_score = epoch_accuracy
patience_counter = 0
torch.save(
{
"epoch": epoch,
"model": model.state_dict(),
"best_score": best_score,
"epochs_count": epochs_count,
"train_losses": train_losses,
"valid_losses": valid_losses
}, os.path.join(target_dir, "best.pth.tar"))
torch.save(
{
"epoch": epoch,
"model": model.state_dict(),
"best_score": best_score,
"optimizer": optimizer.state_dict(),
"epochs_count": epochs_count,
"train_losses": train_losses,
"valid_losses": valid_losses
}, os.path.join(target_dir, "RE2_{}.pth.tar".format(epoch)))
if patience_counter >= patience:
print("-> Early stopping: patience limit reached, stopping...")
break
print("\nParameters:")
for attr, value in sorted(args.__dict__.items()):
print("\t{}={}".format(attr.upper(), value))
# model
cnn = model.CNN_Text(args)
if args.cuda:
torch.cuda.set_device(args.device)
cnn = cnn.cuda()
# train or predict
if args.predict is not None:
if args.date != '':
util.daily_predict(cnn, args)
output = './input/news/' + args.date[:4] + '/news_' + args.date + '.csv'
os.system('mv ' + output + '_bak ' + output)
else:
mymodels, word2idx, stopWords = util.predictor_preprocess(cnn, args)
print(util.predict(args.predict, mymodels, word2idx, stopWords, args))
elif args.eval is not False:
mymodels, word2idx, stopWords = util.predictor_preprocess(cnn, args)
util.bma_eval(X_test, y_test, mymodels, 'Testing ', args)
else:
print()
try:
util.train(X_train, y_train, X_valid, y_valid, X_test, y_test, cnn,
args)
except KeyboardInterrupt:
print('\n' + '-' * 89)
print('Exiting from training early')
def initial_process():
util.topic_log("Initial training")
util.train(model, args, train_loader, val_loader, 'initial_train')
top1_acc, top5_acc = util.val_epoch(val_loader, model, args, topk=(1, 5))
util.log(args.log_file_path, f"initial_accuracy\t{top1_acc} ({top5_acc})")
