def lstm_init_states(batch_size):
""" Returns a tuple of names and zero arrays for LSTM init states"""
hp = Hyperparams()
init_shapes = lstm.init_states(batch_size=batch_size, num_lstm_layer=hp.num_lstm_layer, num_hidden=hp.num_hidden)
init_names = [s[0] for s in init_shapes]
init_arrays = [mx.nd.zeros(x[1]) for x in init_shapes]
return init_names, init_arrays
def lstm_init_states(batch_size):
hp = Hyperparams()
init_shapes = lstm.init_states(batch_size=batch_size, num_lstm_layer=hp.num_lstm_layer, num_hidden=hp.num_hidden)
init_names = [s[0] for s in init_shapes]
init_arrays = [mx.nd.zeros(x[1]) for x in init_shapes]
return init_names, init_arrays
def test(epoch):
global best_acc
global test_total
global test_correct
net.eval()
test_loss = 0
correct = 0
total = 0
with torch.no_grad():
for batch_idx, (inputs, targets) in enumerate(testloader):
inputs, targets = inputs.cuda(), targets.cuda()
outputs = net(inputs)
loss = criterion(outputs, targets)
test_loss += loss.item()
_, predicted = outputs.max(1)
total += targets.size(0)
correct += predicted.eq(targets).sum().item()
progress_bar(
batch_idx, len(testloader),
'Loss: %.3f | Acc: %.3f%% (%d/%d)' %
(test_loss /
(batch_idx + 1), 100. * correct / total, correct, total))
test_total = total
test_correct = correct
# Save checkpoint.
acc = 100. * correct / total
if acc > best_acc:
all_accs.append(acc)
all_epochs.append(epoch)
all_times.append(round(time_to_train, 2))
print('Saving..')
state = {
'net': net.state_dict(),
'acc': all_accs,
'epoch': all_epochs,
'time_to_train': all_times,
'basic_info': hp.get_info_dict()
}
if not os.path.isdir('checkpoint'):
os.mkdir('checkpoint')
if hp.with_lars:
torch.save(
state,
'./checkpoint/withLars-' + str(hp.batch_size) + '.pth')
else:
torch.save(
state,
'./checkpoint/noLars-' + str(hp.batch_size) + '.pth')
best_acc = acc
import pandas, logging, sys
from model import Model
import numpy as np
from keras.models import load_model
from data_loader import Test_data_loader
from hyperparams import Hyperparams
from random import shuffle
H = Hyperparams()
#logger configuration
FORMAT = "[%(filename)s: %(lineno)3s] %(levelname)s: %(message)s"
logging.basicConfig(level=logging.INFO, format=FORMAT)
logger = logging.getLogger(__name__)
# model = Model().get_model()
# model.load_weights("saved_weights/model_epoch_{}.h5".format(sys.argv[1]))
# logger.info("Saved weight model_epoch_{} loaded".format(sys.argv[1]))
model = load_model("batch_14.h5")
test_batch_generator = Test_data_loader(H.test_batch_size)
test_csv = pandas.read_csv("../test.csv")
# shuffle(test_csv['image_name'])
num_test = test_csv["image_name"].shape[0]
# test_csv = test_csv[:num_test]
predictions = np.ones(shape=(num_test, 4))
logger.info("Empty predictions array initialized of shape : {}".format(
predictions.shape))
for i in range(num_test // H.test_batch_size + 1):
img_batch = test_batch_generator[i]
from hyperparams import Hyperparams
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
if __name__ == '__main__':
parser = argparse.ArgumentParser(description="Run Consumer")
parser.add_argument('--server', type=str, required=True)
parser.add_argument('--model_tag', type=str, required=True)
args = parser.parse_args()
server = int(args.server)
model_tag = args.model_tag
train_size = {242: 411199, 204: 467903}
test_size = {242: 372573, 204: 512064}
hp = Hyperparams(server,
model_tag,
train_size=train_size[server],
test_size=test_size[server])
logger = logging.getLogger(__name__)
logging.root.setLevel(level=logging.INFO)
model_trainer = ModelTrainer(hp)
logger.info('model trainer initialized.')
model_trainer.model_train()
logger.info('model trainer done.')
#coding=utf-8
import os
import sys
import tensorflow as tf
import numpy as np
import logging
import time
from hyperparams import Hyperparams
import data_helper as dh
prj_name = sys.argv[1]
model_id = int(sys.argv[2])
hp = Hyperparams(prj_name)
tf.flags.DEFINE_string("test_file", hp.train_params["test_file"],
"Data for the training data.")
tf.flags.DEFINE_string("model_path", hp.train_params["model_path"],
"Path to save model")
tf.flags.DEFINE_integer("max_seq_len", hp.model_params["max_seq_len"],
"Max length for sequence")
tf.flags.DEFINE_integer("batch_size", hp.train_params["batch_size"],
"Batch size for validation")
FLAGS = tf.flags.FLAGS
def examples(querys_a, ids_a, querys_b, ids_b, labels):
logging.info("Load datas like this:\n")
for query_a, id_a, query_b, id_b, label in zip(querys_a, ids_a, querys_b,
os.mkdir('checkpoint')
if hp.with_lars:
torch.save(
state,
'./checkpoint/withLars-' + str(hp.batch_size) + '.pth')
else:
torch.save(
state,
'./checkpoint/noLars-' + str(hp.batch_size) + '.pth')
best_acc = acc
if hp.with_lars:
print('Resnet50, data=cifar10, With LARS')
else:
print('Resnet50, data=cifar10, Without LARS')
hp.print_hyperparms()
for epoch in range(0, hp.num_of_epoch):
print('\nEpoch: %d' % epoch)
if epoch <= hp.warmup_epoch: # for readability
warmup_scheduler.step()
if epoch > hp.warmup_epoch: # after warmup, start decay scheduler with warmup-ed learning rate
poly_decay_scheduler.base_lrs = warmup_scheduler.get_lr()
for param_group in optimizer.param_groups:
print('lr: ' + str(param_group['lr']))
train(epoch)
test(epoch)
epochs.append(epoch)
train_accs.append(100. * train_correct / train_total)
test_accs.append(100. * test_correct / test_total)
def main():
"""Program entry point"""
args = parse_args()
if not any(args.loss == s for s in ['ctc', 'warpctc']):
raise ValueError(
"Invalid loss '{}' (must be 'ctc' or 'warpctc')".format(args.loss))
hp = Hyperparams()
# Start a multiprocessor captcha image generator
mp_captcha = MPDigitCaptcha(font_paths=get_fonts(args.font_path),
h=hp.seq_length,
w=30,
num_digit_min=3,
num_digit_max=4,
num_processes=args.num_proc,
max_queue_size=hp.batch_size * 2)
try:
# Must call start() before any call to mxnet module (https://github.com/apache/incubator-mxnet/issues/9213)
mp_captcha.start()
if args.gpu:
contexts = [mx.context.gpu(i) for i in range(args.gpu)]
else:
contexts = [mx.context.cpu(i) for i in range(args.cpu)]
init_states = lstm.init_states(hp.batch_size, hp.num_lstm_layer,
hp.num_hidden)
data_train = OCRIter(hp.train_epoch_size // hp.batch_size,
hp.batch_size,
init_states,
captcha=mp_captcha,
name='train')
data_val = OCRIter(hp.eval_epoch_size // hp.batch_size,
hp.batch_size,
init_states,
captcha=mp_captcha,
name='val')
symbol = lstm.lstm_unroll(num_lstm_layer=hp.num_lstm_layer,
seq_len=hp.seq_length,
num_hidden=hp.num_hidden,
num_label=hp.num_label,
loss_type=args.loss)
head = '%(asctime)-15s %(message)s'
logging.basicConfig(level=logging.DEBUG, format=head)
module = mx.mod.Module(symbol,
data_names=[
'data', 'l0_init_c', 'l0_init_h',
'l1_init_c', 'l1_init_h'
],
label_names=['label'],
context=contexts)
metrics = CtcMetrics(hp.seq_length)
module.fit(
train_data=data_train,
eval_data=data_val,
# use metrics.accuracy or metrics.accuracy_lcs
eval_metric=mx.gluon.metric.np(metrics.accuracy,
allow_extra_outputs=True),
optimizer='sgd',
optimizer_params={
'learning_rate': hp.learning_rate,
'momentum': hp.momentum,
'wd': 0.00001,
},
initializer=mx.init.Xavier(factor_type="in", magnitude=2.34),
num_epoch=hp.num_epoch,
batch_end_callback=mx.callback.Speedometer(hp.batch_size, 50),
epoch_end_callback=mx.callback.do_checkpoint(args.prefix),
)
except KeyboardInterrupt:
print("W: interrupt received, stopping...")
finally:
# Reset multiprocessing captcha generator to stop processes
mp_captcha.reset()
import torch
import torch.nn as nn
import torch.optim as optim
from simple_nmt.encoder import Encoder
from simple_nmt.decoder import Decoder
from simple_nmt.seq2seq import Seq2Seq
from data_loader import DataLoader
#from train import
from hyperparams import Hyperparams
if __name__ == "__main__":
hparams = Hyperparams()
cuda = hparams.use_cuda and torch.cuda.is_available()
device = torch.device('cuda' if cuda else 'cpu')
enc, enc_hidden = Encoder()
dec, dec_hidden = Decoder()
model = Seq2Seq(enc, dec)
model.flatten_parameters() # 일자로 펴 준다
model = nn.DataParallel(model).to(device)
optimizer = optim.Adam(model.module.parameters(), lr=hparams.lr)
# loss function의 변수로 criterion 대게 씀. reduction은 loss를 어떻게 넘길지인데, default는 mean이지만 sum이 더 빠르다고 함. 더 정확한 것은 mean
# ignore_index는 loss 계산시 무시할 인덱스인데, PADDING 된 것들에는 loss 계산할 필요가 없다.
criterion = nn.CrossEntropyLoss(reduction='sum',
ignore_index=PAD_token).to(device)
#########
from hyperparams import Hyperparams
from surrogate_model import SurrogateModel
from optimizer import Optimizer
from binarydataset import BinaryDataset
import sys
# get the default hyper parameters
hp = Hyperparams()
config = hp.get_default()
# set any other command line configurations in key=val format
hp.set_command_line_args(argv=sys.argv[2:], config=config)
print('Configuration:', config)
# read the dataset
ds = BinaryDataset(folder_path=sys.argv[1],
config=config)
# set the number of features, useful for the feature batch placeholder in the surogate model
config['num_features'] = ds.num_features
# create the surogate model
model = SurrogateModel(config=config)
model.create_model()
# create the optimizer
optimizer = Optimizer(config=config,
dataset=ds,
model=model)
optimizer.run()
def main(hp):
g = HIN(hp)
model = NSHE(g, hp)
if hp.train_on_gpu:
model.cuda()
if hp.opt_method == "Adam" or hp.opt_method == "adam":
optimizer = optim.Adam(model.parameters(), lr=hp.alpha)
else:
optimizer = optim.SGD(model.parameters(), lr=hp.alpha)
node_emb = run(model, g, optimizer) # 模型训练
print("Train finished")
model.eval()
evaluate(node_emb, g.t_info)
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='GCN')
parser.add_argument("--dataset",
type=str,
default='dblp',
help="dataset to train")
parser.add_argument("--task",
type=str,
default='cla',
help="task to train")
args = parser.parse_args()
hp = Hyperparams(args.dataset, args.task)
warnings.filterwarnings('ignore')
main(hp)
from hyperparams import Hyperparams
from surrogate_model import SurrogateModel
from universal_loss.universal_loss_optimizers import UniversalLossOptimizer
from binarydataset import BinaryDataset
import sys
# get the default hyper parameters
hp = Hyperparams()
config = hp.get_default()
# set any other command line configurations in key=val format
for arg in sys.argv[1:]:
key, raw_value = arg.split('=')
# set binary values
if key == 'stratified_batch' or key == 'init_universal_loss':
value = raw_value.lower() == 'true'
print('Binary configuration:', key, '=', value)
# set float configurations
elif key == 'eta_pred' or key == 'eta_loss' or key == 'max_grad_norm':
value = float(raw_value)
print('Float configuration:', key, '=', value)
# set int configurations
elif key == 'batch_size' or key == 'num_epochs' or key == 'steps_loss' \
or key == 'steps_prediction' or key == 'performance_epoch_frequency':
value = int(raw_value)
print('Int configuration:', key, '=', value)
# set list configurations
elif key == 'prediction_layers' or key == 'loss_embedding_layers' \
or key == 'loss_aggregation_layers':
value = list(map(int, raw_value.split(',')))
print('List configuration:', key, '=', value)
# set string configurations
def face_detection(net, img_raw):
cfg = Hyperparams().cfg_mnet
img = np.float32(img_raw)
# testing scale
target_size = 1600
max_size = 2150
im_shape = img.shape
im_size_min = np.min(im_shape[0:2])
im_size_max = np.max(im_shape[0:2])
resize = float(target_size) / float(im_size_min)
# prevent bigger axis from being more than max_size:防止超过最大尺寸
if np.round(resize * im_size_max) > max_size:
resize = float(max_size) / float(im_size_max)
if True:
resize = 1
if resize != 1:
img = cv2.resize(img,
None,
None,
fx=resize,
fy=resize,
interpolation=cv2.INTER_LINEAR)
im_height, im_width, _ = img.shape
scale = np.array([img.shape[1], img.shape[0], img.shape[1],
img.shape[0]]).astype(np.float64)
img -= (104, 117, 123) # 单通道
img = img.transpose(2, 0, 1) # 转置
img = np.expand_dims(img, 0)
loc, conf, landms, test_size = net(img)
# 面框
priorbox = PriorBox_in_numpy(cfg,
image_size=(im_height, im_width),
test_size=test_size)
priors = priorbox.forward()
#prior_data = priors.data
prior_data = priors
boxes = decode_in_numpy(loc.squeeze(0), prior_data, cfg['variance'])
boxes = boxes * scale / resize
scores = conf.squeeze(0)[:, 1]
landms = decode_landm_in_numpy(landms.squeeze(0), prior_data,
cfg['variance'])
scale1 = np.array([
img.shape[3], img.shape[2], img.shape[3], img.shape[2], img.shape[3],
img.shape[2], img.shape[3], img.shape[2], img.shape[3], img.shape[2]
]).astype(np.float64)
landms = landms * scale1 / resize
# ignore low scores
inds = np.where(scores > 0.02)[0]
boxes = boxes[inds]
landms = landms[inds]
scores = scores[inds]
# keep top-K before NMS
order = scores.argsort()[::-1] # 返回的是从小到大排序的索引
#if len(order) > 10:
# order = order[:10]
# order = scores.argsort()[::-1][:args.top_k]
boxes = boxes[order]
landms = landms[order]
scores = scores[order]
# do NMS
dets = np.hstack((boxes, scores[:, np.newaxis])).astype(np.float32,
copy=False)
#print("pre:{}".format(dets.shape))
keep = py_cpu_nms_in_numpy(dets, 0.4)
# keep = nms(dets, args.nms_threshold,force_cpu=args.cpu)
dets = dets[keep, :]
#print("back:{}".format(dets.shape))
landms = landms[keep]
# keep top-K faster NMS
# dets = dets[:args.keep_top_k, :]
# landms = landms[:args.keep_top_k, :]
dets = np.concatenate((dets, landms), axis=1) # 连接
faces = []
for b in dets:
if b[4] < 0.5:
continue
# text = "{:.4f}".format(b[4])
b = list(map(int, b))
#cv2.rectangle(img_raw, (b[0], b[1]), (b[2], b[3]), (0, 0, 255), 2)
faces.append(b)
#print("facesnum:{}".format(len(faces)))
return faces
