def train_test_model(self):
data, labels = self.load_data()
train_data, test_data, train_labels, test_labels = self.train_test_split(
data, labels)
#training model
from model import CNNModel
cnnmodel = CNNModel(self.IMG_DIMS, self.OUTPUT_DIM)
model = cnnmodel.get_model()
idx = np.random.permutation(train_data.shape[0])
model.fit(train_data[idx],
train_labels[idx],
verbose=2,
epochs=DRPredictor.EPOCHS)
#testing model
preds = np.argmax(model.predict(test_data), axis=1)
test_labels = np.argmax(test_labels, axis=1)
print('predicted : ', preds)
print('actual : ', test_labels)
#metrics
self.find_metrics(test_labels, preds)
def extract_features(path, model_type):
if model_type == 'inceptionv3':
from keras.applications.inception_v3 import preprocess_input
target_size = (299, 299)
elif model_type == 'vgg16':
from keras.applications.vgg16 import preprocess_input
target_size = (224, 224)
# Get CNN Model from model.py
model = CNNModel(model_type)
features = dict()
# Extract features from each photo
for name in tqdm(os.listdir(path)):
# Loading and resizing image
filename = path + name
image = load_img(filename, target_size=target_size)
# Convert the image pixels to a numpy array
image = img_to_array(image)
# Reshape data for the model
image = image.reshape(
(1, image.shape[0], image.shape[1], image.shape[2]))
# Prepare the image for the CNN Model model
image = preprocess_input(image)
# Pass image into model to get encoded features
feature = model.predict(image, verbose=0)
# Store encoded features for the image
image_id = name.split('.')[0]
features[image_id] = feature
return features
def restore_model(model_dir, model_name):
"""
Restore model from disk
:param model_dir: directory where model checkpoint file is stored
:param model_name: name of the stored model
:return: loaded model
"""
restore_file = os.path.join(model_dir, model_name)
try:
os.path.exists(restore_file)
except FileNotFoundError:
print("Model checkpoint file does NOT exist.")
sys.exit(-1)
try:
checkpoint = torch.load(restore_file)
except IOError:
print("Could not load the checkpoint file.")
sys.exit(-1)
model = CNNModel()
model.load_state_dict(checkpoint['state_dict'])
return model
def train():
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
json_file = open("parameters.json")
parameters = json.load(json_file)
json_file.close()
net = CNNModel(1, 10)
optimizer = torch.optim.Adam(net.parameters(), lr=parameters["lr"])
criterion = nn.BCELoss()
if torch.cuda.is_available():
net = torch.nn.DataParallel(net,
device_ids=range(
torch.cuda.device_count())).cuda()
cudnn.benchmark = True
ecg_dataset = EcgDataset(is_train=True)
train_loader = torch.utils.data.DataLoader(dataset=ecg_dataset,
batch_size=10)
for epoch in range(parameters["num_epochs"]):
net.train()
for i, (data, label) in enumerate(train_loader):
data, label = data.to(device), label.to(device)
output = net(data)
optimizer.zero_grad()
loss = criterion(output, label)
loss.backward()
optimizer.step()
print('Epoch [{}/{}], Loss: {:.4f}'.format(epoch + 1,
parameters["num_epochs"],
loss.item()))
evaluation(net)
def async_episode(best_model_num) -> tuple:
logging.debug("episode process started")
import tensorflow as tf
import chess
from MonteCarloTS import MonteCarloTS
physical_devices = tf.config.list_physical_devices('GPU')
if len(physical_devices) != 0:
tf.config.experimental.set_memory_growth(physical_devices[0], True)
logging.debug("tensorflow input processed")
valids, states, improv_policy, win_loss = [], [], [], []
best_model = CNNModel(best_model_num)
logging.info("model initialised")
best_model.load_weights()
logging.debug("model loaded")
board = chess.Board()
logging.debug("chess board created")
mcts = MonteCarloTS(board.copy(), best_model)
logging.debug("mcts tree created")
while not board.is_game_over() and board.fullmove_number < 150:
move = mcts.search()
board.push(move)
print(move)
reward_white = {"1-0": 1, "1/2-1/2": 0, "*": -1, "0-1": -1}
logging.info(f'finished game with {board.result()}')
for node in mcts.visited:
policy = mcts.get_improved_policy(node, include_empty_spots=True)
z = reward_white[board.result()]
if node.state.board.turn == chess.BLACK:
z *= -1
states.append(node.state.get_representation())
valids.append(node.state.get_valid_vector())
improv_policy.append(policy)
win_loss.append(z)
return states, valids, improv_policy, win_loss
def __init__(self):
print('DarkOCR initialization...')
self.data = ImageData()
# reading data
self.data.read_origin_data(pickle_path)
self.model = CNNModel(image_dim, image_dim, classes_count)
self.models_fold = [
CNNModel(image_dim, image_dim, classes_count)
for i in range(fold_count)
]
print('Complete')
class ACAgent():
def __init__(self, dir):
super(ACAgent, self).__init__()
self.Model = CNNModel()
self.Model.load_weights(dir)
def choice_action(self, state):
data = self.Model(np.array(state)[np.newaxis, :].astype(np.float32))
prob_weights = data[0].numpy()
action = np.random.choice(range(prob_weights.shape[1]),
p=prob_weights.ravel())
return action
def gen_result(model_path,
test_img_dir,
target_dir,
batch_size=64,
num_gpus=1):
cnn_model = CNNModel(model_path, False, gpu_nums=num_gpus)
result_dict = {'filename': [], 'probability': []}
for img_path in glob(os.path.join(test_img_dir, '*.jpg')):
print(img_path)
img_pred = cnn_model.predict_one_img(img_path, batch_size)
result_dict['filename'].append(os.path.split(img_path)[-1])
result_dict['probability'].append(img_pred if img_pred < 1 else 0.999)
result_df = pd.DataFrame(result_dict)
result_df.to_csv(os.path.join(target_dir, 'result.csv'), index=False)
def main():
trained_model = './trained_model.pth'
test_batch_dir = './cifar-10/test_batch'
classifier = CNNModel()
classifier.load_state_dict(torch.load(trained_model))
classifier.cuda()
classifier.eval()
test_x, test_y = unpickle(test_batch_dir)
test_x, test_y = torch.tensor(np.reshape(
test_x, (len(test_x), 3, 32, 32))).to(
'cuda', dtype=torch.float), torch.tensor(test_y).cuda()
classes = [
'Airplane', 'Automobile', 'Bird', 'Cat', 'Deer', 'Dog', 'Frog',
'Horse', 'Ship', 'Truck'
]
# calculating the accuracy of our classifier;
print("Calculating accuracy...")
correct = 0
total = len(test_x)
with torch.no_grad():
out = classifier(test_x)
_, predicted = torch.max(out, 1)
# calculate the total accuracy
correct += (predicted == test_y).sum().item()
print('Accuracy: %5d %%' % (correct / total * 100))
def main(argv=None):
cnn = CNNModel()
ckpt = tf.train.get_checkpoint_state(CKPT_PATH)
if ckpt:
cnn.saver.restore(cnn.sess, ckpt.model_checkpoint_path)
else:
print('ckpt is not exist.')
exit(1)
reader = RcImageReader(EVAL_FILE)
correct_count = 0
for index in range(len(reader.bytes_array)):
record = reader.read(index)
x = record.image_array.reshape([1, 60, 160, 1])
t = record.steer_array.reshape([1, 180])
p, acc = cnn.sess.run([cnn.predictions, cnn.accuracy],
feed_dict={
cnn.input_holder: x,
cnn.label_holder: t,
cnn.keepprob_holder: 1.0
})
correct_count += acc
label = np.argmax(record.steer_array)
answer = np.argmax(p, 1)
print('label%3d - answer%3d' % (label, answer))
print('total accuracy : %f' % (float(correct_count) / index))
def load(args, checkpoint_dir):
state_dict = torch.load(os.path.join(checkpoint_dir, 'checkpoint.pth'))
from collections import OrderedDict
new_state_dict = OrderedDict()
for k, v in state_dict.items():
if 'module' in k:
namekey = k[7:] # remove `module.`
else:
namekey = k
new_state_dict[namekey] = v
if args.model_type == 'bert':
config = BertConfig.from_json_file(os.path.join(checkpoint_dir, 'config.bin'))
model = BertForSequenceClassification(config)
model.load_state_dict(new_state_dict)
elif args.model_type == 'cnn':
model = CNNModel(n_vocab=args.vocab_size, embed_size=args.embed_size, num_classes=args.num_labels,
num_filters=args.num_filters, filter_sizes=args.filter_sizes, device=args.device)
model.load_state_dict(new_state_dict)
elif args.model_type == 'lstm':
model = LSTMModel(n_vocab=args.vocab_size, embed_size=args.embed_size, num_classes=args.num_labels,
hidden_size=args.hidden_size, device=args.device)
model.load_state_dict(new_state_dict)
elif args.model_type == 'char-cnn':
model = CharCNN(num_features=args.num_features, num_classes=args.num_labels)
model.load_state_dict(new_state_dict)
else:
raise ValueError('model type is not found!')
return model.to(args.device)
def my_model(features, labels, mode, params):
"""DNN with three hidden layers, and dropout of 0.1 probability."""
vocab, vocab2id = load_vocab()
relations, relation2id = load_relation()
word_embed = np.load(os.path.join(FLAGS.out_dir, FLAGS.word_embed_file))
training = mode == tf.estimator.ModeKeys.TRAIN
m = CNNModel(params, word_embed, features, labels, training)
# Compute evaluation metrics.
metrics = {'accuracy': m.accuracy, 'mask_accuracy': m.mask_accuracy}
tf.summary.scalar('accuracy', m.accuracy[1])
if mode == tf.estimator.ModeKeys.EVAL:
p_hook = PatTopKHook(m.prob, labels)
return tf.estimator.EstimatorSpec(mode,
loss=m.total_loss,
eval_metric_ops=metrics,
evaluation_hooks=[p_hook])
# Create training op.
assert mode == tf.estimator.ModeKeys.TRAIN
logging_hook = tf.train.LoggingTensorHook(
{
"loss": m.total_loss,
"accuracy": m.accuracy[0],
'mask_accuracy': m.mask_accuracy[0]
},
every_n_iter=FLAGS.log_freq)
return tf.estimator.EstimatorSpec(mode,
loss=m.total_loss,
train_op=m.train_op,
training_hooks=[logging_hook])
def run(args):
train_loader = torch.utils.data.DataLoader(
datasets.ImageFolder('../../ssl_data_96/supervised/train',
transform=data_transforms),
batch_size=args.batch_size,
shuffle=True,
num_workers=1) #n_worker to 4, to use 4 gpu
val_loader = torch.utils.data.DataLoader(
datasets.ImageFolder('../../ssl_data_96/supervised/val',
transform=validation_data_transforms),
batch_size=args.batch_size,
shuffle=False,
num_workers=1) #n_worker to 4, to use 4 gpu
model = nn.DataParallel(CNNModel())
model.cuda()
optimizer = optim.RMSprop(model.parameters(),
lr=args.lr,
momentum=args.momentum,
weight_decay=1e-3)
scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=5)
for epoch in range(1, args.epochs + 1):
scheduler.step()
train(epoch, model, optimizer, train_loader, args.log_interval)
validation(epoch, model, val_loader)
model_file = 'model_' + str(epoch) + '.pth'
torch.save(model.state_dict(), model_file)
writer.close()
async def setup_learner():
try:
cnn = CNNModel()
return cnn
except RuntimeError as e:
print(e)
message = '\n\nRuntimeError'
raise RuntimeError(message)
def run(args):
train_loader = torch.utils.data.DataLoader(datasets.ImageFolder(
args.data + '/train', transform=data_transforms),
batch_size=args.batch_size,
shuffle=True,
num_workers=16)
val_loader = torch.utils.data.DataLoader(datasets.ImageFolder(
args.data + '/val', transform=validation_data_transforms),
batch_size=args.batch_size,
shuffle=False,
num_workers=16)
model = CNNModel()
model = nn.DataParallel(model)
model = model.to(args.device)
if args.checkpoint is not None:
model.load_state_dict(torch.load(args.checkpoint))
optimizer = optim.Adam(model.parameters(), lr=args.lr, weight_decay=1e-3)
scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=args.step_size)
for epoch in range(1, args.epochs + 1):
scheduler.step()
train(epoch, model, optimizer, train_loader, args.log_interval)
validation(epoch, model, val_loader)
model_file = 'model_' + str(epoch) + '.pth'
torch.save(model.state_dict(), model_file)
writer.close()
def __init__(self, talker):
super(ACBrain, self).__init__()
if use_RNN:
self.model = RNNModel()
self.model.call(
np.random.random(
(batch_size, IMG_H, IMG_W, k)).astype(np.float32),
np.zeros((batch_size, hidden_unit_num), dtype=np.float32),
np.zeros((batch_size, hidden_unit_num), dtype=np.float32))
else:
self.model = CNNModel()
self.model.call(
np.random.random(
(batch_size, IMG_H, IMG_W, k)).astype(np.float32))
self.talker = talker
self.i = 1
self.optimizer = optim.Adam(learning_rate=CustomSchedule(lr))
self.states_list = self.talker.states_list
self.one_episode_reward_index = 0
def __init__(self):
self.settings = ParseArgs()
self.dataset = PNGReader(self.settings.train_file,
self.settings.train_root,
self.settings.val_file,
self.settings.val_root)
self.sampler = UniformSampler(self.dataset)
self.model = CNNModel(32, 32, 3, self.settings.num_class,
self.settings.lr, self.dataset.mean,
self.dataset.std)
def train(train_config):
use_cuda = train_config.num_gpus > 0
if use_cuda:
torch.cuda.manual_seed(train_config.num_gpus)
logger.info("Number of GPUs available: {}".format(train_config.num_gpus))
device = torch.device('cuda' if use_cuda else 'cpu')
model = CNNModel().to(device)
optimizer = optim.Adam(model.parameters(), lr=train_config.lr)
best_train_loss = 0
for epoch in range(1, train_config.num_epochs + 1):
model.train()
train_loss = 0
for batch_idx, sample_batch in enumerate(train_loader):
pdb = sample_batch['pdb']
x = sample_batch['pocket']
y_true = sample_batch['label']
x, y_true = x.to(device), y_true.to(device)
x, y_true = Variable(x), Variable(y_true)
optimizer.zero_grad()
output = model(x)
loss = F.mse_loss(output, y_true)
train_loss += loss.data[0]
loss.backward()
optimizer.step()
if batch_idx % train_config.log_train_freq == 0:
logger.info("Train epoch: {}, Loss: {:.06f}"
.format(epoch, loss.data[0]))
if train_loss < best_train_loss:
utils.save_model(model, train_config.model_dir, logger)
def training_pipeline():
model_num = 1
best_model_num = 0
best_model = CNNModel(best_model_num)
best_model.save_weights()
for _ in range(NUM_TRAINS):
states, valids, improved_policy, win_loss = self_play(best_model_num)
contender = CNNModel(model_num)
contender.train_model(np.array(states, np.uint32),
np.array(valids, np.float32), np.array(win_loss),
np.array(improved_policy))
contender_wins = bot_fight(best_model.model_num, contender.model_num)
if contender_wins >= np.ceil(BOT_GAMES * 0.55):
best_model = contender
best_model_num = contender.model_num
logging.info(
f'best model: {best_model_num}, new model won {contender_wins}')
best_model.save_weights(best=True)
model_num += 1
def predict_test_set(model_path,
batch_size,
pkl_data_dir,
target_dir,
num_gpus=1):
dataset = DataSet(data_dir=config['data']['data_dir'],
test=True,
pkl_file_dir=pkl_data_dir)
cnn_model = CNNModel(model_path, False, gpu_nums=num_gpus)
test_data_gen = dataset.get_test_data_gen(batch_size)
pred_list = []
label_list = []
fp_dict = {'file_path': [], 'pkl_idx': [], 'p': []}
fn_dict = {'file_path': [], 'pkl_idx': [], 'p': []}
for batch_data, batch_label, batch_info in test_data_gen:
preds = cnn_model.predict(batch_data['inputs'], batch_size)
idx = 0
for pred in preds:
l = batch_label['out_class'][idx][1]
p = pred[1]
label_list.append(l)
pred_list.append(p)
if p >= 0.5 and l != 1:
fp_dict['file_path'].append(batch_info['file_path'][idx])
fp_dict['pkl_idx'].append(batch_info['pkl_idx'][idx])
fp_dict['p'].append(p)
if p < 0.5 and l != 0:
fn_dict['file_path'].append(batch_info['file_path'][idx])
fn_dict['pkl_idx'].append(batch_info['pkl_idx'][idx])
fn_dict['p'].append(p)
idx += 1
auc = cal_roc_and_auc(np.array(pred_list), np.array(label_list))
fn_df = pd.DataFrame(fn_dict)
fp_df = pd.DataFrame(fp_dict)
fn_df.to_csv(os.path.join(target_dir, 'fn_result.csv'), index=False)
fp_df.to_csv(os.path.join(target_dir, 'fp_result.csv'), index=False)
draw_roc(np.array(pred_list), np.array(label_list), target_dir)
print('auc:%f' % auc)
def async_arena(iteration, best_model_num, new_model_num):
import tensorflow as tf
physical_devices = tf.config.list_physical_devices('GPU')
if len(physical_devices) != 0:
tf.config.experimental.set_memory_growth(physical_devices[0], True)
new_model_wins = 0
board = chess.Board()
best_model = CNNModel(best_model_num)
best_model.load_weights()
new_model = CNNModel(new_model_num)
new_model.load_weights()
mcts_best = MonteCarloTS(chess.Board(), best_model)
mcts_new = MonteCarloTS(chess.Board(), new_model)
if iteration % 2 == 0:
turns = {"best": chess.WHITE, "new": chess.BLACK}
else:
turns = {"best": chess.BLACK, "new": chess.WHITE}
while not board.is_game_over(
) and board.fullmove_number < 150 and not board.is_repetition(count=4):
if turns["best"] == chess.WHITE:
move = mcts_best.search(training=True)
board.push(move)
mcts_new.enemy_move(move)
move = mcts_new.search(training=True)
if move is None:
break
board.push(move)
mcts_best.enemy_move(move)
else:
move = mcts_new.search(training=True)
board.push(move)
mcts_best.enemy_move(move)
move = mcts_best.search(training=True)
if move is None:
break
board.push(move)
mcts_new.enemy_move(move)
s = board.result()
if s == "1-0" and turns["new"] == chess.WHITE:
new_model_wins += 1
elif s == "0-1" and turns["new"] == chess.BLACK:
new_model_wins += 1
if new_model_wins == 1:
logging.info("new_model won")
return new_model_wins
def on_message(client, userdata, msg):
try:
print("Model from trainer received!")
print('Topic: ', msg.topic)
#print('Message: ', msg.payload)
model_str = msg.payload
buff = io.BytesIO(bytes(model_str))
# Create a dummy model to read weights
model = CNNModel()
model.load_state_dict(torch.load(buff))
global trainer_weights
trainer_weights.append(copy.deepcopy(model.state_dict()))
# Wait until we get trained weights from all trainers
if len(trainer_weights) == NUM_TRAINERS:
update_global_weights_and_send(trainer_weights)
trainer_weights.clear()
except:
print("Unexpected error:", sys.exc_info())
def train_model(work_dir,
pkl_data_dir,
initial_epoch=0,
batch_size=16,
initial_lr=0.001,
model_name='cnn',
num_gpus=1):
model_path = None
if initial_epoch > 0:
model_paths = glob(work_dir + '%s_e%02d*.hd5' %
(model_name, initial_epoch))
if len(model_paths) != 1:
print('cannot found model save file!!!!')
assert (False)
else:
model_path = model_paths[0]
initial_lr = initial_lr * (config['train']['lr_decay']
**(initial_epoch - 1))
else:
initial_epoch = 0
dataset = DataSet(data_dir=config['data']['data_dir'],
train=True,
pkl_file_dir=pkl_data_dir)
cnn_model = CNNModel(model_path, True, initial_lr, num_gpus)
train_data_gen = dataset.get_train_data_gen(batch_size=batch_size)
train_data_num = dataset.get_train_data_num()
eval_data_gen = dataset.get_eval_data_gen(batch_size=batch_size)
eval_data_num = dataset.get_eval_data_num()
cnn_model.train_model(train_data_gen,
train_data_num,
eval_data_gen,
eval_data_num,
batch_size,
work_dir,
model_name=model_name,
initial_epoch=0)
def main_simple_cnn():
TEXT = data.Field(sequential=True, include_lengths=True)
LABEL = data.Field(sequential=False)
train, val, test = datasets.SNLI.splits(TEXT, LABEL)
TEXT.build_vocab(train, vectors="glove.840B.300d")
LABEL.build_vocab(train)
vocab = TEXT.vocab
train_iter, val_iter, test_iter = data.Iterator.splits(
(train, val, test),
batch_size=50,
repeat=True,
shuffle=False)
config = Config()
criterion = nn.CrossEntropyLoss()
model = CNNModel(vocab, config)
# model = Model(vocab, config)
if args.cuda:
model.cuda()
optimizer = optim.Adam([param for param in model.parameters() if param.requires_grad], lr=1e-3)
for epoch in range(args.max_epoch):
train_acc = 0.0
train_cnt = 0
for batch in train_iter:
x, y = batch, batch.label - 1
f_x = model(x)
acc = (f_x.max(1)[1] == y).type(torch.FloatTensor).mean().float()
loss = criterion(f_x, y)
model.zero_grad()
loss.backward()
optimizer.step()
if train_cnt % 100 == 0:
print 'cnt = {}, acc = {}, loss = {}'.format(train_cnt, acc, loss.float())
train_cnt += 1
train_acc += acc
test_acc = 0.0
test_cnt = 0
for batch in test_iter:
x, y = batch, batch.label - 1
f_x = model(x)
test_acc += (f_x.max(1)[1] == y).type(torch.FloatTensor).mean().float()
test_cnt += 1
print 'epoch = {}, train_acc = {}, test_acc = {}'.format(epoch, train_acc / train_cnt, test_acc / test_cnt)
def main():
batch_size = 128
epochs = 100
maxlen = 300
model_path = "cnn_model.h5"
num_words = 40000
num_label = 2
x, y = load_dataset("data/amazon_reviews_multilingual_JP_v1_00.tsv")
x = preprocess_dataset(x)
x_train, x_test, y_train, y_test = train_test_split(x,
y,
test_size=0.2,
random_state=42)
vocab = build_vocabulary(x_train, num_words)
x_train = vocab.texts_to_sequences(x_train)
x_test = vocab.texts_to_sequences(x_test)
x_train = pad_sequences(x_train, maxlen=maxlen, truncating="post")
x_test = pad_sequences(x_test, maxlen=maxlen, truncating="post")
wv = load_fasttext("data/cc.ja.300.vec.gz")
wv = filter_embeddings(wv, vocab.word_index, num_words)
model = CNNModel(num_words, num_label, embeddings=wv).build()
model.compile(optimizer="adam",
loss="sparse_categorical_crossentropy",
metrics=["acc"])
callbakcs = [
EarlyStopping(patience=3),
ModelCheckpoint(model_path, save_best_only=True)
]
model.fit(x=x_train,
y=y_train,
batch_size=batch_size,
epochs=epochs,
validation_split=0.2,
callbacks=callbakcs,
shuffle=True)
model = load_model(model_path)
api = InferenceAPI(model, vocab, preprocess_dataset)
y_pred = api.predict_from_sequence(x_test)
print("precision: {:.4f}".format(
precision_score(y_test, y_pred, average="binary")))
print("recall: {:.4f}".format(
recall_score(y_test, y_pred, average="binary")))
print("f1: {:.4f}".format(f1_score(y_test, y_pred, average="binary")))
def train():
train_dl = DataLoader(FSLDataset('train', iteration=600),
batch_size=B,
num_workers=4,
collate_fn=collate_fn)
val_dl = DataLoader(FSLDataset('val', iteration=200),
batch_size=B,
num_workers=4,
collate_fn=collate_fn)
test_dl = DataLoader(FSLDataset('test', iteration=200),
batch_size=B,
num_workers=4,
collate_fn=collate_fn)
model = CNNModel(n_class=N).to(device)
for epoch in range(100):
do_train(model, train_dl)
val_acc = do_eval(model, val_dl)
test_acc = do_eval(model, test_dl)
print('@ {} Val={:.2f} Test={:.2f}'.format(epoch, val_acc, test_acc))
def main():
argument_parser = argparse.ArgumentParser()
argument_parser.add_argument("dataset_path", help="Path to the directory with the binaries for the dataset "
"(e.g ~/security.ece.cmu.edu/byteweight/elf_32")
args = argument_parser.parse_args()
print("Preprocessing")
dataset = FunctionIdentificationDataset(args.dataset_path, block_size=1000, padding_size=kernel_size - 1)
train_size = int(len(dataset) * 0.9)
test_size = len(dataset) - train_size
train_dataset, test_dataset = data.random_split(dataset, [train_size, test_size])
model = CNNModel(embedding_dim=64, vocab_size=258, hidden_dim=16, tagset_size=2, kernel_size=kernel_size)
print("Training")
train_model(model, train_dataset)
print("Testing")
test_model(model, test_dataset)
def main(argv=None):
os.system('rm -rf ' + SUMMARY_PATH)
cnn = CNNModel()
step = 0
for epoch in range(1, EPOCH_NUM + 1):
start_time = time.time()
print('Epoch %d: %s' % (epoch, TRAIN_FILE))
reader = RcImageReader(TRAIN_FILE)
reader.shuffle()
batch_images = []
batch_steers = []
for index in range(len(reader.bytes_array)):
record = reader.read(index)
# mini batch用データ作成
batch_images.append(record.image_array)
batch_steers.append(record.steer_array)
if len(batch_images) < BATCH_SIZE:
continue
cnn.sess.run(cnn.train_step,
feed_dict={
cnn.input_holder: batch_images,
cnn.label_holder: batch_steers,
cnn.keepprob_holder: KEEP_PROB
})
step += 1
_eval(cnn, step)
del batch_images[:]
del batch_steers[:]
cnn.saver.save(cnn.sess, CKPT_PATH + 'model', global_step=step)
duration = time.time() - start_time
print('duration %d' % (duration))
def train():
df_tweets = pd.read_csv("data/df_tweets", index_col=0)
df_tweets["text"] = preprocess_dataset(df_tweets["text"])
df_tweets = df_tweets.dropna(how='any')
df_tweets = df_tweets.drop(df_tweets.index[df_tweets["Irrelevant"] == 1])
x = df_tweets["text"]
# y = df_tweets[["posi_and_nega", "posi", "nega", "neutral", "Irrelevant"]]
y = df_tweets[["posi_and_nega", "posi", "nega", "neutral"]]
y = np.asarray(y)
x_train, x_test, y_train, y_test = train_test_split(x,
y,
test_size=0.2,
random_state=42)
vocab = build_vocabulary(x_train, num_words)
with open('model/tokenizer.pickle', 'wb') as handle:
pickle.dump(vocab, handle, protocol=pickle.HIGHEST_PROTOCOL)
x_train = vocab.texts_to_sequences(x_train)
x_test = vocab.texts_to_sequences(x_test)
x_train = pad_sequences(x_train, maxlen=maxlen, truncating="post")
x_test = pad_sequences(x_test, maxlen=maxlen, truncating="post")
wv = KeyedVectors.load("model/word2vec.model", mmap='r')
wv = filter_embeddings(wv, vocab.word_index, num_words)
model = CNNModel(num_words, num_label, embeddings=wv).build()
model.compile(optimizer="adam",
loss="binary_crossentropy",
metrics=["acc"])
callbakcs = [
EarlyStopping(patience=3),
ModelCheckpoint(model_path, save_best_only=True)
]
model.fit(x=x_train,
y=y_train,
batch_size=batch_size,
epochs=epochs,
validation_split=0.5,
callbacks=callbakcs,
shuffle=True)
inputs, lbl = inputs.cuda(), lbl.cuda()
# set the gradient for each parameters zero
optimizer.zero_grad()
outputs = model(inputs)
loss = criterion(outputs, lbl)
loss.backward()
optimizer.step()
print('-[step: %d, loss: %f]' % (i + 1, loss.item()))
scheduler.step()
print('Finished Training')
if __name__ == '__main__':
cnn = CNNModel()
batch = 2000
if torch.cuda.is_available():
cnn.cuda()
trainingDataset = LoadTrainingData()
dataLoader = DataLoader(dataset=trainingDataset,
batch_size=batch,
shuffle=True,
num_workers=2)
train_model(cnn, dataLoader, epoch=40, batch_size=batch)
# save model
torch.save(cnn.state_dict(), './trained_model.pth')
print "=> {0} train data loaded. train_length = {1}".format(str(train_path), str(len(x_train)) )
print "unit_length = %s" % str(unit_length)
sys.stdout.flush()
x_test, y_test, z_test, _ = load_data(test_path, MAX_WIDTH, word2vec_hash, tgt, cform, surface, first, last, classias, tgt_center, tgt_sentence, data_size)
print "=> {0} test data loaded. test_length = {1}".format(str(test_path), str(len(x_test)) )
sys.stdout.flush()
N_train = len(x_train) #train, testの長さ
N_test = len(x_test)
n_epoch = 100
batchsize = args.batchsize
filters, units_1, drop_ratio, conv_width = args.filters, args.units, args.drop_ratio, args.conv_width
#modelをCNNモデルに設定
model = CNNModel(filters, units_1, unit_length, drop_ratio, conv_width)
# Setup optimizer，最適化手法としてAdamを使う
opt = optimizers.Adam()
opt.setup(model)
if args.decay_rate != 0.0:
opt.add_hook(optimizer.WeightDecay(args.decay_rate))
cur_at = time.time()
models = []
counter = Counter()
print "epoch, sum_loss_train/N, sum_accuracy_train/N, sum_loss_test, sum_accuracy_test, epoch_time"
