def parse_command_line():
parser = argparse.ArgumentParser(
description="""Train, validate, and test a classifier that will detect clouds in
remote sensing data.""")
parser.add_argument("-p", "--prepare-data", help="Prepare training and validation data.",
action="store_true")
parser.add_argument("-t", "--train", help="""Train classifier. Use --graph to generate quality
graphs""", action="store_true")
parser.add_argument("-g", "--graph", help="Generate training graphs.", action="store_true")
parser.add_argument("--weights", help="""The trained model weights to use; if not provided
defaults to the network that was just trained""", type=str, default=None)
parser.add_argument("--note", help="Adds extra note onto generated quality graph.", type=str)
args = vars(parser.parse_args())
if os.environ.get("CAFFE_HOME") == None:
print "You must set CAFFE_HOME to point to where Caffe is installed. Example:"
print "export CAFFE_HOME=/usr/local/caffe"
exit(1)
# Ensure the random number generator always starts from the same place for consistent tests.
random.seed(0)
if args["prepare_data"] == True:
prepare_data();
if args["train"] == True:
train(args["graph"], weight_file=args["weights"], note=args["note"])
def get_data(folder):
main_data = [0] * 3
train_data, val_data, test_data, vocab_size, eos_id = ptb_raw_data(folder)
train_data, max_length = prepare_data(train_data, eos_id, vocab_size)
val_data, max_length = prepare_data(val_data, eos_id, vocab_size,
max_length)
test_data, max_length = prepare_data(test_data, eos_id, vocab_size,
max_length)
return train_data, val_data, test_data, vocab_size
def pet_brainmask_convnet(source_dir,
target_dir,
ratios,
feature_dim=2,
batch_size=2,
nb_epoch=10,
images_to_predict=None,
clobber=False,
model_name=False):
images = prepare_data(source_dir, target_dir, ratios, batch_size,
feature_dim, clobber)
### 1) Define architecture of neural network
model = make_model(batch_size)
### 2) Train network on data
if model_name == None: model_name = set_model_name(target_dir, feature_dim)
if not exists(model_name) or clobber:
#If model_name does not exist, or user wishes to write over (clobber) existing model
#then train a new model and save it
X_train = np.load(prepare_data.train_x_fn + '.npy')
Y_train = np.load(prepare_data.train_y_fn + '.npy')
X_test = np.load(prepare_data.test_x_fn + '.npy')
Y_test = np.load(prepare_data.test_y_fn + '.npy')
model = compile_and_run(model, X_train, Y_train, X_test, Y_test,
prepare_data.batch_size, nb_epoch)
model.save(model_name)
### 3) Produce prediction
predict(model_name, source_dir, target_dir, images, images_to_predict)
return 0
def __init__(self, config):
self.config = config
self.word2numF, self.num2word, self.words, self.poems = prepare_data()
self.poem = self.poems.split(']')
self.poem_num = len(self.poem)
if os.path.exists(self.config.model_path):
self.model = load_model(self.config.model_path)
def mri_keras(mainDir, data_dir, report_dir, input_str, ext, labelName, covPath, ratios=[0.75,0.15], batch_size=2, feature_dim=2, pad_base=0, createDataset=True):
#setup_dirs(target_dir)
imagesDf = prepare_data(mainDir, data_dir, report_dir, input_str, ext, labelName, covPath, ratios, batch_size, feature_dim, pad_base)
if createDataset:
create_PNG_dataset(imagesDf.paths)
def __init__(self, Config):
self.config = Config
self.model = None
self.word2numF, self.num2word, \
self.words, self.poems \
= prepare_data()
self.poem = self.poems.split(']')
self.poem_num = len(self.poem)
if os.path.exists(self.config.model_path):
self.model = load_model(self.config.model_path)
else:
self.train()
def minc_keras(source_dir, target_dir, input_str, label_str, ratios, feature_dim=2, batch_size=2, nb_epoch=10, images_to_predict=None, clobber=False, model_fn='model.hdf5',model_type='model_0_0', images_fn='images.csv', verbose=1 ):
data_dir = target_dir + os.sep + 'data'+os.sep
report_dir = target_dir+os.sep+'report'+os.sep
train_dir = target_dir+os.sep+'predict'+os.sep+'train'+os.sep
test_dir = target_dir+os.sep+'predict'+os.sep+'test'+os.sep
validate_dir = target_dir+os.sep+'predict'+os.sep+'validate'+os.sep
model_dir=target_dir+os.sep+'model'
if not exists(train_dir): makedirs(train_dir)
if not exists(test_dir): makedirs(test_dir)
if not exists(validate_dir): makedirs(validate_dir)
if not exists(data_dir): makedirs(data_dir)
if not exists(report_dir): makedirs(report_dir)
if not exists(model_dir): makedirs(model_dir)
images_fn = set_model_name(images_fn, report_dir, '.csv')
[images, image_dim] = prepare_data(source_dir, data_dir, report_dir, input_str, label_str, ratios, batch_size,feature_dim, images_fn, clobber=clobber)
### 1) Define architecture of neural network
model = make_model(image_dim, model_type)
### 2) Train network on data
model_fn =set_model_name(model_fn, model_dir)
history_fn = splitext(model_fn)[0] + '_history.json'
print( 'Model:', model_fn)
if not exists(model_fn) or clobber:
#If model_fn does not exist, or user wishes to write over (clobber) existing model
#then train a new model and save it
X_train=np.load(prepare_data.train_x_fn+'.npy')
Y_train=np.load(prepare_data.train_y_fn+'.npy')
X_validate=np.load(prepare_data.validate_x_fn+'.npy')
Y_validate=np.load(prepare_data.validate_y_fn+'.npy')
model,history = compile_and_run(model, model_fn, history_fn, X_train, Y_train, X_validate, Y_validate, nb_epoch)
### 3) Evaluate model on test data
model = load_model(model_fn)
X_test=np.load(prepare_data.test_x_fn+'.npy')
Y_test=np.load(prepare_data.test_y_fn+'.npy')
test_score = model.evaluate(X_test, Y_test, verbose=1)
print('Test: Loss=', test_score[0], 'Dice:', test_score[1])
np.savetxt(report_dir+os.sep+'model_evaluate.csv', np.array(test_score) )
### 4) Produce prediction
#predict(model_fn, validate_dir, data_dir, images_fn, images_to_predict=images_to_predict, category="validate", verbose=verbose)
#predict(model_fn, train_dir, data_dir, images_fn, images_to_predict=images_to_predict, category="train", verbose=verbose)
predict(model_fn, test_dir, data_dir, images_fn, images_to_predict=images_to_predict, category="test", verbose=verbose)
plot_loss(history_fn, model_fn, report_dir)
return 0
def pred_error(f_pred, prepare_data, data, iterator, verbose=False):
"""
Just compute the error
f_pred: Theano fct computing the prediction
prepare_data: usual prepare_data for that dataset.
"""
valid_err = 0
for _, valid_index in iterator:
x, mask, y = prepare_data([data[0][t] for t in valid_index],
numpy.array(data[1])[valid_index],
maxlen=None)
preds = f_pred(x, mask)
targets = numpy.array(data[1])[valid_index]
valid_err += (preds == targets).sum()
valid_err = 1. - numpy_floatX(valid_err) / len(data[0])
return valid_err
def minc_keras(source_dir, target_dir, input_str, label_str, ratios, feature_dim=2, batch_size=2, nb_epoch=10, images_to_predict=None, clobber=False, model_fn='model.hdf5',model_type='custom', images_fn='images.csv',nK="16,32,64,128", n_dil=None, kernel_size=3, drop_out=0, loss='categorical_crossentropy', activation_hidden="relu", activation_output="sigmoid", metric="categorical_accuracy", pad_base=0, verbose=1, make_model_only=False ):
setup_dirs(target_dir)
images_fn = set_model_name(images_fn, report_dir, '.csv')
[images, data] = prepare_data(source_dir, data_dir, report_dir, input_str, label_str, ratios, batch_size,feature_dim, images_fn,pad_base=pad_base, clobber=clobber)
### 1) Define architecture of neural network
Y_validate=np.load(data["validate_y_fn"]+'.npy')
nlabels=len(np.unique(Y_validate))#Number of unique labels in the labeled images
model = make_model(data["image_dim"], nlabels,nK, n_dil, kernel_size, drop_out, model_type, activation_hidden=activation_hidden, activation_output=activation_output)
if make_model_only : return(0)
### 2) Train network on data
model_fn =set_model_name(model_fn, model_dir)
history_fn = splitext(model_fn)[0] + '_history.json'
print( 'Model:', model_fn)
if not exists(model_fn) or clobber:
#If model_fn does not exist, or user wishes to write over (clobber) existing model
#then train a new model and save it
X_train=np.load(data["train_x_fn"]+'.npy')
Y_train=np.load(data["train_y_fn"]+'.npy')
X_validate=np.load(data["validate_x_fn"]+'.npy')
model,history = compile_and_run(model, model_fn, history_fn, X_train, Y_train, X_validate, Y_validate, nb_epoch, nlabels, loss=loss, verbose=verbose)
### 3) Evaluate model on test data
model = load_model(model_fn)
X_test=np.load(data["test_x_fn"]+'.npy')
Y_test=np.load(data["test_y_fn"]+'.npy')
if loss in categorical_functions :
Y_test=to_categorical(Y_test)
test_score = model.evaluate(X_test, Y_test, verbose=1)
print('Test: Loss=', test_score[0], 'Metric=', test_score[1])
#np.savetxt(report_dir+os.sep+'model_evaluate.csv', np.array(test_score) )
### 4) Produce prediction
#predict(model_fn, validate_dir, data_dir, images_fn, images_to_predict=images_to_predict, category="validate", verbose=verbose)
#predict(model_fn, train_dir, data_dir, images_fn, images_to_predict=images_to_predict, category="train", verbose=verbose)
predict(model_fn, test_dir, data_dir, images_fn, loss, images_to_predict=images_to_predict, category="test", verbose=verbose)
plot_loss(metric, history_fn, model_fn, report_dir)
return 0
def eval(sess, test_data, model, model_path):
loss_sum = 0.0
accuracy_sum = 0.0
aux_loss_sum = 0.0
nums = 0
stored_arr = []
for raw_data in test_data:
nums += 1
y, y_, adgroup, member, campaign, item, item_price, cate, commodity, node, \
effect_id, effect, effect_mask, \
pos_id, pos, pos_mask, \
direct_id, direct, direct_mask, \
direct_action_type, direct_action_value, direct_action_mask, \
pos_action_type, pos_action_value, pos_action_mask = prepare_data(raw_data)
prob, loss, acc = \
model.calculate(sess, \
[y, y_, adgroup, member, campaign, item, item_price, cate, commodity, node, \
effect_id, effect, effect_mask, \
pos_id, pos, pos_mask, \
direct_id, direct, direct_mask, \
direct_action_type, direct_action_value, direct_action_mask, \
pos_action_type, pos_action_value, pos_action_mask])
loss_sum += loss
accuracy_sum += acc
prob_1 = prob[:, 0].tolist()
target = y
target_1 = target[:, 0].tolist()
for p, t in zip(prob_1, target_1):
stored_arr.append([p, t])
test_auc = calc_auc(stored_arr)
accuracy_mean = accuracy_sum / nums
loss_mean = loss_sum / nums
global best_auc
global best_acc
if best_acc < accuracy_mean:
best_acc = accuracy_mean
if best_auc < test_auc:
best_auc = test_auc
model.save(sess, model_path)
return test_auc, loss_mean, accuracy_mean
def pred_probs(f_pred_prob, prepare_data, data, iterator, verbose=False):
""" If you want to use a trained model, this is useful to compute
the probabilities of new examples.
"""
n_samples = len(data[0])
probs = numpy.zeros((n_samples, 2)).astype(config.floatX)
n_done = 0
for _, valid_index in iterator:
x, mask, y = prepare_data([data[0][t] for t in valid_index],
numpy.array(data[1])[valid_index],
maxlen=None)
pred_probs = f_pred_prob(x, mask)
probs[valid_index, :] = pred_probs
n_done += len(valid_index)
if verbose:
print('%d/%d samples classified' % (n_done, n_samples))
return probs
def main(args):
model = transfer_DenseNet()
criterion = nn.CrossEntropyLoss()
optimizer = Adam(
model.parameters(),
lr=args.lr) # when you tune the learning_rate, it help a lot
scheduler = lr_scheduler.StepLR(optimizer,
step_size=args.lr_decay_step,
gamma=0.1)
# data augmenttion and preprocess
transforms = transform.Compose([
Zoom(0.2),
transform.RandomCrop(32, padding=4),
Rotation(35),
Shift(0.2),
transform.ToTensor(),
Gray()
])
data = prepare_data(args.train_file, args.valid_file, args.test_file)
if args.use_resample:
data[0], data[1] = resample_equal_prob(data[0], data[1])
print("Finishing resample")
epochs = args.epochs
batch_size = args.batch_size
use_cuda = args.use_cuda
train_model(data,
model,
criterion,
optimizer,
scheduler,
num_epochs=epochs,
batch_size=batch_size,
use_cuda=use_cuda,
transforms=transforms)
def get_w(sess, test_data, model, model_path):
tot_w = ''
for raw_data in test_data:
y, y_, adgroup, member, campaign, item, item_price, cate, commodity, node, \
effect_id, effect, effect_mask, \
pos_id, pos, pos_mask, \
direct_id, direct, direct_mask, \
direct_action_type, direct_action_value, direct_action_mask, \
pos_action_type, pos_action_value, pos_action_mask = prepare_data(raw_data)
w = model.get_w(sess, \
[y, y_, adgroup, member, campaign, item, item_price, cate, commodity, node, \
effect_id, effect, effect_mask, \
pos_id, pos, pos_mask, \
direct_id, direct, direct_mask, \
direct_action_type, direct_action_value, direct_action_mask, \
pos_action_type, pos_action_value, pos_action_mask])
w = w.tolist()
for x, label, z in zip(w, y, effect):
w_w = '['
for v in x:
w_w = w_w + str(v) + ','
w_w = w_w[0:-1]
w_w = w_w + ']'
w_w = w_w + '+' + str(label[0])
w_w = w_w + '+' + '['
for v in z:
w_w = w_w + '('
for vv in v:
vv = vv.tolist()
w_w = w_w + str(vv) + ','
w_w = w_w[0:-1]
w_w = w_w + ')' + '$'
w_w = w_w[0:-1]
w_w = w_w + ']' + '\n'
tot_w = tot_w + w_w
print(tot_w)
def pred_lstm(
dim_proj=128, # word embeding dimension and LSTM number of hidden units.
patience=10, # Number of epoch to wait before early stop if no progress
max_epochs=5000, # The maximum number of epoch to run
dispFreq=10, # Display to stdout the training progress every N updates
decay_c=0., # Weight decay for the classifier applied to the U weights.
lrate=0.0001, # Learning rate for sgd (not used for adadelta and rmsprop)
n_words=10000, # Vocabulary size
optimizer=adadelta, # sgd, adadelta and rmsprop available, sgd very hard to use, not recommanded (probably need momentum and decaying learning rate).
encoder='lstm', # TODO: can be removed must be lstm.
saveto='lstm_model.npz', # The best model will be saved there
validFreq=2000, # Compute the validation error after this number of update.
saveFreq=1110, # Save the parameters after every saveFreq updates
maxlen=100, # Sequence longer then this get ignored
batch_size=16, # The batch size during training.
valid_batch_size=64, # The batch size used for validation/test set.
dataset='imdb',
# Parameter for extra option
noise_std=0.,
use_dropout=True, # if False slightly faster, but worst test error
# This frequently need a bigger model.
reload_model=True, # Path to a saved model we want to start from.
test_size=-1, # If >0, we keep only this number of test example.
output_file_path='output_file_path'):
# Model options
model_options = locals().copy()
print("model options", model_options)
load_data, prepare_data = get_dataset(dataset)
print('Loading data')
test = load_data(n_words=n_words, maxlen=maxlen)
ydim = len(test[0]) + 1
model_options['ydim'] = ydim
print('Building model')
# This create the initial parameters as numpy ndarrays.
# Dict name (string) -> numpy ndarray
params = init_params(model_options)
if reload_model:
load_params('lstm_model.npz', params)
# This create Theano Shared Variable from the parameters.
# Dict name (string) -> Theano Tensor Shared Variable
# params and tparams have different copy of the weights.
tparams = init_tparams(params)
# use_noise is for dropout
(use_noise, x, mask, y, f_pred_prob, f_pred,
cost) = build_model(tparams, model_options)
#kf_valid = get_minibatches_idx(len(valid[0]), valid_batch_size)
#kf_test = get_minibatches_idx(len(test[0]), valid_batch_size)
#print("%d train examples" % len(train[0]))
#print("%d valid examples" % len(valid[0]))
print("%d test examples" % len(test[0]))
use_noise.set_value(0.)
kf_test = get_minibatches_idx(len(test[0]), valid_batch_size)
#kf_train_sorted = get_minibatches_idx(len(train[0]), batch_size)
#train_probs = pred_error(f_pred, prepare_data, train, kf_train_sorted)
#valid_err = pred_error(f_pred, prepare_data, valid, kf_valid)
#test_probs = pred_probs(f_pred_prob, prepare_data, test, kf_test)
#test_probs = pred_probs(f_pred_prob, prepare_data, test, len(test[0]))
#test_probs = f_pred_prob(test, mask)
test_result = []
ind = 0
triple = []
lastid = '0'
for i in test[0]:
x, mask, y = prepare_data([i], [0], maxlen)
result = f_pred(x, mask)
index_ok = "".join(itertools.chain(*test[2][ind]))
if result == 1:
id = index_ok
if (lastid == id):
triple.append([i[-3], i[-2], i[-1]])
else:
triple = []
if (lastid != '0'):
test_result.append({
"sentenceId": lastid,
"results": triple
})
triple.append([i[-3], i[-2], i[-1]])
lastid = id
#test_result.append(result)
ind += 1
test_result.append({"sentenceId": lastid, "results": triple})
json.dump(test_result, open(output_file_path, 'w'))
#caluclate total profit and loss also
self.total_portfolio_delta = transaction_cost + cost
#todo need to take care that portfolio don't go much negative
self.portfolio += action
return transaction_cost + portfolio_change
def agent_start(self):
stock_data = self.stock_data
action = self.AGENT.agent_start(self.make_input_vector())
reward = self.find_reward(action)
epoch = 0
n_epoch = 100000
while (epoch < n_epoch):
self.stock_data = np.vstack(
[self.stock_data, self.other_stored_data[0]])
self.other_stored_data = np.delete(self.other_stored_data, [0], 0)
self.stock_data = np.delete(self.stock_data, [0], 0)
#we can put conditional here to start taking from API one csv_data finished
action = self.AGENT.agent_step(reward, self.make_input_vector())
print("epoch is: ", epoch)
print("action is: ", action)
print("reward is: ", reward)
reward = self.find_reward(action)
epoch += 1
print("portfolio: ", self.portfolio)
print("total_portfolio_delta: ", self.total_portfolio_delta)
k = PortfolioAgent(prepare_data("CANFINHOM"))
k.agent_start()
#caluclate total profit and loss also
self.total_portfolio_delta = transaction_cost + cost
#todo need to take care that portfolio don't go much negative
self.portfolio += action
return transaction_cost + portfolio_change
def agent_start(self):
stock_data = self.stock_data
action = self.AGENT.agent_start(self.make_input_vector())
reward = self.find_reward(action)
epoch = 0
n_epoch = 100000
while (epoch < n_epoch):
self.stock_data = np.vstack([self.stock_data, self.other_stored_data[0]])
self.other_stored_data = np.delete(self.other_stored_data, [0],0)
self.stock_data = np.delete(self.stock_data, [0], 0)
#we can put conditional here to start taking from API one csv_data finished
action = self.AGENT.agent_step(reward, self.make_input_vector())
print("epoch is: ", epoch)
print("action is: ", action)
print("reward is: ", reward)
reward = self.find_reward(action)
epoch += 1
print("portfolio: ", self.portfolio)
print("total_portfolio_delta: ", self.total_portfolio_delta)
k = PortfolioAgent(prepare_data("CANFINHOM"))
k.agent_start()
# instance_index = 0
# for train_instance in train_items:
# if get_index_pair(train_instance, doc) is None:
# continue
# question_token = nltk.word_tokenize(train_instance['question'])
# question_lower = [token.lower() for token in question_token]
# question_lemma = [lmz(token) for token in question_token]
#
# para_token = nltk.word_tokenize(doc[train_instance['docid']]['text'][train_instance['answer_paragraph']])
# for token_index in range(len(para_token)):
# if para_token[token_index] in question_token:
# exact_match[instance_index, token_index, 0] = 1
# if para_token[token_index].lower() in question_lower:
# exact_match[instance_index, token_index, 1] = 1
# if lmz(para_token[token_index]) in question_lemma:
# exact_match[instance_index, token_index, 2] = 1
# instance_index += 1
#
# print(exact_match)
# np.save('exact_match.npy', exact_match)
from paraRetrival_bigram import *
from prepare_data import *
from train import *
from pred import *
main_process(fileName='testing.json')
prepare_data()
train()
predict()
def train(
file='ad_action_state',
batch_size=32,
test_iter=10000,
save_iter=1000000000,
model_type='DSPN',
seed=3,
shuffle_each_epoch=True,
):
is_shuffle = 'noshuffle_'
if shuffle_each_epoch:
is_shuffle = 'shuffle_'
model_path = model_type + '_best_model/' + is_shuffle + '_' + str(seed)
gpu_options = tf.GPUOptions(allow_growth=True)
with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess:
print("data preparation...")
train_data, test_data = getData(file=file,
batch_size=batch_size,
shuffle_each_epoch=shuffle_each_epoch)
print("data ready.")
n_adgroup, n_member, n_campaign, \
n_item, n_cate, n_commodity, \
n_node, n_effect, n_pos, n_direct, \
n_direct_action, n_pos_action = get_n()
print(get_n())
if model_type == 'MLP':
model = Model_MLP(n_adgroup, n_member, n_campaign, \
n_item, n_cate, n_commodity, \
n_node, n_effect, n_pos, n_direct, \
n_direct_action, n_pos_action, \
EMBEDDING_DIM, ATTENTION_SIZE)
elif model_type == 'PNN':
model = Model_PNN(n_adgroup, n_member, n_campaign, \
n_item, n_cate, n_commodity, \
n_node, n_effect, n_pos, n_direct, \
n_direct_action, n_pos_action, \
EMBEDDING_DIM, ATTENTION_SIZE)
elif model_type == 'WideDeep':
model = Model_WideDeep(n_adgroup, n_member, n_campaign, \
n_item, n_cate, n_commodity, \
n_node, n_effect, n_pos, n_direct, \
n_direct_action, n_pos_action, \
EMBEDDING_DIM, ATTENTION_SIZE)
elif model_type == 'DeepFM':
model = Model_DeepFM(n_adgroup, n_member, n_campaign, \
n_item, n_cate, n_commodity, \
n_node, n_effect, n_pos, n_direct, \
n_direct_action, n_pos_action, \
EMBEDDING_DIM, ATTENTION_SIZE)
elif model_type == 'DIN':
model = Model_DIN(n_adgroup, n_member, n_campaign, \
n_item, n_cate, n_commodity, \
n_node, n_effect, n_pos, n_direct, \
n_direct_action, n_pos_action, \
EMBEDDING_DIM, ATTENTION_SIZE)
elif model_type == 'RNN':
model = Model_RNN(n_adgroup, n_member, n_campaign, \
n_item, n_cate, n_commodity, \
n_node, n_effect, n_pos, n_direct, \
n_direct_action, n_pos_action, \
EMBEDDING_DIM, ATTENTION_SIZE)
elif model_type == 'biRNN':
model = Model_biRNN(n_adgroup, n_member, n_campaign, \
n_item, n_cate, n_commodity, \
n_node, n_effect, n_pos, n_direct, \
n_direct_action, n_pos_action, \
EMBEDDING_DIM, ATTENTION_SIZE)
elif model_type == 'DSPN':
model = Model_DSPN(n_adgroup, n_member, n_campaign, \
n_item, n_cate, n_commodity, \
n_node, n_effect, n_pos, n_direct, \
n_direct_action, n_pos_action, \
EMBEDDING_DIM, ATTENTION_SIZE)
elif model_type == 'DSPN_RNN':
model = Model_DSPN_RNN(n_adgroup, n_member, n_campaign, \
n_item, n_cate, n_commodity, \
n_node, n_effect, n_pos, n_direct, \
n_direct_action, n_pos_action, \
EMBEDDING_DIM, ATTENTION_SIZE)
elif model_type == 'DSPN_MLP':
model = Model_DSPN_MLP(n_adgroup, n_member, n_campaign, \
n_item, n_cate, n_commodity, \
n_node, n_effect, n_pos, n_direct, \
n_direct_action, n_pos_action, \
EMBEDDING_DIM, ATTENTION_SIZE)
elif model_type == 'DSPN_no_att':
model = Model_DSPN_no_att(n_adgroup, n_member, n_campaign, \
n_item, n_cate, n_commodity, \
n_node, n_effect, n_pos, n_direct, \
n_direct_action, n_pos_action, \
EMBEDDING_DIM, ATTENTION_SIZE)
elif model_type == 'DSPN_noID':
model = Model_DSPN_noID(n_adgroup, n_member, n_campaign, \
n_item, n_cate, n_commodity, \
n_node, n_effect, n_pos, n_direct, \
n_direct_action, n_pos_action, \
EMBEDDING_DIM, ATTENTION_SIZE)
elif model_type == 'DSPN_noAction':
model = Model_DSPN_noAction(n_adgroup, n_member, n_campaign, \
n_item, n_cate, n_commodity, \
n_node, n_effect, n_pos, n_direct, \
n_direct_action, n_pos_action, \
EMBEDDING_DIM, ATTENTION_SIZE)
elif model_type == 'DSPN_noReport':
model = Model_DSPN_noReport(n_adgroup, n_member, n_campaign, \
n_item, n_cate, n_commodity, \
n_node, n_effect, n_pos, n_direct, \
n_direct_action, n_pos_action, \
EMBEDDING_DIM, ATTENTION_SIZE)
elif model_type == 'DSPN_no_intent':
model = Model_DSPN_no_intent(n_adgroup, n_member, n_campaign, \
n_item, n_cate, n_commodity, \
n_node, n_effect, n_pos, n_direct, \
n_direct_action, n_pos_action, \
EMBEDDING_DIM, ATTENTION_SIZE)
elif model_type == 'DSPN_ID':
model = Model_DSPN_ID(n_adgroup, n_member, n_campaign, \
n_item, n_cate, n_commodity, \
n_node, n_effect, n_pos, n_direct, \
n_direct_action, n_pos_action, \
EMBEDDING_DIM, ATTENTION_SIZE)
elif model_type == 'DSPN_Action':
model = Model_DSPN_Action(n_adgroup, n_member, n_campaign, \
n_item, n_cate, n_commodity, \
n_node, n_effect, n_pos, n_direct, \
n_direct_action, n_pos_action, \
EMBEDDING_DIM, ATTENTION_SIZE)
elif model_type == 'DSPN_Report':
model = Model_DSPN_Report(n_adgroup, n_member, n_campaign, \
n_item, n_cate, n_commodity, \
n_node, n_effect, n_pos, n_direct, \
n_direct_action, n_pos_action, \
EMBEDDING_DIM, ATTENTION_SIZE)
elif model_type == 'DSPN_attention':
model = Model_DSPN_attention(n_adgroup, n_member, n_campaign, \
n_item, n_cate, n_commodity, \
n_node, n_effect, n_pos, n_direct, \
n_direct_action, n_pos_action, \
EMBEDDING_DIM, ATTENTION_SIZE)
elif model_type == 'DSPN_no_attention':
model = Model_DSPN_no_attention(n_adgroup, n_member, n_campaign, \
n_item, n_cate, n_commodity, \
n_node, n_effect, n_pos, n_direct, \
n_direct_action, n_pos_action, \
EMBEDDING_DIM, ATTENTION_SIZE)
else:
print("Invalid model_type: %s" % (model_type))
return
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
t_auc, t_loss, t_accuracy = eval(sess, test_data, model, model_path)
sys.stdout.flush()
print('test_auc: %.4f ---- test_loss: %.4f ---- test_accuracy: %.4f' %
(t_auc, t_loss, t_accuracy))
sys.stdout.flush()
lr = 0.001
loss_sum = 0.0
accuracy_sum = 0.0
eval_sum = 0.0
path = 'result_plot_' + model_type + '_batch_size_' + str(
batch_size) + '_iter_num_' + str(test_iter)
global best_auc
global best_acc
iter = 0
for itr in range(3):
print("start iteration %d." % (itr + 1))
for raw_data in train_data:
iter += 1
y, y_, adgroup, member, campaign, item, item_price, cate, commodity, node, \
effect_id, effect, effect_mask, \
pos_id, pos, pos_mask, \
direct_id, direct, direct_mask, \
direct_action_type, direct_action_value, direct_action_mask, \
pos_action_type, pos_action_value, pos_action_mask = prepare_data(raw_data)
loss, acc = \
model.train(sess, \
[lr, y, y_, adgroup, member, campaign, item, item_price, cate, commodity, node, \
effect_id, effect, effect_mask, \
pos_id, pos, pos_mask, \
direct_id, direct, direct_mask, \
direct_action_type, direct_action_value, direct_action_mask, \
pos_action_type, pos_action_value, pos_action_mask])
loss_sum += loss
accuracy_sum += acc
sys.stdout.flush()
if iter % test_iter == 0:
t_auc, t_loss, t_accuracy = eval(sess, test_data, model,
model_path)
print('iter: %d ----> train_loss: %.4f ---- train_accuracy: %.4f' % \
(iter, loss_sum / test_iter, accuracy_sum / test_iter))
print('test_auc: %.4f ----test_loss: %.4f ---- test_accuracy: %.4f' % \
(t_auc, t_loss, t_accuracy))
print("best_auc = %f" % best_auc)
print("best_acc = %f" % best_acc)
loss_sum = 0.0
accuracy_sum = 0.0
aux_loss_sum = 0.0
test_iterations.append(len(test_iterations))
test_auc_.append(t_auc)
test_loss_.append(t_loss)
test_accuracy_.append(t_accuracy)
if iter % save_iter == 0:
print('save model iter: %d' % iter)
model.save(sess, model_path + "--" + str(iter))
lr *= 0.5
draw(path)
print(model_path)
print(path)
print(test_iterations)
print(test_auc_)
print(test_loss_)
print(test_accuracy_)
print('done.')
t_auc, t_loss, t_accuracy = eval(sess, test_data, model, model_path)
print('test_auc: %.4f ----test_loss: %.4f ---- test_accuracy: %.4f' % \
(t_auc, t_loss, t_accuracy))
print("best_auc = %f" % best_auc)
print("best_acc = %f" % best_acc)
from prepare_data import *
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense
import os
#INPUT OF 385
#OUTPUT OF 4096
#get data
moves = []
boards = []
for filename in os.listdir("stored_games/"):
try:
_moves, _boards = prepare_data(f"stored_games/{filename}")
moves = moves + _moves
boards = boards + _boards
except Exception as e:
print(
"*******************************************************************************************************************************"
)
print(e)
X, Y = get_XY(moves, boards)
print(X)
print(Y)
model = Sequential()
model.add(Dense(385, activation="relu"))
model.add(Dense(500, activation="relu"))
def main(args):
p_data, q_data, train_triples, test_triples = prepare_data(args.post_data_tsvfile, args.qa_data_tsvfile, \
args.train_ids_file, args.test_ids_file, args.sim_ques_fname)
pretrained_emb = load_pretrained_emb(args.word_vec_fname, p_data)
#N = int(len(triples)*0.8)
#train_triples = triples[:N]
#test_triples = triples[N:]
# Initialize models
#p_encoder = EncoderAvgEmb(pretrained_emb)
p_encoder = EncoderRNN(p_data.n_words,
hidden_size,
n_layers,
dropout=dropout)
q_encoder = EncoderRNN(q_data.n_words,
hidden_size,
n_layers,
dropout=dropout)
decoder = AttnDecoderRNN(attn_model, hidden_size, q_data.n_words, n_layers)
# Initialize optimizers and criterion
p_encoder_optimizer = optim.Adam(p_encoder.parameters(), lr=learning_rate)
q_encoder_optimizer = optim.Adam(q_encoder.parameters(), lr=learning_rate)
decoder_optimizer = optim.Adam(decoder.parameters(),
lr=learning_rate * decoder_learning_ratio)
criterion = nn.CrossEntropyLoss()
# Move models to GPU
if USE_CUDA:
p_encoder.cuda()
q_encoder.cuda()
decoder.cuda()
import sconce
job = sconce.Job(
'seq2seq-translate', {
'attn_model': attn_model,
'n_layers': n_layers,
'dropout': dropout,
'hidden_size': hidden_size,
'learning_rate': learning_rate,
'clip': clip,
'teacher_forcing_ratio': teacher_forcing_ratio,
'decoder_learning_ratio': decoder_learning_ratio,
})
job.plot_every = plot_every
job.log_every = print_every
# Keep track of time elapsed and running averages
start = time.time()
plot_losses = []
print_loss_total = 0 # Reset every print_every
plot_loss_total = 0 # Reset every plot_every
ecs = []
dcs = []
eca = 0
dca = 0
epoch = 0.0
print 'No. of train_triples %d' % len(train_triples)
print 'No. of test_triples %d' % len(test_triples)
while epoch < n_epochs:
epoch += 1
# Get training data for this cycle
p_input_batches, p_input_lengths, q_input_batches, q_input_lengths, target_batches, target_lengths = \
random_batch(batch_size, p_data, q_data, train_triples)
# Run the train function
loss, ec, dc = train(p_input_batches, p_input_lengths, q_input_batches,
q_input_lengths, target_batches, target_lengths,
p_encoder, q_encoder, decoder,
p_encoder_optimizer, q_encoder_optimizer,
decoder_optimizer, criterion)
# Keep track of loss
print_loss_total += loss
plot_loss_total += loss
eca += ec
dca += dc
job.record(epoch, loss)
if epoch % print_every == 0:
print_loss_avg = print_loss_total / print_every
print_loss_total = 0
print_summary = '%s (%d %d%%) %.4f' % (time_since(
start, epoch / n_epochs), epoch, epoch / n_epochs * 100,
print_loss_avg)
print(print_summary)
if epoch % evaluate_every == 0:
evaluate_randomly(p_data, q_data, test_triples, p_encoder,
q_encoder, decoder)
if epoch % plot_every == 0:
plot_loss_avg = plot_loss_total / plot_every
plot_losses.append(plot_loss_avg)
plot_loss_total = 0
# TODO: Running average helper
ecs.append(eca / plot_every)
dcs.append(dca / plot_every)
ecs_win = 'encoder grad (%s)' % hostname
dcs_win = 'decoder grad (%s)' % hostname
#vis.line(np.array(ecs), win=ecs_win, opts={'title': ecs_win})
#vis.line(np.array(dcs), win=dcs_win, opts={'title': dcs_win})
eca = 0
dca = 0
def main():
asd, asg, amin, d = prepare_data(f_cd, f_yd, f_ctm, m, b, a, start, stop, step)
plot_graph(asd, asg, amin, d, f_cd, f_yd, m, b, a)
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
'--pretrained',
default=False,
type=bool,
help='Whether the model uses pretrained sentence embeddings or not')
parser.add_argument('--data_path',
default='data/text/',
type=str,
help='Folder to store the annotated text files')
parser.add_argument(
'--save_path',
default='saved/',
type=str,
help='Folder where predictions and models will be saved')
parser.add_argument('--cat_path',
default='categories.txt',
type=str,
help='Path to file containing category details')
parser.add_argument('--dataset_size',
default=50,
type=int,
help='Total no. of docs')
parser.add_argument('--num_folds',
default=5,
type=int,
help='No. of folds to divide the dataset into')
parser.add_argument('--device',
default='cuda',
type=str,
help='cuda / cpu')
parser.add_argument('--batch_size', default=32, type=int)
parser.add_argument(
'--print_every',
default=10,
type=int,
help=
'Epoch interval after which validation macro f1 and loss will be printed'
)
parser.add_argument('--lr', default=0.01, type=float, help='Learning Rate')
parser.add_argument('--reg',
default=0,
type=float,
help='L2 Regularization')
parser.add_argument('--emb_dim',
default=200,
type=int,
help='Sentence embedding dimension')
parser.add_argument(
'--word_emb_dim',
default=100,
type=int,
help='Word embedding dimension, applicable only if pretrained = False')
parser.add_argument('--epochs', default=300, type=int)
parser.add_argument(
'--val_fold',
default='cross',
type=str,
help=
'Fold number to be used as validation, use cross for num_folds cross validation'
)
args = parser.parse_args()
print('\nPreparing data ...', end=' ')
idx_order = prepare_folds(args)
x, y, word2idx, tag2idx = prepare_data(idx_order, args)
print('Done')
print('Vocabulary size:', len(word2idx))
print('#Tags:', len(tag2idx))
# Dump word2idx and tag2idx
with open(args.save_path + 'word2idx.json', 'w') as fp:
json.dump(word2idx, fp)
with open(args.save_path + 'tag2idx.json', 'w') as fp:
json.dump(tag2idx, fp)
if args.val_fold == 'cross':
print('\nCross-validation\n')
for f in range(args.num_folds):
print('\nInitializing model ...', end=' ')
model = Hier_LSTM_CRF_Classifier(len(tag2idx),
args.emb_dim,
tag2idx['<start>'],
tag2idx['<end>'],
tag2idx['<pad>'],
vocab_size=len(word2idx),
word_emb_dim=args.word_emb_dim,
pretrained=args.pretrained,
device=args.device).to(
args.device)
print('Done')
print('\nEvaluating on fold', f, '...')
learn(model, x, y, tag2idx, f, args)
else:
print('\nInitializing model ...', end=' ')
model = Hier_LSTM_CRF_Classifier(len(tag2idx),
args.emb_dim,
tag2idx['<start>'],
tag2idx['<end>'],
tag2idx['<pad>'],
vocab_size=len(word2idx),
word_emb_dim=args.word_emb_dim,
pretrained=args.pretrained,
device=args.device).to(args.device)
print('Done')
print('\nEvaluating on fold', args.val_fold, '...')
learn(model, x, y, tag2idx, int(args.val_fold), args)
def get_test_data(filename, max_length=82):
data, vocab_size, eos_id = ptb_raw_data_file(filename)
main_data, max_length = prepare_data(data, eos_id, vocab_size, max_length)
return main_data
os.mkdir(str(Path(SaveDir, 'crops')))
os.mkdir(str(
Path(SaveDir,
'fullCT_original'))) #Testing without morphological operations
os.mkdir(str(
Path(SaveDir, 'fullCT_morph' +
str(thresholdI)))) #Testing with morphological operations
print(
'The results of the training, validation and testing will be saved to:'
+ str(SaveDir))
#############################################Downloading and unzipping the dataset######################################
dataset_zip_dir = 'computed-tomography-images-for-intracranial-hemorrhage-detection-and-segmentation-1.3.1.zip'
crossvalid_dir = 'DataV1'
prepare_data(dataset_zip_dir, crossvalid_dir, numSubj, imageLen, windowLen,
strideLen, num_Moves, window_specs
) #preparing the data and saving it to ICH_DataSegmentV1.pkl
# Loading full image mask from the crops predictions
with open(str(Path(crossvalid_dir, 'ICH_DataSegmentV1.pkl')),
'rb') as Dataset1:
[
hemorrhageDiagnosisArray, AllCTscans, testMasks,
subject_nums_shaffled
] = pickle.load(Dataset1)
del AllCTscans
testMasks = np.uint8(testMasks)
testMasksAvg = np.where(
np.sum(np.sum(testMasks, axis=1), axis=1) > detectionSen, 1, 0) #
testPredictions = np.zeros((testMasks.shape[0], imageLen, imageLen),
dtype=np.uint8) #predicted segmentation
model = transfer_DenseNet()
model.load_state_dict(torch.load('model/model.pkl'))
model.eval()
model.cuda()
test_dataloader = get_loader(X_test, y_test, batch_size=128, shuffle=False)
all_equal = 0.0
all_count = X_test.shape[0]
for i, (input, target) in enumerate(test_dataloader):
model.train(False)
if use_cuda:
input = Variable(input, volatile=True).cuda()
target = Variable(target.squeeze(), volatile=True).cuda()
else:
input = Variable(input, volatile=True)
target = Variable(target.squeeze(), volatile=True)
output = model(input)
_, pred = torch.max(output.data, 1)
all_equal += torch.sum(pred == target.data)
print('-' * 20)
print('test:\n')
print( all_equal / all_count)
print('-' * 20)
if __name__ == "__main__":
data = prepare_data('data/train.p', 'data/valid.p', 'data/test.p')
sample_test(data[4], data[5], True)