def main():
# Utility object for reading, writing parameters, etc.
utils = UTILS()
# Reading parameters from para.dat file
parameters = utils.read_parameter_file()
# Command line specified parameters overide parameter file values
utils.read_command_line_arg(parameters, sys.argv)
parameters_nonint = parameters.copy()
parameters_nonint['J'] = 0.0
# Printing parameters for user
utils.print_parameters(parameters)
# Defining Hamiltonian
H = HAMILTONIAN(**parameters)
H_nonint = HAMILTONIAN(**parameters_nonint)
# Defines the model, and precomputes evolution matrices given set of states
model = MODEL(H, parameters)
model_nonint = MODEL(H_nonint, parameters_nonint)
# evolve with non-int H but starting and targeting interacting states
model_nonint.psi_i = model.psi_i.copy()
model_nonint.psi_target = model.psi_target.copy()
model_nonint.H_target = model.H_target.copy()
# model_nonint.param=model.param.copy()
model = model_nonint
parameters = parameters_nonint
#print( abs(model.psi_i.T.dot(model.psi_target))**2 )
#exit()
root = 'data/non-int-evo/'
#root='data/data_non-int/'
# Run simulated annealing
if parameters['task'] == 'SA':
print("Simulated annealing")
run_SA(parameters, model, utils, root)
elif parameters['task'] == 'GB':
print("Gibbs sampling")
run_GS(parameters, model)
elif parameters['task'] == 'SD' or parameters['task'] == 'SD2':
print("Stochastic descent")
run_SD(parameters, model, utils, root)
elif parameters['task'] == 'ES':
print("Exact spectrum")
run_ES(parameters, model, utils, root)
elif parameters['task'] == 'SASD':
print("Simulating annealing followed by stochastic descent")
run_SA(parameters, model, utils, root)
exit()
def _load_model(self, network):
"""Model Loader Module
Load model from `MODEL`
"""
with tf.variable_scope('Model'):
self.model = MODEL(**config_model)
self.model.build_model(network=network)
def test_one_dataset(params, file_name, test_q_data, test_qa_data):
print("\n\nStart testing ......................")
g_model = MODEL(n_question=params.n_question,
seqlen=params.seqlen,
batch_size=params.batch_size,
q_embed_dim=params.q_embed_dim,
qa_embed_dim=params.qa_embed_dim,
memory_size=params.memory_size,
memory_key_state_dim=params.memory_key_state_dim,
memory_value_state_dim=params.memory_value_state_dim,
final_fc_dim=params.final_fc_dim)
# create a module by given a Symbol
test_net = mx.mod.Module(symbol=g_model.sym_gen(),
data_names=['q_data', 'qa_data'],
label_names=['target'],
context=params.ctx)
# cresate memory by given input shapes
test_net.bind(data_shapes=[
mx.io.DataDesc(name='q_data', shape=(params.seqlen, params.batch_size), layout='SN'),
mx.io.DataDesc(name='qa_data', shape=(params.seqlen, params.batch_size), layout='SN')],
label_shapes=[mx.io.DataDesc(name='target', shape=(params.seqlen, params.batch_size), layout='SN')])
arg_params, aux_params = load_params(prefix=os.path.join('model', params.load, file_name),
epoch=100)
test_net.init_params(arg_params=arg_params, aux_params=aux_params,
allow_missing=False)
test_loss, test_accuracy, test_auc = test(test_net, params, test_q_data, test_qa_data, label='Test')
log_info = "\ntest_auc:\t{}\ntest_accuracy:\t{}\ntest_loss:\t{}\n".format(test_auc, test_accuracy, test_loss)
print(log_info)
f_save_log = open(os.path.join('result', params.save, file_name), 'a')
f_save_log.write(log_info)
def main():
# Utility object for reading, writing parameters, etc.
utils = UTILS()
# Reading parameters from para.dat file
parameters = utils.read_parameter_file(file="../para.dat")
parameters['root'] = "../data/"
utils.read_command_line_arg(parameters,sys.argv)
# Printing parameters for user
utils.print_parameters(parameters)
# Defining Hamiltonian
H = HAMILTONIAN(**parameters)
# Defines the model, and precomputes
model = MODEL(H, parameters)
# save interacting states
if abs(parameters['J'] - 1.0) < 0.001 :
with open('psi_L=2_J=1.pkl','wb') as f :
pickle.dump([model.psi_i, model.psi_target], f)
# load interacting states
if abs(parameters['J']) < 0.0001:
with open('psi_L=2_J=1.pkl','rb') as f :
model.psi_i, model.psi_target = pickle.load(f)
print(model.psi_i)
def main(argv):
manager = DataManager(FLAGS.data)
manager.load()
sess = tf.Session()
normaliser = (FLAGS.latent_size / 10) * ((64 * 64) / manager.input_size)
model = MODEL(latent_size=FLAGS.latent_size,
gamma=normaliser * FLAGS.gamma,
capacity_limit=FLAGS.capacity_limit,
capacity_change_duration=FLAGS.capacity_change_duration,
learning_rate=FLAGS.learning_rate,
n_channels=manager.n_channels)
sess.run(tf.global_variables_initializer())
saver = load_checkpoints(sess)
if FLAGS.training:
# Train
train(sess, model, manager, saver)
else:
reconstruct_check_images = manager.get_random_images(10)
# Image reconstruction check
reconstruct_check(sess, model, reconstruct_check_images)
# Disentangle check
disentangle_check(sess, model, manager)
def main(_):
if not os.path.exists(conf["checkpoint_dir"]):
os.makedirs(conf["checkpoint_dir"])
if not os.path.exists(conf["output_dir"]):
os.makedirs(conf["output_dir"])
with tf.Session() as sess:
srcnn = MODEL(sess,
mode=conf["mode"],
epoch=conf["epoch"],
batch_size=conf["batch_size"],
image_size=conf["image_size"],
label_size=conf["label_size"],
learning_rate=conf["learning_rate"],
color_dim=conf["color_dim"],
scale=conf["scale"],
train_extract_stride=conf["train_extract_stride"],
test_extract_stride=conf["test_extract_stride"],
checkpoint_dir=conf["checkpoint_dir"],
log_dir=conf["log_dir"],
output_dir=conf["output_dir"],
train_dir=conf["train_dir"],
test_dir=conf["test_dir"],
h5_dir=conf["h5_dir"],
train_h5_name=conf["train_h5_name"],
test_h5_name=conf["test_h5_name"],
ckpt_name=conf["ckpt_name"],
is_train=conf["is_train"],
model_ticket=conf["model_ticket"],
curr_epoch=conf["curr_epoch"])
if conf["is_train"]:
srcnn.train()
def test_one_dataset(params, file_name, test_q_data, test_qa_data, best_epoch):
print("\n\nStart testing ......................\n Best epoch:", best_epoch)
val_best_epochs.append(best_epoch)
g_model = MODEL(n_question=params.n_question,
seqlen=params.seqlen,
batch_size=params.batch_size,
q_embed_dim=params.q_embed_dim,
qa_embed_dim=params.qa_embed_dim,
memory_size=params.memory_size,
memory_key_state_dim=params.memory_key_state_dim,
memory_value_state_dim=params.memory_value_state_dim,
final_fc_dim=params.final_fc_dim)
# create a module by given a Symbol
test_net = mx.mod.Module(symbol=g_model.sym_gen(),
data_names=['q_data', 'qa_data'],
label_names=['target'],
context=params.ctx)
# cresate memory by given input shapes
test_net.bind(data_shapes=[
mx.io.DataDesc(name='q_data', shape=(params.seqlen, params.batch_size), layout='SN'),
mx.io.DataDesc(name='qa_data', shape=(params.seqlen, params.batch_size), layout='SN')],
label_shapes=[mx.io.DataDesc(name='target', shape=(params.seqlen, params.batch_size), layout='SN')])
arg_params, aux_params = load_params(prefix=os.path.join('model', params.load, file_name),
epoch=best_epoch)
test_net.init_params(arg_params=arg_params, aux_params=aux_params,
allow_missing=False)
test_loss, test_accuracy, test_auc = test(test_net, params, test_q_data, test_qa_data, label='Test')
print("\ntest_auc\t", test_auc)
print("test_accuracy\t", test_accuracy)
print("test_loss\t", test_loss)
val_auc_acc.append(test_auc)
val_accuracy.append(test_accuracy)
val_loss.append(test_loss)
def test_one_dataset(params, file_name, test_q_data, test_qa_data, test_tf_data, best_epoch, user_id):
# print("\n\nStart testing ......................\n best_epoch:", best_epoch)
g_model = MODEL(n_question=params.n_question,
seqlen=params.seqlen,
batch_size=params.batch_size,
q_embed_dim=params.q_embed_dim,
qa_embed_dim=params.qa_embed_dim,
memory_size=params.memory_size,
memory_key_state_dim=params.memory_key_state_dim,
memory_value_state_dim=params.memory_value_state_dim,
final_fc_dim=params.final_fc_dim)
# create a module by given a Symbol
test_net = mx.mod.Module(symbol=g_model.sym_gen(),
data_names=['q_data', 'qa_data'],
label_names=['target'],
context=params.ctx)
# create memory by given input shapes
test_net.bind(data_shapes=[
mx.io.DataDesc(name='q_data', shape=(params.seqlen, params.batch_size), layout='SN'),
mx.io.DataDesc(name='qa_data', shape=(params.seqlen, params.batch_size), layout='SN')],
label_shapes=[mx.io.DataDesc(name='target', shape=(params.seqlen, params.batch_size), layout='SN')])
arg_params, aux_params = load_params(prefix=os.path.join('model', params.load, file_name), epoch=best_epoch)
test_net.init_params(arg_params=arg_params, aux_params=aux_params, allow_missing=False)
pred_list, target_list = run.test(test_net, params, test_q_data, test_qa_data, test_tf_data, label='Test')
return pred_list, target_list
def __init__(self):
self.model = MODEL()
self.transform = transforms.Compose([
transforms.Resize((100, 100), interpolation=3),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
])
self.optimizer = torch.optim.Adam(self.model.parameters(), lr=0.0001)
self.logsoftmax = nn.LogSoftmax(dim=1)
def excitations(Fspectrum, parameters):
hopt_n10 = np.argmax(Fspectrum)
print('L', 'T', 'dt', sep='\t')
print(parameters['L'], parameters['T'], parameters['dt'], sep='\t')
print('Optimal fidelity : %.8f' % Fspectrum[hopt_n10])
# Defining Hamiltonian
H = HAMILTONIAN(**parameters)
# Defines the model, and precomputes evolution matrices given set of states
model = MODEL(H, parameters)
n_step = parameters['n_step']
b2_array = lambda n10: np.array(list(np.binary_repr(n10, width=n_step)),
dtype=np.int)
exc_dict = {} # sorting out excitations by they m and m_z
hopt = b2_array(hopt_n10) # optimal protcol in array base 2
mag_opt = np.sum(
hopt
) # same as spin 1/2 ... -> excitations are integer excitations, called spin-1 and spin-2 excitations
for i in range(n_step): # spin 1 excitations
hopt[i] ^= 1
mz = np.sum(hopt) - mag_opt
if (1, mz) not in exc_dict.keys():
exc_dict[(1, mz)] = []
exc_dict[(1, mz)].append(bool2int(hopt))
hopt[i] ^= 1
# --------------------------------
for i in range(n_step): # spin 2 excitations
for j in range(i):
hopt[i] ^= 1
hopt[j] ^= 1
mz = np.sum(hopt) - mag_opt
if (2, mz) not in exc_dict.keys():
exc_dict[(2, mz)] = []
exc_dict[(2, mz)].append(bool2int(hopt))
hopt[i] ^= 1
hopt[j] ^= 1
return exc_dict
def run():
args = ARGPARSE()
if args.is_list_mhcs() == True:
if not args.is_no_trim_mhc_flag_set():
for key, value in hla_sequences.items():
print(key)
else:
for key, value in hla_sequences_180.items():
print(key)
else:
# Load Rosetta database files
init(options='')
modeller = MODEL(args)
modeller.model_mhc_for_each_peptide_beta2m_tcr_chaperone()
def main():
# Utility object for reading, writing parameters, etc.
utils = UTILS()
# Reading parameters from para.dat file
parameters = utils.read_parameter_file()
# Command line specified parameters overide parameter file values
utils.read_command_line_arg(parameters, sys.argv)
# Printing parameters for user
utils.print_parameters(parameters)
# Defining Hamiltonian
H = HAMILTONIAN(**parameters)
# Defines the model, and precomputes evolution matrices given set of states
model = MODEL(H, parameters)
#root='data/data_ES/'
#root='data/data_SD/'
root = 'data/data_GRAPE/'
# Run simulated annealing
if parameters['task'] == 'SA':
print("Simulated annealing")
run_SA(parameters, model, utils, root)
elif parameters['task'] == 'GB':
print("Gibbs sampling")
run_GS(parameters, model)
elif parameters['task'] == 'SD' or parameters['task'] == 'SD2':
print("Stochastic descent")
run_SD(parameters, model, utils, root)
elif parameters['task'] == 'GRAPE':
print("GRAPE")
run_GRAPE(parameters, model, utils, root)
elif parameters['task'] == 'ES':
print("Exact spectrum")
run_ES(parameters, model, utils, root)
elif parameters['task'] == 'SASD':
print("Simulating annealing followed by stochastic descent")
run_SA(parameters, model, utils, root)
exit()
def _init_model(args, data_info):
n_user = data_info[3]
n_item = data_info[4]
n_relation = data_info[5]
n_entity = data_info[6]
model = MODEL(args, n_user, n_item, n_relation, n_entity)
if args.use_cuda:
model.cuda()
optimizer_kge = torch.optim.Adam(
filter(lambda p: p.requires_grad, model.parameters()),
lr=args.lr_kge,
weight_decay=args.l2_weight,
)
optimizer_cf = torch.optim.Adam(
filter(lambda p: p.requires_grad, model.parameters()),
lr=args.lr_cf,
weight_decay=args.l2_weight,
)
return model, optimizer_kge, optimizer_cf
def quick_setup(self, argv=[], file='para.dat'):
argv_tmp = ['myfile.txt'] + argv
from Hamiltonian import HAMILTONIAN
from model import MODEL
# Reading parameters from para.dat file
parameters = self.read_parameter_file(file=file)
# Command line specified parameters overide parameter file values
self.read_command_line_arg(parameters, argv_tmp)
# Printing parameters for user
print(parameters)
#utils.print_parameters(parameters)
# Defining Hamiltonian
H = HAMILTONIAN(**parameters)
# Defines the model, and precomputes evolution matrices given set of states
return MODEL(H, parameters)
def run_cso_classifier(paper,
modules="both",
enhancement="first",
explanation=False):
if modules not in ["syntactic", "semantic", "both"]:
raise ValueError(
"Error: Field modules must be 'syntactic', 'semantic' or 'both'")
if enhancement not in ["first", "all", "no"]:
raise ValueError(
"Error: Field enhances must be 'first', 'all' or 'no'")
if type(explanation) != bool:
raise ValueError(
"Error: Explanation must be set to either True or False")
# Loading ontology and model
cso = CSO()
model = MODEL()
t_paper = Paper(paper, modules)
result = Result(explanation)
# Passing parameters to the two classes (synt and sema) and actioning classifiers
if modules == 'syntactic' or modules == 'both':
synt_module = synt(cso, t_paper)
result.set_syntactic(synt_module.classify_syntactic())
if explanation:
result.dump_temporary_explanation(synt_module.get_explanation())
if modules == 'semantic' or modules == 'both':
sema_module = sema(model, cso, t_paper)
result.set_semantic(sema_module.classify_semantic())
if explanation:
result.dump_temporary_explanation(sema_module.get_explanation())
result.set_enhanced(
cso.climb_ontology(getattr(result, "union"), enhancement))
return result.get_dict()
def main():
# Reading parameters from para.dat file
parameters = utils.read_parameter_file()
# Printing parameters for user
utils.print_parameters(parameters)
# Defining Hamiltonian
H = HAMILTONIAN(**parameters)
# Defines the model, and precomputes evolution matrices given set of states
model = MODEL(H, parameters)
L = 6
T = 0.1
n_step = 28
param = {'L' : L, 'T': T, 'n_step': n_step}
file_name = make_file_name(param, root= "/projectnb/fheating/SGD/ES/dynamicQL/SA/ES/data/")
with open(file_name, 'rb') as f:
fidelities=pickle.load(f)
nfid=fidelities.shape[0]
fid_and_energy=np.empty((nfid,2),dtype=np.float)
for i,f in zip(range(nfid),fidelity):
if i%10000 == 0: print(i)
model.update_protocol(b2_array(i, w = 28))
psi = model.compute_evolved_state()
fid_and_energy[i][0]=model.compute_fidelity(psi_evolve = psi)
fid_and_energy[i][1]=model.compute_energy(psi_evolve = psi)
print(fid_and_energy[0],'\t',f)
break
with open("ES_L-06_T-0.500_n_step-28-test.pkl", ‘wb’) as f:
fidelities=pickle.dump(fid_and_energy,f, protocol=4)
def train_one_dataset(params, file_name, train_q_data, train_qa_data,
valid_q_data, valid_qa_data):
### ================================== model initialization ==================================
g_model = MODEL(n_question=params.n_question,
seqlen=params.seqlen,
batch_size=params.batch_size,
q_embed_dim=params.q_embed_dim,
qa_embed_dim=params.qa_embed_dim,
memory_size=params.memory_size,
memory_key_state_dim=params.memory_key_state_dim,
memory_value_state_dim=params.memory_value_state_dim,
final_fc_dim=params.final_fc_dim)
# create a module by given a Symbol
net = mx.mod.Module(symbol=g_model.sym_gen(),
data_names=['q_data', 'qa_data'],
label_names=['target'],
context=params.ctx)
# create memory by given input shapes
net.bind(data_shapes=[
mx.io.DataDesc(name='q_data',
shape=(params.seqlen, params.batch_size),
layout='SN'),
mx.io.DataDesc(name='qa_data',
shape=(params.seqlen, params.batch_size),
layout='SN')
],
label_shapes=[
mx.io.DataDesc(name='target',
shape=(params.seqlen, params.batch_size),
layout='SN')
])
# initial parameters with the default random initializer
net.init_params(initializer=mx.init.Normal(sigma=params.init_std))
# decay learning rate in the lr_scheduler
lr_scheduler = mx.lr_scheduler.FactorScheduler(
step=20 * (train_q_data.shape[0] / params.batch_size),
factor=0.667,
stop_factor_lr=1e-5)
net.init_optimizer(optimizer='sgd',
optimizer_params={
'learning_rate': params.lr,
'momentum': params.momentum,
'lr_scheduler': lr_scheduler
})
for parameters in net.get_params()[0]:
print parameters, net.get_params()[0][parameters].asnumpy().shape
print "\n"
### ================================== start training ==================================
all_train_loss = {}
all_train_accuracy = {}
all_train_auc = {}
all_valid_loss = {}
all_valid_accuracy = {}
all_valid_auc = {}
best_valid_auc = 0
for idx in xrange(params.max_iter):
train_loss, train_accuracy, train_auc = train(net,
params,
train_q_data,
train_qa_data,
label='Train')
valid_loss, valid_accuracy, valid_auc = test(net,
params,
valid_q_data,
valid_qa_data,
label='Valid')
print 'epoch', idx + 1
print "valid_auc\t", valid_auc, "\ttrain_auc\t", train_auc
print "valid_accuracy\t", valid_accuracy, "\ttrain_accuracy\t", train_accuracy
print "valid_loss\t", valid_loss, "\ttrain_loss\t", train_loss
net.save_checkpoint(prefix=os.path.join('model', params.save,
file_name),
epoch=idx + 1)
# output the epoch with the best validation auc
if valid_auc > best_valid_auc:
best_valid_auc = valid_auc
best_epoch = idx + 1
f_save_log = open(os.path.join('result', params.save, file_name), 'w')
f_save_log.write("valid_auc:\n" + str(all_valid_auc) + "\n\n")
f_save_log.write("train_auc:\n" + str(all_train_auc) + "\n\n")
f_save_log.write("valid_loss:\n" + str(all_valid_loss) + "\n\n")
f_save_log.write("train_loss:\n" + str(all_train_loss) + "\n\n")
f_save_log.write("valid_accuracy:\n" + str(all_valid_accuracy) + "\n\n")
f_save_log.write("train_accuracy:\n" + str(all_train_accuracy) + "\n\n")
f_save_log.close()
return best_epoch
k += 1
plt.savefig(filename, bbox_inches='tight')
plt.close()
def generate_figure3(model, filename):
pass
if __name__ == '__main__':
# parse arguments
args = parse_args()
# instanciates the model
model = MODEL(args, test_only=True)
# trains the model
model.load()
# creates the figure directory
file_dir = os.path.join("figures", model.dataset, model.model_type)
if not os.path.exists(file_dir):
os.makedirs(file_dir)
# sets the random seed for reproducibility
torch.manual_seed(args.seed)
np.random.seed(args.seed)
# generates the figures
generate_figure1(model, filename=os.path.join(file_dir, "figure1.png"))
X_train = np.concatenate([X_train, new], axis=0)
Y_train = np.concatenate([
Y_train,
convert_to_one_hot(np.array([int(f_a)]), len(file_all))
])
print("X_train and Y_train concatenate finish!")
else:
print("There is no new_data file.")
#%%
# create session
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth = True
tf.reset_default_graph()
model = MODEL(LR, FILTER_NUM, BATCH_SIZE)
saver = tf.train.Saver(max_to_keep=1)
with tf.Session(config=config) as sess:
#saver.restore(sess, './AOI/model/'+'0308'+'/tf_AOI_n2n_'+'1'+'.ckpt')
#merged = tf.summary.merge_all()
#writer = tf.summary.FileWriter("TensorBoard/", sess.graph)
init_op = tf.global_variables_initializer()
sess.run(init_op)
#training
print("Total steps: " + str(int(mnist.train.images.shape[0] / BATCH_SIZE)))
for e in range(EPOCH):
rand = np.random.randint(1000)
np.random.seed(rand)
def main():
# Utility object for reading, writing parameters, etc.
utils = UTILS()
# Reading parameters from para.dat file
parameters = utils.read_parameter_file()
# Command line specified parameters overide parameter file values
utils.read_command_line_arg(parameters, sys.argv)
# Printing parameters for user
utils.print_parameters(parameters)
# Defining Hamiltonian
H = HAMILTONIAN(**parameters)
# Defines the model, and precomputes evolution matrices given set of states
model = MODEL(H, parameters)
#n_step = parameters['n_step']
#X,y=sample_m0(10000,n_step,model)
#print(y[0:10])
#plt.hist(y,bins=20)
#plt.show()
rob_vs_T = {}
n_eval = {}
fid = {}
res = {}
visit = {}
T_list = np.arange(0.1, 4.01, 0.1)
n_step = 100
fid_list = []
for T in T_list:
parameters['T'] = T
parameters['n_step'] = n_step
parameters['dt'] = T / n_step
file = utils.make_file_name(parameters, root='data/')
res = parse_data(file, v=3)
fid_list.append(np.mean(res['F']))
#n_eval[(n_step,hash(T))]=res['n_fid']
#fid[(n_step,hash(T))]=res['F']
#visit[(n_step,hash(T))] = res['n_visit']
plt.plot(T_list, fid_list)
plt.xlabel('T')
plt.ylabel('Fidelity')
plt.show()
exit()
n_step_list = [40, 50, 60, 70, 80, 90, 100, 110]
for T in T_list:
for n_step in n_step_list: #[40,50,60,70,80,90,100,110,120]:
##for T in np.arange(0.025,10.001,0.025):
# for n_step in [100,200,400] :
parameters['T'] = T
parameters['n_step'] = n_step
parameters['dt'] = T / n_step
file = utils.make_file_name(parameters, root='data/')
res = parse_data(file)
n_eval[(n_step, hash(T))] = res['n_fid']
fid[(n_step, hash(T))] = res['F']
visit[(n_step, hash(T))] = res['n_visit']
''' with open(file,'rb') as f:
_, data = pickle.load(f)
n_elem = len(data)
n_eval[(n_step,hash(T))]=[]
n_fid[(n_step,hash(T))]=[]
for elem in data:
n_eval[(n_step,hash(T))].append(elem[0])
n_fid[(n_step,hash(T))].append(elem[1])'''
#print(n_eval)
#exit()
n_eval_mean = {}
fid_mean = {}
visit_mean = {}
#print(visit[(40,115292150460684704)])
#exit()
for n_step in n_step_list:
n_eval_mean[n_step] = []
fid_mean[n_step] = []
visit_mean[n_step] = []
for T in T_list:
hT = hash(T)
n_eval_mean[n_step].append(
[T, np.mean(n_eval[(n_step, hT)]) / (n_step * n_step)])
fid_mean[n_step].append([T, np.mean(fid[(n_step, hT)])])
visit_mean[n_step].append(
[T, np.mean(visit[(n_step, hT)]) / (n_step)])
c_list = [
'#d53e4f', '#f46d43', '#fdae61', '#fee08b', '#e6f598', '#abdda4',
'#66c2a5', '#3288bd'
]
for i, n_step in enumerate(n_step_list):
x = np.array(n_eval_mean[n_step])
plt.plot(x[:, 0], x[:, 1], c='black', zorder=0)
plt.scatter(x[:, 0],
x[:, 1],
c=c_list[i],
marker='o',
s=5,
label='$N=%i$' % n_step,
zorder=1)
plt.title('Number of fidelity evaluations vs. ramp time \n for 2 flip')
plt.ylabel('$N_{eval}/N^2$')
plt.xlabel('$T$')
plt.legend(loc='best')
plt.tight_layout()
plt.show()
for i, n_step in enumerate(n_step_list):
x = np.array(visit_mean[n_step])
plt.plot(x[:, 0], x[:, 1], c='black', zorder=0)
plt.scatter(x[:, 0],
x[:, 1],
c=c_list[i],
marker='o',
s=5,
label='$N=%i$' % n_step,
zorder=1)
plt.title('Number of visited states vs. ramp time \n for 2 flip')
plt.ylabel('$N_{visit}/N$')
plt.xlabel('$T$')
plt.legend(loc='best')
plt.tight_layout()
plt.show()
'''
#%%
# Find and load model
# Find the model
files = listdir('input')
model_name = files[0]
# Load the model
with open(model_name, 'rb') as f:
# Load checkpoint
checkpoint = torch.load(f)
# Instantiate the model
model = MODEL(checkpoint['input_size'], checkpoint['output_size'], \
checkpoint['hidden_size'], checkpoint['num_layers'], checkpoint['p'])
model.load_state_dict(checkpoint['state_dict'])
# Move to CUDA if availablr
if torch.cuda.is_available():
model.cuda()
# Configure evaluation mode
model.eval()
# %%
# Export model
torch.onnx.export(
model, (torch.randn(1, 1, 12, device='cuda'), model.init_hidden(1)),
'output/forecast.onnx',
input_names=['input', 'h0', 'c0'],
import time
from inputData import INPUTDATA
from model import MODEL
from flow import FLOWINFO
from algs import ALGS
from dynamicBerths import DYNAMICBERTHS
if __name__ == '__main__':
start = time.process_time()
print('start: ', start)
input_file = './input/'
data_list = ['20200301-20200325', '20200402-20200408', '20200401-20200416','20200301-20200429']
input_data = INPUTDATA(input_file + data_list[3] + '/')
flow_info = FLOWINFO(input_data,False,False)
model = MODEL(flow_info)
algs = ALGS(model)
algs.model.show_encoding()
algs.model.show_fit()
# encoding_record = algs.model.set_encoding_record()
# dynamic loading berths
dynamicBerths = DYNAMICBERTHS(input_data)
# no NC functions
dynamicBerths.no_NC_plots()
# NC functions
dynamicBerths.NC_plots()
print('end: ', time.process_time() - start)
def train_one_dataset(params, file_name, train_q_data, train_qa_data, valid_q_data, valid_qa_data):
### ================================== model initialization ==================================
g_model = MODEL(n_question=params.n_question,
seqlen=params.seqlen,
batch_size=params.batch_size,
q_embed_dim=params.q_embed_dim,
qa_embed_dim=params.qa_embed_dim,
memory_size=params.memory_size,
memory_key_state_dim=params.memory_key_state_dim,
memory_value_state_dim=params.memory_value_state_dim,
final_fc_dim = params.final_fc_dim)
# 创建模型
# create a module by given a Symbol
net = mx.mod.Module(symbol=g_model.sym_gen(),
data_names = ['q_data', 'qa_data'],
label_names = ['target'],
context=params.ctx)
'''
symbol：网络符号
context：执行设备（设备列表）
data_names：数据变量名称列表
label_names：标签变量名称列表
'''
# 中间层接口
# create memory by given input shapes 通过内存分配为计算搭建环境
net.bind(data_shapes=[mx.io.DataDesc(name='q_data', shape=(params.seqlen, params.batch_size), layout='SN'),
mx.io.DataDesc(name='qa_data', shape=(params.seqlen, params.batch_size), layout='SN')],
label_shapes=[mx.io.DataDesc(name='target', shape=(params.seqlen, params.batch_size), layout='SN')])
# initial parameters with the default random initializer 初始化参数
net.init_params(initializer=mx.init.Normal(sigma=params.init_std))
# decay learning rate in the lr_scheduler
lr_scheduler = mx.lr_scheduler.FactorScheduler(step=20*(train_q_data.shape[0]/params.batch_size), factor=0.667, stop_factor_lr=1e-5)
# 初始化优化器
net.init_optimizer(optimizer='sgd', optimizer_params={'learning_rate': params.lr, 'momentum':params.momentum,'lr_scheduler': lr_scheduler})
for parameters in net.get_params()[0]:
print(parameters, net.get_params()[0][parameters].asnumpy().shape)
print("\n")
### ================================== start training ==================================
all_train_loss = {}
all_train_accuracy = {}
all_train_auc = {}
all_valid_loss = {}
all_valid_accuracy = {}
all_valid_auc = {}
best_valid_auc = 0
for idx in range(params.max_iter):
train_loss, train_accuracy, train_auc = train(net, params, train_q_data, train_qa_data, label='Train')
valid_loss, valid_accuracy, valid_auc = test(net, params, valid_q_data, valid_qa_data, label='Valid')
print('epoch', idx + 1)
print("valid_auc\t", valid_auc, "\ttrain_auc\t", train_auc)
print("valid_accuracy\t", valid_accuracy, "\ttrain_accuracy\t", train_accuracy)
print("valid_loss\t", valid_loss, "\ttrain_loss\t", train_loss)
if not os.path.isdir('model'):
os.makedirs('model')
if not os.path.isdir(os.path.join('model', params.save)):
os.makedirs(os.path.join('model', params.save))
all_valid_auc[idx + 1] = valid_auc
all_train_auc[idx + 1] = train_auc
all_valid_loss[idx + 1] = valid_loss
all_train_loss[idx + 1] = train_loss
all_valid_accuracy[idx + 1] = valid_accuracy
all_train_accuracy[idx + 1] = train_accuracy
# output the epoch with the best validation auc
if valid_auc > best_valid_auc :
best_valid_auc = valid_auc
best_epoch = idx+1
# here the epoch is default, set to be 100
# we only save the model in the epoch with the better results
net.save_checkpoint(prefix=os.path.join('model', params.save, file_name), epoch=100)
if not os.path.isdir('result'):
os.makedirs('result')
if not os.path.isdir(os.path.join('result', params.save)):
os.makedirs(os.path.join('result', params.save))
f_save_log = open(os.path.join('result', params.save, file_name), 'w')
f_save_log.write("valid_auc:\n" + str(all_valid_auc) + "\n\n")
f_save_log.write("train_auc:\n" + str(all_train_auc) + "\n\n")
f_save_log.write("valid_loss:\n" + str(all_valid_loss) + "\n\n")
f_save_log.write("train_loss:\n" + str(all_train_loss) + "\n\n")
f_save_log.write("valid_accuracy:\n" + str(all_valid_accuracy) + "\n\n")
f_save_log.write("train_accuracy:\n" + str(all_train_accuracy) + "\n\n")
f_save_log.close()
return best_epoch
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--gpu',
type=int,
default=-1,
help='the gpu will be used, e.g "0,1,2,3"')
parser.add_argument('--max_iter',
type=int,
default=10,
help='number of iterations')
parser.add_argument('--decay_epoch',
type=int,
default=20,
help='number of iterations')
parser.add_argument('--test',
type=bool,
default=False,
help='enable testing')
parser.add_argument('--train_test',
type=bool,
default=True,
help='enable testing')
parser.add_argument('--show',
type=bool,
default=True,
help='print progress')
parser.add_argument('--init_std',
type=float,
default=0.1,
help='weight initialization std')
parser.add_argument('--init_lr',
type=float,
default=0.01,
help='initial learning rate')
parser.add_argument('--lr_decay',
type=float,
default=0.75,
help='learning rate decay')
parser.add_argument(
'--final_lr',
type=float,
default=1E-5,
help='learning rate will not decrease after hitting this threshold')
parser.add_argument('--momentum',
type=float,
default=0.9,
help='momentum rate')
parser.add_argument('--max_grad_norm',
type=float,
default=3.0,
help='maximum gradient norm')
parser.add_argument('--hidden_dim',
type=int,
default=128,
help='hidden layer dimension')
parser.add_argument('--n_hidden',
type=int,
default=2,
help='hidden numbers')
dataset = 'assist2009_updated'
if dataset == 'oj':
parser.add_argument('--batch_size',
type=int,
default=5,
help='the batch size')
parser.add_argument('--qa_embed_dim',
type=int,
default=100,
help='answer and question embedding dimensions')
parser.add_argument(
'--n_question',
type=int,
default=68,
help='the number of unique questions in the dataset')
parser.add_argument('--seqlen',
type=int,
default=200,
help='the allowed maximum length of a sequence')
parser.add_argument('--data_dir',
type=str,
default='./data/oj',
help='data directory')
parser.add_argument('--data_name',
type=str,
default='oj',
help='data set name')
parser.add_argument('--load',
type=str,
default='oj',
help='model file to load')
parser.add_argument('--save',
type=str,
default='oj',
help='path to save model')
elif dataset == 'assistments':
parser.add_argument('--batch_size',
type=int,
default=32,
help='the batch size')
parser.add_argument('--qa_embed_dim',
type=int,
default=200,
help='answer and question embedding dimensions')
parser.add_argument(
'--n_question',
type=int,
default=124,
help='the number of unique questions in the dataset')
parser.add_argument('--seqlen',
type=int,
default=200,
help='the allowed maximum length of a sequence')
parser.add_argument('--data_dir',
type=str,
default='./data/assistments',
help='data directory')
parser.add_argument('--data_name',
type=str,
default='assistments',
help='data set name')
parser.add_argument('--load',
type=str,
default='assistments',
help='model file to load')
parser.add_argument('--save',
type=str,
default='assistments',
help='path to save model')
elif dataset == 'assist2009_updated':
parser.add_argument('--batch_size',
type=int,
default=32,
help='the batch size')
parser.add_argument('--qa_embed_dim',
type=int,
default=200,
help='answer and question embedding dimensions')
parser.add_argument(
'--n_question',
type=int,
default=110,
help='the number of unique questions in the dataset')
parser.add_argument('--seqlen',
type=int,
default=200,
help='the allowed maximum length of a sequence')
parser.add_argument('--data_dir',
type=str,
default='../../dataset/assist2009_updated',
help='data directory')
parser.add_argument('--data_name',
type=str,
default='assist2009_updated',
help='data set name')
parser.add_argument('--load',
type=str,
default='assist2009_updated',
help='model file to load')
parser.add_argument('--save',
type=str,
default='assist2009_updated',
help='path to save model')
elif dataset == 'STATICS':
parser.add_argument('--batch_size',
type=int,
default=10,
help='the batch size')
parser.add_argument('--qa_embed_dim',
type=int,
default=100,
help='answer and question embedding dimensions')
parser.add_argument(
'--n_question',
type=int,
default=1223,
help='the number of unique questions in the dataset')
parser.add_argument('--seqlen',
type=int,
default=800,
help='the allowed maximum length of a sequence')
parser.add_argument('--data_dir',
type=str,
default='./data/STATICS',
help='data directory')
parser.add_argument('--data_name',
type=str,
default='STATICS',
help='data set name')
parser.add_argument('--load',
type=str,
default='STATICS',
help='model file to load')
parser.add_argument('--save',
type=str,
default='STATICS',
help='path to save model')
params = parser.parse_args()
params.lr = params.init_lr
print(params)
dat = DataLoader(',', params.seqlen, 1, 0)
# dat = DATA(n_question=params.n_question, seqlen=params.seqlen, separate_char=',')
# train_data_path = params.data_dir + "/" + "builder_train.csv"
# valid_data_path = params.data_dir + "/" + "builder_test.csv"
train_data_path = params.data_dir + "/" + params.data_name + "_train1.csv"
valid_data_path = params.data_dir + "/" + params.data_name + "_valid1.csv"
# test_data_path = params.data_dir + "/" + params.data_name + "_test.csv"
max_length, min_length, max_q_id = dat.scan_file(train_data_path)
train_q_data, train_q_t_data, train_answer_data = dat.prepare_model_data(
train_data_path, max_q_id)
train_q_data = np.array(train_q_data)
print(train_q_data.shape)
train_q_t_data = np.array(train_q_t_data)
train_answer_data = np.array(train_answer_data)
valid_q_data, valid_q_t_data, valid_answer_data = dat.prepare_model_data(
valid_data_path, max_q_id)
valid_q_data = np.array(valid_q_data)
valid_q_t_data = np.array(valid_q_t_data)
valid_answer_data = np.array(valid_answer_data)
# train_q_data, train_q_t_data, train_answer_data = dat.load_data(train_data_path)
# valid_q_data, valid_q_t_data, valid_answer_data = dat.load_data(valid_data_path)
# test_q_data, test_q_t_data, test_answer_data = dat.load_data(test_data_path)
model = MODEL(n_question=params.n_question,
hidden_dim=params.hidden_dim,
x_embed_dim=params.qa_embed_dim,
hidden_layers=params.n_hidden,
gpu=params.gpu)
model.init_embeddings()
model.init_params()
# model = torch.load(params.data_dir + "/save/"+params.save)
# optimizer = optim.SGD(params=model.parameters(), lr=params.lr, momentum=params.momentum)
optimizer = optim.Adam(params=model.parameters(),
lr=params.lr,
betas=(0.9, 0.9))
if params.gpu >= 0:
print('device: ' + str(params.gpu))
torch.cuda.set_device(params.gpu)
model.cuda()
all_train_loss = {}
all_train_accuracy = {}
all_train_auc = {}
all_valid_loss = {}
all_valid_accuracy = {}
all_valid_auc = {}
best_valid_auc = 0
for idx in range(params.max_iter):
train_loss, train_accuracy, train_auc = train(model, idx, params,
optimizer, train_q_data,
train_q_t_data,
train_answer_data)
print(
'Epoch %d/%d, loss : %3.5f, auc : %3.5f, accuracy : %3.5f' %
(idx + 1, params.max_iter, train_loss, train_auc, train_accuracy))
valid_loss, valid_accuracy, valid_auc = test(model, params, optimizer,
valid_q_data,
valid_q_t_data,
valid_answer_data)
print('Epoch %d/%d, valid auc : %3.5f, valid accuracy : %3.5f' %
(idx + 1, params.max_iter, valid_auc, valid_accuracy))
# test_loss, test_accuracy, test_auc = test(model, params, optimizer, test_q_data, test_q_t_data,
# test_answer_data)
# print('Epoch %d/%d, test auc : %3.5f, test accuracy : %3.5f' % (
# idx + 1, params.max_iter, test_auc, test_accuracy))
all_train_auc[idx + 1] = train_auc
all_train_accuracy[idx + 1] = train_accuracy
all_train_loss[idx + 1] = train_loss
all_valid_loss[idx + 1] = valid_loss
all_valid_accuracy[idx + 1] = valid_accuracy
all_valid_auc[idx + 1] = valid_auc
#
# output the epoch with the best validation auc
if valid_auc > best_valid_auc:
print('%3.4f to %3.4f' % (best_valid_auc, valid_auc))
best_valid_auc = valid_auc
def train(args):
log_file=open('./data/logs/log.txt','a')
log_file.write('\n'+'\n')
log_file.write('-------------------------------------------\n')
log_file.write(str(args)+'\n')
log_file.write('-------------------------------------------\n')
print('model initializing..')
model = MODEL(args)
# CUDA_VISIBLE_DEVICES=args.GPU_DEVICE
with tf.compat.v1.Session() as sess:
sess.run(tf.compat.v1.global_variables_initializer())
sess.run(tf.compat.v1.local_variables_initializer())
saver = tf.compat.v1.train.Saver(write_version=tf.compat.v1.train.SaverDef.V2, max_to_keep=10)
print('loading data..')
load_file = args.file_directory+str(args.first_train_file)
train_data = load_data(args,load_file)
load_file = args.file_directory+str(args.first_test_file)
test_data = load_data(args,load_file)
for step in range(args.n_epochs):
# training
print('epoch:'+str(step))
args.is_train=True
if step != 0:
for i in range(args.train_file_num):
start_list = list(range(0, train_data.size, args.batch_size))
np.random.shuffle(start_list)
for start in start_list:
end = start + args.batch_size
model.train(sess, get_feed_dict(model, train_data, start, end))
args.is_train=False
train_loss = 0
train_time_loss = 0
train_batchs = 0
train_labels = []
train_scores = []
train_time_true = []
train_time_pre = []
for i in range(args.train_file_num):
#load_file=args.file_directory+str(args.first_train_file+i)
#train_data=load_data(args,load_file)
start_list = list(range(0, train_data.size, args.batch_size*10))
train_batchs = train_batchs + len(start_list)
for start in start_list:
end = start + args.batch_size*10
loss, labels, scores, time_loss, time_pre, time_true, time_label = model.test(sess, get_feed_dict(model, train_data, start, end))
train_labels.extend(labels)
train_scores.extend(scores)
for k in range(len(time_label)):
if time_label[k] == 0:
continue
train_time_true.append(time_true[k])
train_time_pre.append(time_pre[k])
train_loss = train_loss+loss
train_time_loss = train_time_loss + time_loss
train_auc, train_f1, train_pre, train_recall, train_time_f1 = model.eval(args, train_labels, train_scores, train_time_true, train_time_pre)
train_loss = train_loss / train_batchs
train_time_loss = train_time_loss/train_batchs
time_stamp = datetime.datetime.now().strftime('%Y.%m.%d-%H:%M:%S')
log_str = 'time:' + time_stamp + ' epoch %d \ntrain_loss:%.4f train_time_loss:%.4f train_time_f1:%.4f train_auc:%.4f train_f1_score:%.4f'\
% (step, train_loss, train_time_loss, train_time_f1, train_auc, train_f1)
print(log_str)
log_file.write(log_str)
log_file.write('\n')
log_file.close()
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--gpu', type=int, default=0)
parser.add_argument('--max_iter', type=int, default=30, help='number of iterations')
parser.add_argument('--decay_epoch', type=int, default=20, help='number of iterations')
parser.add_argument('--test', type=bool, default=False, help='enable testing')
parser.add_argument('--train_test', type=bool, default=True, help='enable testing')
parser.add_argument('--show', type=bool, default=True, help='print progress')
parser.add_argument('--init_std', type=float, default=0.1, help='weight initialization std')
parser.add_argument('--init_lr', type=float, default=0.01, help='initial learning rate')
parser.add_argument('--lr_decay', type=float, default=0.75, help='learning rate decay')
parser.add_argument('--final_lr', type=float, default=1E-5,
help='learning rate will not decrease after hitting this threshold')
parser.add_argument('--momentum', type=float, default=0.9, help='momentum rate')
parser.add_argument('--max_grad_norm', type=float, default=3.0, help='maximum gradient norm')
parser.add_argument('--hidden_dim', type=int, default=64, help='hidden layer dimension')
parser.add_argument('--n_hidden', type=int, default=2, help='hidden numbers')
parser.add_argument('--dataset', type=str, default='assist2009_updated')
parser.add_argument('--batch_size', type=int, default=32, help='the batch size')
parser.add_argument('--qa_embed_dim', type=int, default=200, help='answer and question embedding dimensions')
parser.add_argument('--dropout_rate', type=float, default=0.6)
if parser.parse_args().dataset == 'assist2009_updated':
# parser.add_argument('--batch_size', type=int, default=32, help='the batch size')
# parser.add_argument('--qa_embed_dim', type=int, default=200, help='answer and question embedding dimensions')
parser.add_argument('--n_question', type=int, default=110, help='the number of unique questions in the dataset')
parser.add_argument('--seqlen', type=int, default=200, help='the allowed maximum length of a sequence')
parser.add_argument('--data_dir', type=str, default='../dataset/assist2009_updated', help='data directory')
parser.add_argument('--data_name', type=str, default='assist2009_updated', help='data set name')
parser.add_argument('--load', type=str, default='assist2009_updated', help='model file to load')
parser.add_argument('--save', type=str, default='assist2009_updated', help='path to save model')
elif parser.parse_args().dataset == 'assist2015':
# parser.add_argument('--batch_size', type=int, default=32, help='the batch size')
# parser.add_argument('--qa_embed_dim', type=int, default=200, help='answer and question embedding dimensions')
parser.add_argument('--n_question', type=int, default=100, help='the number of unique questions in the dataset')
parser.add_argument('--seqlen', type=int, default=200, help='the allowed maximum length of a sequence')
parser.add_argument('--data_dir', type=str, default='../dataset/assist2015', help='data directory')
parser.add_argument('--data_name', type=str, default='assist2015', help='data set name')
parser.add_argument('--load', type=str, default='assist2015', help='model file to load')
parser.add_argument('--save', type=str, default='assist2015', help='path to save model')
elif parser.parse_args().dataset == 'STATICS':
# parser.add_argument('--batch_size', type=int, default=32, help='the batch size')
# parser.add_argument('--qa_embed_dim', type=int, default=100, help='answer and question embedding dimensions')
parser.add_argument('--n_question', type=int, default=1223,
help='the number of unique questions in the dataset')
parser.add_argument('--seqlen', type=int, default=200, help='the allowed maximum length of a sequence')
parser.add_argument('--data_dir', type=str, default='../dataset/STATICS', help='data directory')
parser.add_argument('--data_name', type=str, default='STATICS', help='data set name')
parser.add_argument('--load', type=str, default='STATICS', help='model file to load')
parser.add_argument('--save', type=str, default='STATICS', help='path to save model')
elif parser.parse_args().dataset == 'synthetic':
# parser.add_argument('--batch_size', type=int, default=32, help='the batch size')
# parser.add_argument('--qa_embed_dim', type=int, default=100, help='answer and question embedding dimensions')
parser.add_argument('--n_question', type=int, default=50,
help='the number of unique questions in the dataset')
parser.add_argument('--seqlen', type=int, default=200, help='the allowed maximum length of a sequence')
parser.add_argument('--data_dir', type=str, default='../dataset/synthetic', help='data directory')
parser.add_argument('--data_name', type=str, default='synthetic', help='data set name')
parser.add_argument('--load', type=str, default='synthetic', help='model file to load')
parser.add_argument('--save', type=str, default='synthetic', help='path to save model')
elif parser.parse_args().dataset == 'assist2017':
# parser.add_argument('--batch_size', type=int, default=32, help='the batch size')
# parser.add_argument('--qa_embed_dim', type=int, default=100, help='answer and question embedding dimensions')
parser.add_argument('--n_question', type=int, default=102,
help='the number of unique questions in the dataset')
parser.add_argument('--seqlen', type=int, default=200, help='the allowed maximum length of a sequence')
parser.add_argument('--data_dir', type=str, default='../dataset/assist2017/train_valid_test', help='data directory')
parser.add_argument('--data_name', type=str, default='assist2017', help='data set name')
parser.add_argument('--load', type=str, default='assist2017', help='model file to load')
parser.add_argument('--save', type=str, default='assist2017', help='path to save model')
params = parser.parse_args()
params.lr = params.init_lr
print(params)
dat = DATA(n_question=params.n_question, seqlen=params.seqlen, separate_char=',')
if params.dataset != 'synthetic':
train_data_path = params.data_dir + "/" + params.data_name + "_train1.csv"
valid_data_path = params.data_dir + "/" + params.data_name + "_valid1.csv"
test_data_path = params.data_dir + "/" + params.data_name + "_test.csv"
else:
train_data_path = params.data_dir + "/" + "naive_c5_q50_s4000_v0_train1.csv"
valid_data_path = params.data_dir + "/" + "naive_c5_q50_s4000_v0_valid1.csv"
test_data_path = params.data_dir + "/" + "naive_c5_q50_s4000_v0_test.csv"
train_q_data, train_q_t_data, train_answer_data, train_repeated_time_gap, train_past_trail_counts,\
train_seq_time_gap = dat.load_data(train_data_path)
valid_q_data, valid_q_t_data, valid_answer_data, valid_repeated_time_gap, valid_past_trail_counts,\
valid_seq_time_gap = dat.load_data(valid_data_path)
test_q_data, test_q_t_data, test_answer_data, test_repeated_time_gap, test_past_trail_counts,\
test_seq_time_gap = dat.load_data(test_data_path)
model = MODEL(batch_size=params.batch_size,
seqlen=params.seqlen,
n_question=params.n_question,
hidden_dim=params.hidden_dim,
x_embed_dim=params.qa_embed_dim,
hidden_layers=params.n_hidden,
dropout_rate=params.dropout_rate,
gpu=params.gpu)
model.init_embeddings()
model.init_params()
optimizer = optim.Adam(params=model.parameters(), lr=params.lr, betas=(0.9, 0.9))
if params.gpu >= 0:
print('device: ' + str(params.gpu))
torch.cuda.set_device(params.gpu)
model.cuda()
# all_train_loss = {}
# all_train_accuracy = {}
# all_train_auc = {}
# all_valid_loss = {}
# all_valid_accuracy = {}
# all_valid_auc = {}
# all_test_loss = {}
# all_test_accuracy = {}
# all_test_auc = {}
best_valid_auc = 0
cur_test_auc = 0
cur_train_auc = 0
for idx in range(params.max_iter):
train_loss, train_accuracy, train_auc = train(model, params, optimizer, train_q_data, train_q_t_data,
train_answer_data, train_repeated_time_gap,\
train_past_trail_counts, train_seq_time_gap)
print('Epoch %d/%d, loss : %3.5f, auc : %3.5f, accuracy : %3.5f' % (
idx + 1, params.max_iter, train_loss, train_auc, train_accuracy))
valid_loss, valid_accuracy, valid_auc = test(model, params, optimizer, valid_q_data, valid_q_t_data,
valid_answer_data, valid_repeated_time_gap,\
valid_past_trail_counts, valid_seq_time_gap)
print('Epoch %d/%d, valid auc : %3.5f, valid accuracy : %3.5f' % (
idx + 1, params.max_iter, valid_auc, valid_accuracy))
test_loss, test_accuracy, test_auc = test(model, params, optimizer, test_q_data, test_q_t_data,
test_answer_data, test_repeated_time_gap,
test_past_trail_counts, test_seq_time_gap)
print('Epoch %d/%d, test auc : %3.5f, test accuracy : %3.5f' % (
idx + 1, params.max_iter, test_auc, test_accuracy))
# all_train_auc[idx + 1] = train_auc
# all_train_accuracy[idx + 1] = train_accuracy
# all_train_loss[idx + 1] = train_loss
# all_valid_loss[idx + 1] = valid_loss
# all_valid_accuracy[idx + 1] = valid_accuracy
# all_valid_auc[idx + 1] = valid_auc
# all_test_loss[idx + 1] = test_loss
# all_test_accuracy[idx + 1] = test_accuracy
# all_test_auc[idx + 1] = test_auc
if valid_auc > best_valid_auc:
print('%3.4f to %3.4f' % (best_valid_auc, valid_auc))
best_valid_auc = valid_auc
cur_test_auc = test_auc
cur_train_auc = train_auc
print('DATASET: {}, TRAIN AUC: {}, BEST VALID AUC: {}, TEST AUC: {}'.format(params.data_name, cur_train_auc, \
best_valid_auc, cur_test_auc))
def cross_clustering_single_image(clusters_save_path, model_path):
'''
Will combine clusters from a single image using cluster centers for smaller subsets
:return:
'''
full_test_site_shape = (3663, 5077)
stride = 400
full_test_site_used_shape = (stride * (full_test_site_shape[0] // stride),
stride * (full_test_site_shape[1] // stride))
# print(full_test_site_used_shape)
n_clusters = 400
n_iterations = 30
all_cluster_centers = None
all_cluster_labels = np.zeros(full_test_site_used_shape[0] *
full_test_site_used_shape[1])
full_clustered_image = np.zeros(
shape=(stride * int(full_test_site_shape[0] / stride),
stride * int(full_test_site_shape[1] / stride)))
# row and column wise stride on the entire image
# count = -1
for row in range(0, full_test_site_shape[0] // stride):
for col in range(0, full_test_site_shape[1] // stride):
# count += 1
this_cluster_save_path = os.path.join(
clusters_save_path,
'{}_{}_{}_{}.pkl'.format(row * stride, (row + 1) * stride,
col * stride, (col + 1) * stride))
print("log: Reading rows {}-{}, columns {}-{}".format(
row * stride, (row + 1) * stride, col * stride,
(col + 1) * stride))
with open(this_cluster_save_path, 'rb') as read_this_one:
kmeans_subset = pkl.load(read_this_one)
if all_cluster_centers is not None:
all_cluster_centers = np.dstack(
(all_cluster_centers, kmeans_subset.cluster_centers_))
else:
all_cluster_centers = kmeans_subset.cluster_centers_
array = kmeans_subset.labels_.reshape(stride, stride)
full_clustered_image[row * stride:(row + 1) * stride,
col * stride:(col + 1) * stride] = array
# bad bad bad
# all_cluster_labels[count*stride**2:count*stride**2+stride**2] = np.asarray(array.reshape(-1) +
# 10*count, dtype=np.uint16)
pl.imshow(full_clustered_image)
pl.title('Input image')
pl.show()
# verified that this doesn't work
# reshaping loop (because I feel insecure with this thing)
all_cluster_labels = full_clustered_image.reshape((-1, 108))
# print(another == all_cluster_labels)
# print(np.unique(all_cluster_labels), all_cluster_labels.dtype)
all_cluster_centers = all_cluster_centers.transpose((2, 1, 0))
all_cluster_centers = all_cluster_centers.reshape((-1, 13))
# update labels to new values to keep them separated
for i in range(all_cluster_labels.shape[1]):
all_cluster_labels[:, i] += 10 * i
# cluster cluster-centers now
print('log: Clustering now...')
kmeans = call_kmeans(samples_vec=all_cluster_centers,
n_clusters=n_clusters,
n_iterations=n_iterations,
pickle_file=None)
print('log: done clustering')
# print(kmeans.labels_.shape)
# we should have n_clusters unique labels now
# now we assign classes to clusters on the bases of their classification results
# because we have 108 subsets, each having their own 0-9 cluster labels
print('log: assigning new labels...')
# for i in range(108):
for i in range(all_cluster_labels.shape[1]):
for j in range(10):
all_cluster_labels[all_cluster_labels == i * 10 +
j] = kmeans.labels_[i * 10 + j]
# check
# show_image = all_cluster_labels.reshape(full_test_site_used_shape)
# pl.imshow(show_image)
# pl.title('Classified image')
# pl.show()
# return
########### classification starts now ...
# load model for inference
model = MODEL(in_channels=13)
model.load_state_dict(torch.load(model_path))
print('log: loaded saved model {}'.format(model_path))
model.eval()
outer_kmeans_centers = np.expand_dims(kmeans.cluster_centers_, 2)
outer_kmeans_centers_tensor = torch.Tensor(outer_kmeans_centers)
out_x, pred = model(outer_kmeans_centers_tensor)
pred_arr = pred.numpy()
# now we assign classes to clusters on the bases of their classification results
all_cluster_labels = all_cluster_labels.reshape(-1)
for u in range(n_clusters): # because we n_clusters now
# u is the cluster number
all_cluster_labels[all_cluster_labels == u] = pred_arr[u]
pass
show_image = all_cluster_labels.reshape(full_test_site_used_shape)
pl.imshow(show_image)
pl.title('Classified image')
pl.show()
all_labels = {
'new_signature_lowvegetation': 0,
'new_signature_forest': 1,
'new_signature_urban': 2,
'new_signature_cropland': 3,
'new_signature_waterbody': 4
}
reversed_labels = {v: k for k, v in all_labels.items()}
classes_found = np.unique(show_image)
for p in classes_found:
pl.imshow(show_image == p)
pl.title('Class {}'.format(reversed_labels[p]))
pl.show()
pass
def classify_cluster_arrays(example_path,
clusters_save_path,
model_path,
is_normalized=True):
# we'll need the data for it too
all_bands = []
for i in range(1, 14):
all_bands.append(
np.load(os.path.join(example_path, '{}.npy'.format(i)),
mmap_mode='r'))
full_test_site_shape = (3663, 5077)
stride = 400
# row and column wise stride on the entire image
full_classified_image = np.zeros(
shape=(stride * int(full_test_site_shape[0] / stride),
stride * int(full_test_site_shape[1] / stride)))
# load model for inference
model = MODEL(in_channels=13)
model.load_state_dict(torch.load(model_path))
print('log: loaded saved model {}'.format(model_path))
model.eval()
for row in range(0, full_test_site_shape[0] // stride):
for col in range(0, full_test_site_shape[1] // stride):
this_cluster_save_path = os.path.join(
clusters_save_path,
'{}_{}_{}_{}.pkl'.format(row * stride, (row + 1) * stride,
col * stride, (col + 1) * stride))
print("rows {}-{}, columns {}-{}".format(row * stride,
(row + 1) * stride,
col * stride,
(col + 1) * stride))
with open(this_cluster_save_path, 'rb') as read_this_one:
kmeans_subset = pkl.load(read_this_one)
kmeans_subset_labels = kmeans_subset.labels_
# predict classes after receiving the data
full_array = combine_bands(
mapped_bands_list=all_bands,
coordinates=(row * stride, (row + 1) * stride,
col * stride, (col + 1) * stride))
samples_vec = full_array.reshape((-1, 13))
samples_vec = np.nan_to_num(samples_vec) # important nan save
# pick 100 vectors for each cluster, classify and assign to clusters
classified_mini_array = kmeans_subset_labels.copy()
for u in range(
10): # because we have ten clusters in each subset
test_these = samples_vec[kmeans_subset_labels == u]
if is_normalized:
test_these = 2 * (test_these / 4096.).clip(
0, 1) - 1 # if normalization is used
test_vectors = np.expand_dims(test_these[random.sample(
range(len(test_these)), 100)],
axis=2)
test_tensor = torch.Tensor(test_vectors)
# print(test_tensor.shape)
out_x, pred = model(test_tensor)
pred_numpy = pred.numpy()
vals, counts = np.unique(pred_numpy, return_counts=True)
this_index = np.argmax(counts)
classified_mini_array[classified_mini_array ==
u] = vals[this_index]
# convert classified array to image
mini_image_array = classified_mini_array.reshape(
stride, stride)
full_classified_image[row * stride:(row + 1) * stride,
col * stride:(col + 1) *
stride] = mini_image_array
pl.imshow(full_classified_image, cmap='tab10', alpha=1.0)
pl.show()
pass
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--gpu', type=int, default=0, help='the gpu will be used, e.g "0,1,2,3"')
parser.add_argument('--max_iter', type=int, default=50, help='number of iterations')
parser.add_argument('--decay_epoch', type=int, default=20, help='number of iterations')
parser.add_argument('--test', type=bool, default=False, help='enable testing')
parser.add_argument('--train_test', type=bool, default=True, help='enable testing')
parser.add_argument('--show', type=bool, default=True, help='print progress')
parser.add_argument('--init_std', type=float, default=0.1, help='weight initialization std')
parser.add_argument('--init_lr', type=float, default=0.01, help='initial learning rate')
parser.add_argument('--lr_decay', type=float, default=0.75, help='learning rate decay')
parser.add_argument('--final_lr', type=float, default=1E-5,
help='learning rate will not decrease after hitting this threshold')
parser.add_argument('--momentum', type=float, default=0.9, help='momentum rate')
parser.add_argument('--max_grad_norm', type=float, default=50.0, help='maximum gradient norm')
# parser.add_argument('--final_fc_dim', type=float, default=200, help='hidden state dim for final fc layer')
parser.add_argument('--first_k', type=int, default=8, help='first k question without loss calculation')
parser.add_argument('--dataset', type=str, default='assist2009_updated')
parser.add_argument('--train_set', type=int, default=1)
parser.add_argument('--memory_size', type=int, default=20, help='memory size')
parser.add_argument('--q_embed_dim', type=int, default=50, help='question embedding dimensions')
parser.add_argument('--qa_embed_dim', type=int, default=200, help='answer and question embedding dimensions')
if parser.parse_args().dataset == 'assist2009_updated':
# memory_size: 20, q_embed_dim: 50, qa_embed_dim: 200
parser.add_argument('--batch_size', type=int, default=128, help='the batch size')
parser.add_argument('--n_question', type=int, default=110, help='the number of unique questions in the dataset')
parser.add_argument('--seqlen', type=int, default=200, help='the allowed maximum length of a sequence')
parser.add_argument('--data_dir', type=str, default='../dataset/assist2009_updated', help='data directory')
parser.add_argument('--data_name', type=str, default='assist2009_updated', help='data set name')
parser.add_argument('--load', type=str, default='assist2009_updated', help='model file to load')
parser.add_argument('--save', type=str, default='assist2009_updated', help='path to save model')
parser.add_argument('--final_fc_dim', type=float, default=110, help='hidden state dim for final fc layer')
elif parser.parse_args().dataset == 'assist2015':
# memory_size: 50, q_embed_dim: 50, qa_embed_dim: 200
parser.add_argument('--batch_size', type=int, default=128, help='the batch size')
parser.add_argument('--n_question', type=int, default=100, help='the number of unique questions in the dataset')
parser.add_argument('--seqlen', type=int, default=200, help='the allowed maximum length of a sequence')
parser.add_argument('--data_dir', type=str, default='../dataset/assist2015', help='data directory')
parser.add_argument('--data_name', type=str, default='assist2015', help='data set name')
parser.add_argument('--load', type=str, default='assist2015', help='model file to load')
parser.add_argument('--save', type=str, default='assist2015', help='path to save model')
parser.add_argument('--final_fc_dim', type=float, default=100, help='hidden state dim for final fc layer')
elif parser.parse_args().dataset == 'assist2017':
# memory_size: 20, q_embed_dim: 50, qa_embed_dim: 100
parser.add_argument('--batch_size', type=int, default=32, help='the batch size')
parser.add_argument('--n_question', type=int, default=102,
help='the number of unique questions in the dataset')
parser.add_argument('--seqlen', type=int, default=200, help='the allowed maximum length of a sequence')
parser.add_argument('--data_dir', type=str, default='../dataset/assist2017/train_valid_test/', help='data directory')
parser.add_argument('--data_name', type=str, default='assist2017', help='data set name')
parser.add_argument('--load', type=str, default='assist2017', help='model file to load')
parser.add_argument('--save', type=str, default='assist2017', help='path to save model')
parser.add_argument('--final_fc_dim', type=float, default=102, help='hidden state dim for final fc layer')
elif parser.parse_args().dataset == 'STATICS':
# memory_size: 50, q_embed_dim: 50, qa_embed_dim: 100
parser.add_argument('--batch_size', type=int, default=32, help='the batch size')
parser.add_argument('--n_question', type=int, default=1223, help='the number of unique questions in the dataset')
parser.add_argument('--seqlen', type=int, default=200, help='the allowed maximum length of a sequence')
parser.add_argument('--data_dir', type=str, default='../dataset/STATICS', help='data directory')
parser.add_argument('--data_name', type=str, default='STATICS', help='data set name')
parser.add_argument('--load', type=str, default='STATICS', help='model file to load')
parser.add_argument('--save', type=str, default='STATICS', help='path to save model')
parser.add_argument('--final_fc_dim', type=float, default=1223, help='hidden state dim for final fc layer')
elif parser.parse_args().dataset == 'synthetic':
# memory_size: 20, q_embed_dim: 50, qa_embed_dim: 100
parser.add_argument('--batch_size', type=int, default=128, help='the batch size')
parser.add_argument('--n_question', type=int, default=50,
help='the number of unique questions in the dataset')
parser.add_argument('--seqlen', type=int, default=200, help='the allowed maximum length of a sequence')
parser.add_argument('--data_dir', type=str, default='../dataset/synthetic/', help='data directory')
parser.add_argument('--data_name', type=str, default='synthetic', help='data set name')
parser.add_argument('--load', type=str, default='synthetic', help='model file to load')
parser.add_argument('--save', type=str, default='synthetic', help='path to save model')
parser.add_argument('--final_fc_dim', type=float, default=50, help='hidden state dim for final fc layer')
params = parser.parse_args()
params.lr = params.init_lr
params.memory_key_state_dim = params.q_embed_dim
params.memory_value_state_dim = params.qa_embed_dim
print(params)
dat = DATA(n_question=params.n_question, seqlen=params.seqlen, separate_char=',')
if params.dataset != 'synthetic':
train_data_path = params.data_dir + "/" + params.data_name + "_train" + str(params.train_set) + ".csv"
valid_data_path = params.data_dir + "/" + params.data_name + "_valid" + str(params.train_set) + ".csv"
test_data_path = params.data_dir + "/" + params.data_name + "_test.csv"
else:
train_data_path = params.data_dir + "/" + "naive_c5_q50_s4000_v0_train" + str(params.train_set) + ".csv"
valid_data_path = params.data_dir + "/" + "naive_c5_q50_s4000_v0_valid" + str(params.train_set) + ".csv"
test_data_path = params.data_dir + "/" + "naive_c5_q50_s4000_v0_test.csv"
train_q_data, train_qa_data, train_a_data = dat.load_data(train_data_path)
valid_q_data, valid_qa_data, valid_a_data = dat.load_data(valid_data_path)
test_q_data, test_qa_data, test_a_data = dat.load_data(test_data_path)
params.memory_key_state_dim = params.q_embed_dim
params.memory_value_state_dim = params.qa_embed_dim
model = MODEL(n_question=params.n_question,
batch_size=params.batch_size,
q_embed_dim=params.q_embed_dim,
qa_embed_dim=params.qa_embed_dim,
memory_size=params.memory_size,
memory_key_state_dim=params.memory_key_state_dim,
memory_value_state_dim=params.memory_value_state_dim,
final_fc_dim=params.final_fc_dim,
first_k=params.first_k,
gpu=params.gpu)
model.init_embeddings()
model.init_params()
optimizer = optim.Adam(params=model.parameters(), lr=params.lr, betas=(0.9, 0.9))
if params.gpu >= 0:
print('device: ' + str(params.gpu))
torch.cuda.set_device(params.gpu)
model.cuda()
best_valid_auc = 0
correspond_train_auc = 0
correspond_test_auc = 0
for idx in range(params.max_iter):
train_loss, train_accuracy, train_auc = train(model, params, optimizer, train_q_data, train_qa_data, train_a_data)
print('Epoch %d/%d, loss : %3.5f, auc : %3.5f, accuracy : %3.5f' % (idx + 1, params.max_iter, train_loss, train_auc, train_accuracy))
valid_loss, valid_accuracy, valid_auc = test(model, params, optimizer, valid_q_data, valid_qa_data, valid_a_data)
print('Epoch %d/%d, valid auc : %3.5f, valid accuracy : %3.5f' % (idx + 1, params.max_iter, valid_auc, valid_accuracy))
test_loss, test_accuracy, test_auc = test(model, params, optimizer, test_q_data, test_qa_data, test_a_data)
print('Epoch %d/%d, test auc : %3.5f, test accuracy : %3.5f' % (idx + 1, params.max_iter, test_auc, test_accuracy))
# output the epoch with the best validation auc
if valid_auc > best_valid_auc:
print('%3.4f to %3.4f' % (best_valid_auc, valid_auc))
best_valid_auc = valid_auc
correspond_train_auc = train_auc
correspond_test_auc = test_auc
print("DATASET: {}, MEMO_SIZE: {}, Q_EMBED_SIZE: {}, QA_EMBED_SIZE: {}, LR: {}".format(params.data_name, params.memory_size, params.q_embed_dim, params.qa_embed_dim, params.init_lr))
print("BEST VALID AUC: {}, CORRESPOND TRAIN AUC: {}, CORRESPOND TEST AUC: {}".format(best_valid_auc, correspond_train_auc, correspond_test_auc))
def classify_cluster_centers(clusters_save_path, model_path, save_image=None):
full_test_site_shape = (3663, 5077)
stride = 400
# row and column wise stride on the entire image
full_classified_image = np.zeros(
shape=(stride * int(full_test_site_shape[0] / stride),
stride * int(full_test_site_shape[1] / stride)))
# load model for inference
model = MODEL(in_channels=13)
model.load_state_dict(torch.load(model_path))
print('log: loaded saved model {}'.format(model_path))
model.eval()
for row in range(0, full_test_site_shape[0] // stride):
for col in range(0, full_test_site_shape[1] // stride):
this_cluster_save_path = os.path.join(
clusters_save_path,
'{}_{}_{}_{}.pkl'.format(row * stride, (row + 1) * stride,
col * stride, (col + 1) * stride))
print("rows {}-{}, columns {}-{}".format(row * stride,
(row + 1) * stride,
col * stride,
(col + 1) * stride))
with open(this_cluster_save_path, 'rb') as read_this_one:
kmeans_subset = pkl.load(read_this_one)
kmeans_subset_labels = kmeans_subset.labels_
kmeans_subset_cluster_centers = np.expand_dims(
kmeans_subset.cluster_centers_, 2)
kmeans_subset_cluster_centers_tensor = torch.Tensor(
kmeans_subset_cluster_centers)
out_x, pred = model(kmeans_subset_cluster_centers_tensor)
pred_arr = pred.numpy()
# now we assign classes to clusters on the bases of their classification results
classified_mini_array = kmeans_subset_labels.copy()
for u in range(
10): # because we have ten clusters in each subset
# u is the cluster number
classified_mini_array[kmeans_subset_labels ==
u] = pred_arr[u]
# convert classified array to image
mini_image_array = classified_mini_array.reshape(
stride, stride)
full_classified_image[row * stride:(row + 1) * stride,
col * stride:(col + 1) *
stride] = mini_image_array
# apply median filter for clearer results
full_classified_image = ndimage.median_filter(full_classified_image,
size=5)
print(full_classified_image.shape)
pl.imshow(full_classified_image, cmap='tab10', alpha=1.0)
pl.show()
if save_image:
# new array for saving rgb image
image = np.zeros(shape=(full_classified_image.shape[0],
full_classified_image.shape[1], 3))
for j in range(5):
image[full_classified_image == j, :] = color_mapping[j]
pl.imsave(save_image, image)
print('log: Saved {}'.format(save_image))
pass