def load_data():
data = dataloader()
batch = []
cti = load_tkn_to_idx(sys.argv[2]) # char_to_idx
wti = load_tkn_to_idx(sys.argv[3]) # word_to_idx
print("loading %s..." % sys.argv[4])
with open(sys.argv[4], "r") as fo:
text = fo.read().strip().split("\n" * (HRE + 1))
for block in text:
for line in block.split("\n"):
x, y = line.split("\t")
x = [x.split(":") for x in x.split(" ")]
y = [int(y)] if HRE else [int(x) for x in y.split(" ")] + [len(x)]
xc, xw = zip(*[(list(map(int, xc.split("+"))), int(xw))
for xc, xw in x])
data.append_item(xc=xc, xw=xw, y0=y)
data.append_row()
data.strip()
for _batch in data.split():
xc, xw = data.tensor(*_batch.sort(), eos=True)
_, y0 = data.tensor(None, _batch.y0)
batch.append((xc, xw, y0))
print("data size: %d" % (len(data.y0)))
print("batch size: %d" % BATCH_SIZE)
return batch, cti, wti
def load_data():
data = dataloader()
batch = []
cti = load_tkn_to_idx(
'./prepare_data/train.txt.char_to_idx') # char_to_idx
wti = load_tkn_to_idx(
'./prepare_data/train.txt.word_to_idx') # word_to_idx
itt = load_idx_to_tkn('./prepare_data/train.txt.tag_to_idx') # idx_to_tkn
print("loading %s..." % './prepare_data/train.txt.csv')
with open('./prepare_data/train.txt.csv') as fo:
text = fo.read().strip().split("\n" * (HRE + 1))
for block in text:
for line in block.split("\n"):
x, y = line.split("\t")
x = [x.split(":") for x in x.split(" ")]
y = [int(y)] if HRE else [int(x) for x in y.split(" ")]
xc, xw = zip(*[(list(map(int, xc.split("+"))), int(xw))
for xc, xw in x])
data.append_item(xc=xc, xw=xw, y0=y)
data.append_row()
data.strip()
for _batch in data.split():
xc, xw, y0, lens = _batch.sort()
xc, xw = data.tensor(xc, xw, lens)
_, y0 = data.tensor(None, y0, sos=True)
batch.append((xc, xw, y0))
print("data size: %d" % len(data.y0))
print("batch size: %d" % BATCH_SIZE)
return batch, cti, wti, itt
def train():
#config
batch_size = 4
lr = 0.0005
model_dir = 'model2/'
logfile = 'second.log'
fop = open(logfile, 'w')
#prepare data
dataline = open('data/train.txt').readlines()
datalength = len(dataline)
traindata = dataline[int(datalength / 5):]
print(len(traindata))
vecmodel = word2vec.sentence2vec('sgns.weibo.bigram-char')
a = dataloader(traindata, vecmodel, batch_size)
a.start()
#build model
inputdata = tf.placeholder(tf.float32, [batch_size, None, 300])
inputpadding = tf.placeholder(tf.float32, [batch_size, None])
pos = tf.placeholder(tf.float32, [None, 32])
inputlabel = tf.placeholder(tf.int32, [batch_size])
classifier = model(True, batch_size)
outlogit = classifier(inputdata, inputpadding, pos)
loss = tf.losses.softmax_cross_entropy(tf.one_hot(inputlabel, 2), outlogit)
print(1)
train_op = tf.train.AdamOptimizer(lr).minimize(loss)
saver = tf.train.Saver(max_to_keep=0)
print('build finished')
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
model_file = tf.train.latest_checkpoint(model_dir)
saver.restore(sess, model_file)
#sess.run(tf.global_variables_initializer())
#sess.run(tf.local_variables_initializer())
#train step
for step in tqdm(range(26000, 400000)):
data, label, padding = a.getdata()
#data=np.zeros((3,20,300))
#padding=np.ones((3,20))
length = data.shape[1]
trainloss, _ = sess.run(
[loss, train_op],
feed_dict={
inputdata: data,
inputpadding: padding,
pos: model_utils.get_position_encoding(length, 32),
inputlabel: label
})
if step % 100 == 0:
print('loss:' + str(trainloss))
fop.write('loss:' + str(trainloss) + '\n')
if step % 1000 == 0:
saver.save(sess, model_dir + '/transform.ckpt', global_step=step)
def load_data(args):
data = dataloader()
batch = []
cti = load_tkn_to_idx(args[1]) # char_to_idx
wti = load_tkn_to_idx(args[2]) # word_to_idx
itt = load_idx_to_tkn(args[3]) # idx_to_tkn
print("loading %s..." % args[4])
with open(args[4]) as fo:
text = fo.read().strip().split("\n" * (HRE + 1))
for block in text:
data.append_row()
for line in block.split("\n"):
x, y = line.split("\t")
x = [x.split(":") for x in x.split(" ")]
y = tuple(map(int, y.split(" ")))
xc, xw = zip(*[(list(map(int, xc.split("+"))), int(xw))
for xc, xw in x])
data.append_item(xc=xc, xw=xw, y0=y)
for _batch in data.split():
xc, xw, y0, lens = _batch.xc, _batch.xw, _batch.y0, _batch.lens
xc, xw = data.tensor(bc=xc, bw=xw, lens=lens)
_, y0 = data.tensor(bw=y0, sos=True)
batch.append((xc, xw, y0))
print("data size: %d" % len(data.y0))
print("batch size: %d" % BATCH_SIZE)
return batch, cti, wti, itt
def __init__(self, args):
self.z_dim = 62
self.sample_num = 100
self.dataset = args.dataset
self.batch_size = args.batch_size
self.input_size = args.input_size
self.CUDA = args.CUDA
self.epoch = args.epoch
self.model_name = args.model_name
self.save_dir = args.save_dir
# self.result_dir = args.result_dir
self.data_loader = dataloader(self.dataset, self.input_size, self.batch_size)
data = self.data_loader.__iter__().__next__()[0]
self.G = Generator(input_dim=self.z_dim, output_dim=data.shape[1], input_size=self.input_size)
# LSGAN: remove the sigmoid layer from the discriminator
self.D = Discriminator(input_channel=data.shape[1], output_dim=1, input_size=self.input_size, sigmoid=False)
self.G_optimizer = optim.Adam(self.G.parameters(), lr=args.LR_G, betas=(args.beta1, args.beta2))
self.D_optimizer = optim.Adam(self.D.parameters(), lr=args.LR_D, betas=(args.beta1, args.beta2))
self.writer = SummaryWriter(self.model_name)
if self.CUDA:
self.G.cuda()
self.D.cuda()
# LSGAN: L2 loss
self.L2_loss = nn.MSELoss().cuda()
else:
self.L2_loss = nn.MSELoss()
self.sample_z_ = torch.rand((self.batch_size, self.z_dim))
if self.CUDA:
self.sample_z_ = self.sample_z_.cuda()
def load_data():
data = dataloader()
batch = []
x_cti = load_tkn_to_idx(sys.argv[2]) # source char_to_idx
x_wti = load_tkn_to_idx(sys.argv[3]) # source word_to_idx
y_wti = load_tkn_to_idx(sys.argv[4]) # target word_to_idx
print("loading %s..." % sys.argv[5])
fo = open(sys.argv[5])
for line in fo:
x, y = line.strip().split("\t")
x = [x.split(":") for x in x.split(" ")]
y = [int(x) for x in y.split(" ")]
xc, xw = zip(*[(list(map(int, xc.split("+"))), int(xw))
for xc, xw in x])
data.append_item(xc=xc, xw=xw, y0=y)
data.append_row()
fo.close()
data.strip()
for _batch in data.split():
xc, xw, y0, lens = _batch.sort()
xc, xw = data.tensor(xc, xw, lens, eos=True)
_, y0 = data.tensor(None, y0, eos=True)
batch.append((xc, xw, y0))
print("data size: %d" % (len(data.y0)))
print("batch size: %d" % BATCH_SIZE)
return batch, x_cti, x_wti, y_wti
def train(net, pth):
print('Train start')
loader = dataloader(minibatch_train, train=True)
loss_function = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(net.parameters(), lr=learning_rate, momentum=0.9)
for epoch in range(1, epoch_num):
for i, data in enumerate(loader):
# get the inputs
(img, of, target) = data
img, target, of = img.cuda(), target.cuda(), of.cuda()
# print(img)
# print(of)
# print(target)
# zero the parameter gradients
optimizer.zero_grad()
# forward + backward + optimize
output = net(img, of)
#print(output_S, target_S)
#print(output_T, target_T)
loss = loss_function(output, target.to(DEVICE, dtype=torch.int64))
loss.backward()
optimizer.step()
print('Net: [{:03}, {:03}] loss:{:.2f}'
.format(epoch, i, loss.item()), end=' ', flush=True)
# get the accuracies
'''
result_S = [int(torch.argmax(output_S, 1)[j]) == int(target_S[j]) for j in range(len(target_S))]
result_T = [int(torch.argmax(output_T, 1)[j]) == int(target_T[j]) for j in range(len(target_T))]
accuracy_S = round((sum(result_S) / len(result_S)) * 100)
accuracy_T = round((sum(result_T) / len(result_T)) * 100)
'''
# print statistics
'''
print('Spatial : [{:03}, {:03}] loss:{:.2f}, accuracy:{:03}%'
.format(epoch, i, loss_S.item(), accuracy_S), end=' ', flush=True)
print('Temporal: [{:03}, {:03}] loss:{:.2f}, accuracy:{:03}%'
.format(epoch, i, loss_T.item(), accuracy_T), end=' ', flush=True)
'''
if i == len(loader):
print()
else:
print(end='\n')
if (epoch % save_net) == 0:
torch.save({'net_state_dict': net.state_dict()}, pth, _use_new_zipfile_serialization=False)
print('Save Finished')
print('Train Finished')
def predict(model, cti, wti, itt, filename):
data = dataloader()
fo = open(filename)
for line in fo:
data.append_row()
line = line.strip()
x0, y0 = re.findall("(.+?)(?:\t(.+))?$", line)[0]
x1 = list(map(normalize, tokenize(x0)))
xc = [[cti[c] if c in cti else UNK_IDX for c in w] for w in x1]
xw = [wti[w] if w in wti else UNK_IDX for w in x1]
data.append_item(x0 = x0, x1 = x1, xc = xc, xw = xw, y0 = y0)
fo.close()
return run_model(model, data, itt)
def predict(filename, model, x_cti, x_wti, y_itw):
data = dataloader()
fo = open(filename)
for x0 in fo:
x0 = x0.strip()
x1 = tokenize(x0, UNIT)
xc = [[x_cti[c] if c in x_cti else UNK_IDX for c in w] for w in x1]
xw = [x_wti[w] if w in x_wti else UNK_IDX for w in x1]
data.append_item(x0, x1, xc, xw)
for _ in range(BEAM_SIZE - 1):
data.append_row()
data.append_item(x0, x1, xc, xw)
data.append_row()
fo.close()
data.strip()
return run_model(model, data, y_itw)
def predict(filename, model, x_cti, x_wti, y_itw):
data = dataloader()
fo = open(filename)
for x0 in fo:
x0 = x0.strip()
y0 = None
if x0.count("\t") == 1:
x0, y0 = x0.split("\t")
x1 = tokenize(x0, UNIT)
xc = [[x_cti.get(c, UNK_IDX) for c in w] for w in x1]
xw = [x_wti.get(w, UNK_IDX) for w in x1]
data.append_item(x0, x1, xc, xw, y0)
for _ in range(BEAM_SIZE - 1):
data.append_row()
data.append_item(x0, x1, xc, xw, y0)
data.append_row()
fo.close()
data.strip()
return run_model(model, data, y_itw)
def predict(filename, model, cti, wti, itt):
data = dataloader()
with open(filename) as fo:
text = fo.read().strip().split("\n" * (HRE + 1))
for block in text:
for x0 in block.split("\n"):
if re.match("\S+/\S+( \S+/\S+)*$", x0): # word/tag
x0, y0 = zip(
*[re.split("/(?=[^/]+$)", x) for x in x0.split(" ")])
x0 = " ".join(x0)
elif re.match("[^\t]+\t\S+$", x0): # sentence \t label
x0, *y0 = x0.split("\t")
else: # no ground truth provided
y0 = []
x1 = tokenize(x0)
xc = [[cti[c] if c in cti else UNK_IDX for c in w] for w in x1]
xw = [wti[w] if w in wti else UNK_IDX for w in x1]
data.append_item(x0, x1, xc, xw, y0)
data.append_row()
data.strip()
return run_model(model, data, itt)
def predict(model, cti, wti, itt, filename):
data = dataloader()
with open(filename) as fo:
text = fo.read().strip().split("\n" * (HRE + 1))
for block in text:
data.append_row()
for x0 in block.split("\n"):
if re.match("\S+/\S+( \S+/\S+)*$", x0): # word/tag
x0, y0 = zip(
*[re.split("/(?=[^/]+$)", x) for x in x0.split(" ")])
x1 = list(map(normalize, x0))
else:
y0 = []
if re.match("[^\t]+\t\S+$", x0): # sentence \t label
x0, *y0 = x0.split("\t")
x0 = tokenize(x0)
x1 = list(map(normalize, x0))
xc = [[cti[c] if c in cti else UNK_IDX for c in w] for w in x1]
xw = [wti[w] if w in wti else UNK_IDX for w in x1]
data.append_item(x0=x0, x1=x1, xc=xc, xw=xw, y0=y0)
return run_model(model, data, itt)
def train_model():
fcn_model.train()
for epoch in epochs:
scheduler.step()
train_data = dataloader(training=True)
train_loader = torch.utils.data.DataLoader(train_data,
batch_size=batch_size_train,
shuffle=True,
num_workers=0)
len_traindata = len(train_loader)
loss_running = 0
for iter, batch in enumerate(train_loader):
optimizer.zero_grad()
inputs = batch['img'].cuda()
labels = batch['mask'].cuda()
outputs = fcn_model(inputs)
loss = criterion(outputs, labels)
loss.backward()
optimizer.step()
loss_running += loss.data.cpu().item() * batch_size_train
print("epoch={},iter={},loss={}".format(epoch, iter,
loss.data.cpu().item()))
del loss
loss_epoch = loss_running / len_traindata
print("~~train data~~~~~~~loss_epoch" + str(epoch) +
"={}".format(loss_epoch))
np.save(score_dir + "\\loss_epoch" + str(epoch), loss_epoch)
del loss_epoch
del loss_running
if epoch >= 10 and epoch % 10 == 0:
torch.save(fcn_model, model_dir + "\\model_epoch" + str(epoch))
with torch.no_grad():
val_model(epoch)
def load_data(args):
data = dataloader()
batch = []
cti = load_tkn_to_idx(args[1]) # char_to_idx
wti = load_tkn_to_idx(args[2]) # word_to_idx
itt = load_idx_to_tkn(args[3]) # idx_to_tkn
print("loading %s..." % args[4])
with open(args[4], "r") as fo:
for line in fo:
line = line.strip()
*x, y = [x.split(":") for x in line.split(" ")]
xc, xw = zip(*[(list(map(int, xc.split("+"))), int(xw))
for xc, xw in x])
data.append_row()
data.append_item(xc=xc, xw=xw, y0=int(y[0]))
for _batch in data.split():
xc, xw, y0, lens = _batch.xc, _batch.xw, _batch.y0, _batch.lens
xc, xw = data.tensor(bc=xc, bw=xw, sos=True, eos=True)
_, y0 = data.tensor(bw=y0)
batch.append((xc, xw, y0))
print("data size: %d" % len(data.y0))
print("batch size: %d" % BATCH_SIZE)
return batch, cti, wti, itt
def predict(filename, model, cti, wti):
data = dataloader()
with open(filename) as fo:
text = fo.read().strip().split("\n" * (HRE + 1))
for block in text:
for x0 in block.split("\n"):
if re.match("[^\t]+\t[0-9]+( [0-9]+)*$", x0):
x0, y0 = x0.split("\t")
y0 = [int(x) for x in y0.split(" ")]
else: # no ground truth provided
y0 = []
x1 = tokenize(x0)
xc = [[cti[c] if c in cti else UNK_IDX for c in w] for w in x1]
xw = [wti[w] if w in wti else UNK_IDX for w in x1]
data.append_item(x0, x1, xc, xw, y0)
for _ in range(BEAM_SIZE - 1):
data.x0.append(data.x0[-1])
data.x1.append(data.x1[-1])
data.xc.append(data.xc[-1])
data.xw.append(data.xw[-1])
data.y0.append(data.y0[-1])
data.append_row()
data.strip()
return run_model(model, data)
else:
print('Creating model with fresh parameters.')
init_op = tf.global_variables_initializer()
sess.run(init_op)
return model, merged, writer
if __name__ == '__main__':
#################### data load ####################################
#with K.get_session():
dataset = ['yelp', 'amazon']
#对原来amazon的截断情况
#trunc = [70000,-1]#对每个数据集取数据的长度，-1则不截断
#对amazon1的截断情况
trunc = [60000, 60000]
dataloader = dataloader(dataset, trunc)
FLAGS.vocab_size = dataloader.vocab_size
word_embeddings = load_embeddings(dataloader, FLAGS.embedding_path)
print("data preparation finished")
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
model, merged, writer = create_model(sess,
FLAGS,
pretrained=False,
data=dataloader,
word_embeddings=word_embeddings)
var_list = model.get_trainable_weights()
sliceNum = 16 #24
n_class = 2
use_gpu = torch.cuda.is_available()
num_gpu = list(range(torch.cuda.device_count()))
print('use_gpu:', use_gpu)
print('num_gpu:', num_gpu)
fcn_model = FCNmodel_3pool(n_class)
fcn_model.cuda()
criterion = nn.BCEWithLogitsLoss()
criterion.cuda()
optimizer = optim.SGD(fcn_model.parameters(), lr=lr, momentum=0.9)
train_data = dataloader(training=True)
train_loader = torch.utils.data.DataLoader(train_data,
batch_size=train_batch_size,
shuffle=False,
num_workers=num_workers)
test_data = dataloader(training=False)
test_loader = torch.utils.data.DataLoader(test_data,
batch_size=test_batch_size,
shuffle=False,
num_workers=num_workers)
fcn_model.train()
for i, batch in enumerate(test_loader):
if i == 0:
def __init__(self, args, SUPERVISED=False):
# parameters
print(SUPERVISED, 'supervised')
self.missing_mixt = args.missing_mixt
self.epoch = args.epoch
self.batch_size = args.batch_size
self.save_dir = args.save_dir
self.result_dir = args.result_dir
self.dataset = args.dataset
self.log_dir = args.log_dir
self.gpu_mode = args.gpu_mode
self.model_name = args.gan_type
self.input_size = args.input_size
self.z_dim = 62
self.SUPERVISED = SUPERVISED # if it is true, label info is directly used for code
self.len_discrete_code = 7 # categorical distribution (i.e. label)
self.len_continuous_code = 2 # gaussian distribution (e.g. rotation, thickness)
self.sample_num = self.len_discrete_code**2
# load dataset
#remove samples for zero-shot
self.data_loader = dataloader(self.dataset, self.input_size,
self.batch_size, self.missing_mixt)
self.data_batch = self.data_loader.__iter__().__next__()
data = self.data_batch[0]
labels = self.data_batch[1]
# networks init
self.G = generator(input_dim=self.z_dim,
output_dim=data.shape[1],
input_size=self.input_size,
len_discrete_code=self.len_discrete_code,
len_continuous_code=self.len_continuous_code)
self.D = discriminator(input_dim=data.shape[1],
output_dim=1,
input_size=self.input_size,
len_discrete_code=self.len_discrete_code,
len_continuous_code=self.len_continuous_code)
self.G_optimizer = optim.Adam(self.G.parameters(),
lr=args.lrG,
betas=(args.beta1, args.beta2))
self.D_optimizer = optim.Adam(self.D.parameters(),
lr=args.lrD,
betas=(args.beta1, args.beta2))
self.info_optimizer = optim.Adam(itertools.chain(
self.G.parameters(), self.D.parameters()),
lr=args.lrD,
betas=(args.beta1, args.beta2))
if self.gpu_mode:
self.G.cpu()
self.D.cpu()
self.BCE_loss = nn.BCELoss().cpu()
self.CE_loss = nn.CrossEntropyLoss().cpu()
self.MSE_loss = nn.MSELoss().cpu()
else:
self.BCE_loss = nn.BCELoss()
self.CE_loss = nn.CrossEntropyLoss()
self.MSE_loss = nn.MSELoss()
print('---------- Networks architecture -------------')
utils.print_network(self.G)
utils.print_network(self.D)
print('-----------------------------------------------')
# fixed noise & condition
self.sample_z_ = torch.zeros((self.sample_num, self.z_dim))
for i in range(self.len_discrete_code):
self.sample_z_[i * self.len_discrete_code] = torch.rand(
1, self.z_dim)
for j in range(1, self.len_discrete_code):
self.sample_z_[i * self.len_discrete_code +
j] = self.sample_z_[i * self.len_discrete_code]
temp = torch.zeros((self.len_discrete_code, 1))
for i in range(self.len_discrete_code):
temp[i, 0] = i
temp_y = torch.zeros((self.sample_num, 1))
for i in range(self.len_discrete_code):
temp_y[i * self.len_discrete_code:(i + 1) *
self.len_discrete_code] = temp
self.sample_y_ = torch.zeros(
(self.sample_num,
self.len_discrete_code)).scatter_(1,
temp_y.type(torch.LongTensor),
1)
self.sample_c_ = torch.zeros(
(self.sample_num, self.len_continuous_code))
# manipulating two continuous code
self.sample_z2_ = torch.rand(
(1, self.z_dim)).expand(self.sample_num, self.z_dim)
self.sample_y2_ = torch.zeros(self.sample_num, self.len_discrete_code)
self.sample_y2_[:, 0] = 1
temp_c = torch.linspace(-1, 1, 7)
self.sample_c2_ = torch.zeros((self.sample_num, 2))
for i in range(self.len_discrete_code):
for j in range(self.len_discrete_code):
self.sample_c2_[i * self.len_discrete_code + j, 0] = temp_c[i]
self.sample_c2_[i * self.len_discrete_code + j, 1] = temp_c[j]
if self.gpu_mode:
self.sample_z_, self.sample_y_, self.sample_c_, self.sample_z2_, self.sample_y2_, self.sample_c2_ = \
self.sample_z_.cpu(), self.sample_y_.cpu(), self.sample_c_.cpu(), self.sample_z2_.cpu(), \
self.sample_y2_.cpu(), self.sample_c2_.cpu()
from model_factory import *
import argparse
import warnings
from sklearn.model_selection import cross_val_score
warnings.filterwarnings("ignore")
pareser = argparse.ArgumentParser()
pareser.add_argument("--model_name", type=str, help='The classifier name')
pareser.add_argument('--train_path', type=str, default='./data/adult.data', help='The path of train data')
pareser.add_argument('--test_path', type=str, default='./data/adult.test', help='The path of test data')
args = pareser.parse_args()
if __name__ == "__main__":
train_path = args.train_path
test_path = args.test_path
dataloader = dataloader(train_path, test_path)
print('==> processing test')
dataloader.process_test()
print("==> generating train and test data")
train_data, train_y, test_data, test_y = dataloader.gen_dataset()
def score_LR(model, test_data, test_y):
pred = list(map(lambda x: 0 if x < 0.5 else 1, list(model.predict(test_data))))
label = test_y
return list((np.array(pred) - np.array(label))).count(0) / len(pred)
classifier_factory = classifier_model(args.model_name)
print('training ' + args.model_name)
clf = classifier_factory.get_classifier_fn()
clf.fit(train_data.as_matrix(), train_y.as_matrix())
labels = clf.predict(test_data)
lists = open('data list').readlines()
lists = [elm.split('.')[0] for elm in lists]
splits = 'data split path'
batch_size = 64
correlations = 0.
testloss = 0.
n_epoch = 10
for sbj in range(21):
for splno in range(1, 6):
train_data = lists[sbj] + '.npy'
splits = '/mnt/dataset/eeg/seedvig/PSD_LDS_' + str(splno) + '/'
net = network.cnnvig().cuda()
trainset = data.DataLoader(dataloader(train_data,
d_type='train',
data_dir=splits),
batch_size=batch_size,
shuffle=True)
testset = data.DataLoader(dataloader(train_data,
d_type='test',
data_dir=splits),
batch_size=177)
criterion = torch.nn.MSELoss(size_average=False).cuda()
optimizer = torch.optim.Adam(net.parameters(), lr=0.0001)
save_name = '/mnt/EEG/SEED-VIG/model/psd_lds_' + lists[
sbj] + '_' + str(splno) + '.pth'
for ep in range(n_epoch):
net.train()
def main():
parse = argparse.ArgumentParser()
# ---------- environment setting: which gpu -------
parse.add_argument('-gpu',
'--gpu',
type=str,
default='0',
help='which gpu to use: 0 or 1')
parse.add_argument('-folder_name',
'--folder_name',
type=str,
default='datasets/didi-data/data/')
parse.add_argument('-output_folder_name',
'--output_folder_name',
type=str,
default='output/didi-data/data/')
# ---------- input/output settings -------
parse.add_argument('-input_steps',
'--input_steps',
type=int,
default=6,
help='number of input steps')
# ---------- model ----------
parse.add_argument('-model',
'--model',
type=str,
default='GCN',
help='model: GCN, ConvLSTM, flow_ConvLSTM')
parse.add_argument('-num_layers',
'--num_layers',
type=int,
default=2,
help='number of layers in model')
parse.add_argument('-num_units',
'--num_units',
type=int,
default=64,
help='dim of hidden states')
parse.add_argument('-kernel_size',
'--kernel_size',
type=int,
default=3,
help='kernel size in convolutional operations')
#
parse.add_argument(
'-dy_adj',
'--dy_adj',
type=int,
default=1,
help=
'whether to use dynamic adjacent matrix for lower feature extraction layer'
)
parse.add_argument(
'-dy_filter',
'--dy_filter',
type=int,
default=0,
help='whether to use dynamic filter generate region-specific filter ')
parse.add_argument(
'-att_dynamic_adj',
'--att_dynamic_adj',
type=int,
default=0,
help='whether to use dynamic adjacent matrix in attention parts')
#
parse.add_argument('-model_save',
'--model_save',
type=str,
default='gcn',
help='folder name to save model')
parse.add_argument('-pretrained_model',
'--pretrained_model_path',
type=str,
default=None,
help='path to the pretrained model')
# ---------- params for CNN ------------
parse.add_argument('-num_filters',
'--num_filters',
type=int,
default=32,
help='number of filters in CNN')
parse.add_argument('-pooling_units',
'--pooling_units',
type=int,
default=64,
help='number of pooling units')
parse.add_argument('-dropout_keep_prob',
'--dropout_keep_prob',
type=float,
default=0.5,
help='keep probability in dropout layer')
# ---------- training parameters --------
parse.add_argument('-n_epochs',
'--n_epochs',
type=int,
default=20,
help='number of epochs')
parse.add_argument('-batch_size',
'--batch_size',
type=int,
default=8,
help='batch size for training')
parse.add_argument('-show_batches',
'--show_batches',
type=int,
default=100,
help='show how many batches have been processed.')
parse.add_argument('-lr',
'--learning_rate',
type=float,
default=0.0002,
help='learning rate')
parse.add_argument('-update_rule',
'--update_rule',
type=str,
default='adam',
help='update rule')
# ---------- train or predict -------
parse.add_argument('-train',
'--train',
type=int,
default=1,
help='whether to train')
parse.add_argument('-test', '--test', type=int, default=0, help='if test')
#
args = parse.parse_args()
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
print('load train, test data...')
#
# train: 20161101 - 20161125
# validate: 20161126 - 20161127
# test: 20161128 - 20161130
split = [2400, 192, 288]
data, train_data, val_data, test_data = load_npy_data(
filename=[args.folder_name + 'cd_didi_data.npy'], split=split)
# data: [num, station_num, 2]
print(data.shape)
#
if 'GCN' in args.model or 'FC' in args.model:
dataloader = DataLoader_graph
else:
data = np.reshape(data, (-1, 20, 20, 2))
train_data = np.reshape(train_data, (-1, 20, 20, 2))
val_data = np.reshape(val_data, (-1, 20, 20, 2))
test_data = np.reshape(test_data, (-1, 20, 20, 2))
# data: [num, height, width, 2]
print(data.shape)
#
dataloader = DataLoader_map
#
map_size = data.shape[1:-1]
input_dim = data.shape[-1]
num_station = np.prod(data.shape[1:-1])
#
f_data, train_f_data, val_f_data, test_f_data = load_npy_data(
[args.folder_name + 'cd_didi_flow_in.npy'], split=split)
print(len(f_data))
print('preprocess train/val/test flow data...')
#f_preprocessing = StandardScaler()
f_preprocessing = MinMaxNormalization01()
f_preprocessing.fit(train_f_data)
train_f_data = f_preprocessing.transform(train_f_data)
val_f_data = f_preprocessing.transform(val_f_data)
test_f_data = f_preprocessing.transform(test_f_data)
print('preprocess train/val/test data...')
# pre_process = StandardScaler()
pre_process = MinMaxNormalization01()
pre_process.fit(train_data)
train_data = pre_process.transform(train_data)
val_data = pre_process.transform(val_data)
test_data = pre_process.transform(test_data)
#
print('number of station: %d' % num_station)
#
train_loader = dataloader(train_data,
train_f_data,
args.input_steps,
flow_format='identity')
val_loader = dataloader(val_data,
val_f_data,
args.input_steps,
flow_format='identity')
test_loader = dataloader(test_data,
test_f_data,
args.input_steps,
flow_format='identity')
# f_adj_mx = None
if os.path.isfile(args.folder_name + 'f_adj_mx.npy'):
f_adj_mx = np.load(args.folder_name + 'f_adj_mx.npy')
else:
f_adj_mx = train_loader.get_flow_adj_mx()
np.save(args.folder_name + 'f_adj_mx.npy', f_adj_mx)
#
# if args.model == 'FC_LSTM':
# model = FC_LSTM(num_station, args.input_steps,
# num_layers=args.num_layers, num_units=args.num_units,
# batch_size=args.batch_size)
if args.model == 'FC_GRU':
model = FC_GRU(num_station,
args.input_steps,
num_layers=args.num_layers,
num_units=args.num_units,
batch_size=args.batch_size)
if args.model == 'GCN':
model = GCN(num_station,
args.input_steps,
num_layers=args.num_layers,
num_units=args.num_units,
dy_adj=args.dy_adj,
dy_filter=args.dy_filter,
f_adj_mx=f_adj_mx,
batch_size=args.batch_size)
if args.model == 'ConvGRU':
model = ConvGRU(input_shape=[map_size[0], map_size[1], input_dim],
input_steps=args.input_steps,
num_layers=args.num_layers,
num_units=args.num_units,
kernel_shape=[args.kernel_size, args.kernel_size],
batch_size=args.batch_size)
# if args.model == 'flow_ConvGRU':
# model = flow_ConvGRU(input_shape=[20, 20, input_dim], input_steps=args.input_steps,
# num_layers=args.num_layers, num_units=args.num_units, kernel_shape=[args.kernel_size, args.kernel_size],
# f_adj_mx=f_adj_mx,
# batch_size=args.batch_size)
if args.model == 'Coupled_ConvGRU':
model = CoupledConvGRU(
input_shape=[20, 20, input_dim],
input_steps=args.input_steps,
num_layers=args.num_layers,
num_units=args.num_units,
kernel_shape=[args.kernel_size, args.kernel_size],
batch_size=args.batch_size)
##
# flow_ConvGRU_2 is Stack_ConvGRU with 2 conv layers.
if args.model == 'flow_ConvGRU_2':
model = flow_ConvGRU_2(
input_shape=[20, 20, input_dim],
input_steps=args.input_steps,
num_layers=args.num_layers,
num_units=args.num_units,
kernel_shape=[args.kernel_size, args.kernel_size],
f_adj_mx=f_adj_mx,
batch_size=args.batch_size)
if args.model == 'Stack_ConvGRU':
model = Stack_ConvGRU(
input_shape=[20, 20, input_dim],
input_steps=args.input_steps,
num_layers=args.num_layers,
num_units=args.num_units,
kernel_shape=[args.kernel_size, args.kernel_size],
f_adj_mx=f_adj_mx,
batch_size=args.batch_size)
#
model_path = os.path.join(args.output_folder_name, 'model_save',
args.model_save)
if not os.path.exists(model_path):
os.makedirs(model_path)
#model_path = os.path.join(args.folder_name, 'model_save', args.model_save)
solver = ModelSolver(
model,
train_loader,
val_loader,
test_loader,
pre_process,
batch_size=args.batch_size,
show_batches=args.show_batches,
n_epochs=args.n_epochs,
pretrained_model=args.pretrained_model_path,
update_rule=args.update_rule,
learning_rate=args.learning_rate,
model_path=model_path,
)
results_path = os.path.join(model_path, 'results')
if not os.path.exists(results_path):
os.makedirs(results_path)
if args.train:
print('==================== begin training ======================')
test_target, test_prediction = solver.train(
os.path.join(model_path, 'out'))
np.save(os.path.join(results_path, 'test_target.npy'), test_target)
np.save(os.path.join(results_path, 'test_prediction.npy'),
test_prediction)
if args.test:
print('==================== begin test ==========================')
test_target, test_prediction = solver.test()
np.save(os.path.join(results_path, 'test_target.npy'), test_target)
np.save(os.path.join(results_path, 'test_prediction.npy'),
test_prediction)
#main.py
#Created by ImKe on 2020/2/28
#Copyright © 2020 ImKe. All rights reserved.
import time
import matplotlib.pyplot as plt
from SVT import *
from dataloader import *
#parameters
num_of_entries = 1000
iter_list = [50,100, 200, 300, 500, 700, 900]
plot = True
if __name__ == '__main__':
start = time.clock()
M_original = dataloader('ratings')
M1 = M_original[:num_of_entries]
rmse1 = []
s = 1
for i in iter_list:
rmse1.append(SVT(M1, i)[-1])
print("The %d iteration is completed"%s)
s += 1
if plot:
plt.plot(iter_list, rmse1, '*-')
plt.xlabel('Number of iterations')
plt.ylabel('RMSE')
plt.show()
print("Time:%f s" %(time.clock() - start))
from dataloader import *
import torch.optim as optim
from torch.autograd import Variable
data_dir = '../data/'
data_path = 'dlist.txt' # data list
num_class = 6
learning_rate = 0.001
model_path = '../model/' # path to save model
model_name_to_save = 'eeg_6class_simple_gru.pth' # name of model to save
use_saved_model = False
batch_size = 32
nEpoch = 100
prev_acc = 0
trainset = data.DataLoader(dataloader(data_path, dtype='train'),
batch_size=batch_size,
shuffle=True)
valset = data.DataLoader(dataloader(data_path, dtype='test'),
batch_size=batch_size)
net = RNN_Encoder(num_class, model_type='GRU', hidden_dim=70).cuda()
print(net)
criterion = torch.nn.CrossEntropyLoss().cuda()
optimizer = torch.optim.Adam(net.parameters(), lr=learning_rate)
if use_saved_model:
print('saved model loaded')
net.load_state_dict(torch.load(model_path + model_name_to_save))
print('start train/test')
'wc3': tf.get_variable('W2', shape=(3,3,64,128), initializer=tf.contrib.layers.xavier_initializer()),
'wd1': tf.get_variable('W3', shape=(4*4*128,128), initializer=tf.contrib.layers.xavier_initializer()),
'out': tf.get_variable('W6', shape=(128,num_classes), initializer=tf.contrib.layers.xavier_initializer()),
}
biases = {
'bc1': tf.get_variable('B0', shape=(32), initializer=tf.contrib.layers.xavier_initializer()),
'bc2': tf.get_variable('B1', shape=(64), initializer=tf.contrib.layers.xavier_initializer()),
'bc3': tf.get_variable('B2', shape=(128), initializer=tf.contrib.layers.xavier_initializer()),
'bd1': tf.get_variable('B3', shape=(128), initializer=tf.contrib.layers.xavier_initializer()),
'out': tf.get_variable('B4', shape=(10), initializer=tf.contrib.layers.xavier_initializer()),
}
return weights, biases
dl = dataloader(data_root,image_size,batch_size,mode, num_classes)
input_batch = tf.placeholder("float", [None, 32,32,3])
ground_truth = tf.placeholder("float", [None, num_classes])
weight, bias = weight_and_biases()
pred = CNN(input_batch, weight, bias)
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=ground_truth))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)
init = tf.global_variables_initializer()
#correct_prediction = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1))
#accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
with tf.Session() as sess:
n_class = 3
batch_size_train = 10
batch_size_test = 1
epochs = range(0, 3001)
lr = 1e-4
momentum = 0.9
L2_factor = 1e-5
L1_factor = 1e-5
step_size = 2000
gamma = 0.1
root = 'E:/project_chop/project_leglength/code_for_training/dataTrain/data5_deep_learning/segmentation/'
model_dir = root + "models_3_segmentation"
score_dir = root + "scores_3_segmentation" #os.path.join(model_dir,configs)
test_data = dataloader(training=False)
test_loader = torch.utils.data.DataLoader(test_data,
batch_size=batch_size_test,
shuffle=False,
num_workers=0)
len_testdata = len(test_loader)
dir_model = model_dir + "\\model_epoch100"
fcn_model = torch.load(dir_model)
#fcn_model = FCNmodel_3pool(n_class)
cudaNum = torch.device('cuda:1')
fcn_model.cuda(cudaNum)
criterion = nn.CrossEntropyLoss()
criterion.cuda(cudaNum)
optimizer = optim.SGD(fcn_model.parameters(),