def __init__(self, c):
self.c = c
self.nnet = nnet.NNet(c.num_kernels, c.num_res_blocks)
self.nnet.cuda()
self.evaluator = evaluateAI.Evaluator('random', c.num_eval_games,
c.max_moves)
self.buffer = Buffer(c)
if c.load:
self.rm = jsonManager.load(c.run_manager_file_name)
self.rm.c = c
nnet_file_name = self.rm.number_file_name(c.nnet_file_name)
# We load a nnet only if there exists one (ie. with a non zero number).
if self.rm.get_number_file_name(nnet_file_name) > 0:
self.nnet.load_state_dict(torch.load(nnet_file_name))
self.buffer.load()
self.buffer.c = c
else:
self.rm = runManager.RunManager(c)
# Evaluation of the initial AI.
if self.rm.iter_count == 0:
self.evaluation()
def run():
print "Loading data..."
# load training data
trainImages,trainLabels=dl.load_mnist_train()
imDim = trainImages.shape[0]
inputDim = 50
outputDim = 10
layerSizes = [16]*2
trainImages = trainImages.reshape(imDim**2,-1)
pcer = pc.Preprocess()
pcer.computePCA(trainImages)
whitenedTrain = pcer.whiten(trainImages, inputDim)
minibatch = whitenedTrain.shape[1]
print "minibatch size: %d" % (minibatch)
epochs = 10000
stepSize = 1e-2
nn = nnet.NNet(inputDim,outputDim,layerSizes,minibatch)
nn.initParams()
SGD = sgd.SGD(nn,alpha=stepSize,minibatch=minibatch)
for e in range(epochs):
print "Running epoch %d"%e
SGD.run(whitenedTrain,trainLabels)
SGD.dumptrace()
def run():
print "Loading data..."
# load training data
trainImages,trainLabels=data_loader.load_mnist_train()
imDim = trainImages.shape[0]
inputDim = imDim**2
outputDim = 10
layerSizes = [4048]*6
trainImages = trainImages.reshape(inputDim,-1)
mbSize = 256
nn = nnet.NNet(inputDim,outputDim,layerSizes,mbSize)
nn.initParams()
epochs = 2
SGD = sgd.SGD(nn,epochs=20,alpha=1e-2,minibatch=mbSize,gpu=True)
# run SGD loop
print "Training..."
SGD.run(trainImages,trainLabels)
def random(n_units, mean=None):
h_network = nnet.random_linear_then_tanh_chain(n_units[:-1])
mean_network = nnet.Linear.random(n_units[-2], n_units[-1])
if mean is not None:
mean_network.b.set_value(mean.astype(floatX))
sigma_network = nnet.NNet().add_layer(
nnet.Linear.random(n_units[-2],
n_units[-1])).add_layer(nnet.Exponential())
return GaussianSampler(h_network, mean_network, sigma_network)
def main():
config.init_config()
print('go to model')
print '*' * 80
_log_file = open(
config.LOG_FILE_PRE +
time.strftime("_%Y_%m_%d_%H%M%S", time.localtime()), 'w')
# log configuration.
config.log_config(_log_file)
# initialize model
weights_path = None
if config.MODE == 2:
if config.DATASET == 'WSJ0':
weights_path = './_tmp_weights/ASAM_WSJ0_weight_00031.h5'
elif config.DATASET == 'THCHS-30':
weights_path = './_tmp_weights/ASAM_THCHS30_weight_00034.h5'
dl4ss_model = nnet.NNet(_log_file, weights_path)
if config.MODE == 1:
print 'Start to train model ...'
_log_file.write('Start to train model ...\n')
dl4ss_model.train()
print 'valid spk number: 2'
_log_file.write('valid spk number: 2\n')
dl4ss_model.predict(config.VALID_LIST, spk_num=2)
print 'test spk number: 2'
_log_file.write('test spk number: 2\n')
dl4ss_model.predict(config.TEST_LIST, spk_num=2)
print 'test spk number: 3'
_log_file.write('test spk number: 3\n')
dl4ss_model.predict(config.TEST_LIST, spk_num=3)
print 'test spk number: 2 with bg noise'
_log_file.write('test spk number: 2 with bg noise\n')
dl4ss_model.predict(config.TEST_LIST, spk_num=2, add_bgd_noise=True)
for supp_time in [0.25, 0.5, 1, 2, 4, 8, 16, 32]:
print 'unk spk and supplemental wav span: %02d' % supp_time
_log_file.write('unk spk and supplemental wav span: %02d\n' %
supp_time)
dl4ss_model.predict(config.UNK_LIST,
spk_num=2,
unk_spk=True,
supp_time=supp_time)
else:
print 'Wrong mode: %s' % config.MODE
_log_file.write('Wrong mode: %s\n' % config.MODE)
_log_file.close()
def main():
# ############## Test,测试区 ############
# ############## Test,测试区 ############
config.init_config()
print('go to model')
print '*' * 80
_log_file = open(
config.LOG_FILE_PRE +
time.strftime("_%Y_%m_%d_%H%M%S", time.localtime()), 'w')
# 记录参数
config.log_config(_log_file)
# 初始化网络 initialize model
weights_path = './_tmp_weights/WSJ0_weight_00021.h5'
weights_path = './_tmp_weights/WSJ0_weight_00150_forImages.h5'
# weights_path = None
dl4ss_model = nnet.NNet(_log_file, weights_path)
if config.MODE == 1:
print 'still train'
dl4ss_model.train()
elif (config.MODE == 2) and (weights_path is not None):
# 在训练好的模型上测试TEST
# print 'valid spk number: 2'
# _log_file.write('valid spk number: 2\n')
# dl4ss_model.predict(config.VALID_LIST, spk_num=2)
print 'test spk number: 2'
_log_file.write('test spk number: 2\n')
dl4ss_model.predict(config.TEST_LIST, spk_num=2)
# print 'tes spk number: 3'
# _log_file.write('test spk number: 3\n')
# dl4ss_model.predict(config.TEST_LIST, spk_num=3)
print 'test spk number: 2 with bg noise'
_log_file.write('test spk number: 2 with bg noise\n')
dl4ss_model.predict(config.TEST_LIST, spk_num=2, add_bgd_noise=True)
# 在训练好的模型上测试UNK
# for supp_time in [0.25, 0.5, 1, 2, 4, 8, 16, 32]:
# for supp_time in [0.25, 0.5, 1, 2, 4, 8, 16, 32]:
# print 'unk spk and supplemental wav span: %02d' % supp_time
# _log_file.write('unk spk and supplemental wav span: %02d\n' % supp_time)
# dl4ss_model.predict(config.UNK_LIST, spk_num=2, unk_spk=True, supp_time=supp_time)
# else:
# print 'Wrong mode: %s' % config.MODE
# _log_file.write('Wrong mode: %s\n' % config.MODE)
_log_file.close()
def test(opts):
import editDistance as ed
print "Testing model %s" % opts.inFile
phone_map = get_phone_map_swbd()
with open(opts.inFile, 'r') as fid:
old_opts = pickle.load(fid)
_ = pickle.load(fid)
_ = pickle.load(fid)
loader = dl.DataLoader(opts.dataDir, old_opts.rawDim,
old_opts.inputDim)
if 'layers' not in dir(old_opts):
old_opts.layers = [old_opts.layerSize] * old_opts.numLayers
nn = nnet.NNet(old_opts.inputDim,
old_opts.outputDim,
old_opts.layers,
train=False)
nn.initParams()
nn.fromFile(fid)
totdist = numphones = 0
fid = open('hyp.txt', 'w')
for i in range(1, opts.numFiles + 1):
data_dict, alis, keys, sizes = loader.loadDataFileDict(i)
for k in keys:
gp.free_reuse_cache()
hyp = nn.costAndGrad(data_dict[k])
hyp = [phone_map[h] for h in hyp]
ref = [phone_map[int(r)] for r in alis[k]]
dist, ins, dels, subs, corr = ed.edit_distance(ref, hyp)
print "Distance %d/%d" % (dist, len(ref))
fid.write(k + ' ' + ' '.join(hyp) + '\n')
totdist += dist
numphones += len(alis[k])
fid.close()
print "PER : %f" % (100 * totdist / float(numphones))
def __init__(self, horizon=1):
super(NeuralNetwork, self).__init__(window_size=1, horizon=horizon)
import nnet
self.nnet_instance = nnet.NNet()
self.nnet_instance.compile_model()
def __init__(self, param):
self.param = param
def costAndGradVec(self, param):
return np.sum(np.array(param)**2), 2 * np.array(param)
def paramVec(self):
return self.param
if __name__ == '__main__':
# gcc = gradcheckCheck(np.random.rand(10,1))
# gradcheck(gcc)
inputDim = 5
outputDim = 10
layerSizes = [3, 3]
mbSize = 5
# fake data
data = gp.rand(inputDim, mbSize)
import random
labels = [random.randint(0, 9)] * mbSize
# make nnet
nn = nnet.NNet(inputDim, outputDim, layerSizes, mbSize, train=True)
nn.initParams()
# run
gradcheck(nn, 1000, data, labels)
def load_nnet(self):
self.nnet = nnet.NNet(self.c.num_kernels, self.c.num_res_blocks)
self.nnet.cuda()
self.nnet.load_state_dict(torch.load(self.c.nnet_file_name))
def main():
pa = parameters.Parameters()
type_exp = 'pg_re' # 'pg_su' 'pg_su_compact' 'v_su', 'pg_v_re', 'pg_re', q_re', 'test'
pg_resume = None
v_resume = None
q_resume = None
log = None
render = False
plot = False
try:
opts, args = getopt.getopt(sys.argv[1:], "hi:o:", [
"exp_type=", "num_res=", "num_nw=", "simu_len=", "num_ex=",
"num_seq_per_batch=", "eps_max_len=", "num_epochs=",
"time_horizon=", "res_slot=", "max_job_len=", "max_job_size=",
"new_job_rate=", "dist=", "lr_rate=", "ba_size=", "pg_re=",
"v_re=", "q_re=", "out_freq=", "ofile=", "log=", "render=",
"unseen=", "plot="
])
except getopt.GetoptError:
script_usage()
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
script_usage()
sys.exit()
elif opt in ("-e", "--exp_type"):
type_exp = arg
elif opt in ("-n", "--num_res"):
pa.num_resources = int(arg)
elif opt in ("-w", "--num_nw"):
pa.num_nw = int(arg)
elif opt in ("-s", "--simu_len"):
pa.simu_len = int(arg)
elif opt in ("-n", "--num_ex"):
pa.num_ex = int(arg)
elif opt in ("-sp", "--num_seq_per_batch"):
pa.num_seq_per_batch = int(arg)
elif opt in ("-el", "--eps_max_len"):
pa.episode_max_length = int(arg)
elif opt in ("-ne", "--num_epochs"):
pa.num_epochs = int(arg)
elif opt in ("-t", "--time_horizon"):
pa.time_horizon = int(arg)
elif opt in ("-rs", "--res_slot"):
pa.res_slot = int(arg)
elif opt in ("-ml", "--max_job_len"):
pa.max_job_len = int(arg)
elif opt in ("-ms", "--max_job_size"):
pa.max_job_size = int(arg)
elif opt in ("-nr", "--new_job_rate"):
pa.new_job_rate = float(arg)
elif opt in ("-d", "--dist"):
pa.discount = float(arg)
elif opt in ("-l", "--lr_rate"):
pa.lr_rate = float(arg)
elif opt in ("-b", "--ba_size"):
pa.batch_size = int(arg)
elif opt in ("-p", "--pg_re"):
pg_resume = arg
elif opt in ("-v", "--v_re"):
v_resume = arg
elif opt in ("-q", "--q_re"):
q_resume = arg
elif opt in ("-f", "--out_freq"):
pa.output_freq = int(arg)
elif opt in ("-o", "--ofile"):
pa.output_filename = arg
elif opt in ("-lg", "--log"):
log = arg
elif opt in ("-r", "--render"):
render = (arg == 'True')
elif opt in ("-pl", "--plot"):
plot = (arg == 'True')
elif opt in ("-u", "--unseen"):
pa.generate_unseen = (arg == 'True')
else:
script_usage()
sys.exit()
if log is not None:
orig_stdout = sys.stdout
f = open(log, 'w')
sys.stdout = f
if pg_resume is None:
print("PG resume is empty!")
sys.exit(1)
pa.compute_dependent_parameters()
repre = 'image'
end = 'all_done'
env = environment.Env(pa, render=render, repre=repre, end=end)
pg_learner = pg_network.PGLearner(pa)
net_handle = open(pg_resume, 'rb')
net_params = cPickle.load(net_handle)
pg_learner.set_net_params(net_params)
outputFileName = pa.output_filename + '_' + \
ntpath.basename(pg_resume) + '_test.pkl'
pg_learner.write_net_to_nnet(outputFileName)
nnetFilename = outputFileName + '.nnet'
r = nnet.NNet(nnetFilename)
smallHigherBound = 1.0e-10
smallLowerBound = -1.0e-10
for wIdx, w in enumerate(r.weights):
smallRows = np.all((w <= smallHigherBound) & (w >= smallLowerBound),
axis=1)
smallRowsIndices = np.where(smallRows == True)
for row in smallRowsIndices[0]:
rowBias = r.biases[wIdx][row]
#add it's bias to all biasses of next layer since we assume fully connected
if ((wIdx + 1) < len(r.biases)):
r.biases[wIdx + 1] = r.biases[wIdx + 1] + rowBias
#Now we delete the lines with 'dead' neurons
r.weights[wIdx] = np.delete(w, smallRowsIndices[0], axis=0)
if ((wIdx + 1) < len(r.weights)):
r.weights[wIdx + 1] = np.delete(r.weights[wIdx + 1],
smallRowsIndices[0],
axis=1)
r.biases[wIdx] = np.delete(r.biases[wIdx], smallRowsIndices[0], axis=0)
# now we export the file once again after the 'dead' neuron filtration
for wIdx, w in enumerate(r.weights):
r.weights[wIdx] = r.weights[wIdx].transpose()
writeNNet.writeNNet(r.weights, r.biases, r.mins, r.maxes, r.means,
r.ranges, outputFileName + '_cleaned_' + '.nnet')
if log is not None:
sys.stdout = orig_stdout
f.close()
def run(args=None):
usage = "usage : %prog [options]"
parser = optparse.OptionParser(usage=usage)
parser.add_option("--test",
action="store_true",
dest="test",
default=False)
# Architecture
parser.add_option("--layers",
dest="layers",
type="string",
default="100,100",
help="layer1size,layer2size,...,layernsize")
parser.add_option("--temporal_layer",
dest="temporalLayer",
type="int",
default=-1)
# Optimization
parser.add_option("--optimizer",
dest="optimizer",
type="string",
default="momentum")
parser.add_option("--momentum", dest="momentum", type="float", default=0.9)
parser.add_option("--epochs", dest="epochs", type="int", default=1)
parser.add_option("--step", dest="step", type="float", default=1e-4)
parser.add_option(
"--anneal",
dest="anneal",
type="float",
default=1,
help="Sets (learning rate := learning rate / anneal) after each epoch."
)
# Data
parser.add_option(
"--dataDir",
dest="dataDir",
type="string",
default=
"/scail/group/deeplearning/speech/awni/kaldi-stanford/kaldi-trunk/egs/swbd/s5b/exp/train_ctc/"
)
parser.add_option("--numFiles", dest="numFiles", type="int", default=384)
parser.add_option("--inputDim",
dest="inputDim",
type="int",
default=41 * 15)
parser.add_option("--rawDim", dest="rawDim", type="int", default=41 * 15)
parser.add_option("--outputDim", dest="outputDim", type="int", default=34)
parser.add_option("--outFile",
dest="outFile",
type="string",
default="models/test.bin")
parser.add_option("--inFile", dest="inFile", type="string", default=None)
(opts, args) = parser.parse_args(args)
opts.layers = [int(l) for l in opts.layers.split(',')]
# Testing
if opts.test:
test(opts)
return
loader = dl.DataLoader(opts.dataDir, opts.rawDim, opts.inputDim)
#NOTE at some point we need to unify the nnet and rnnet modules
nn = None
if opts.temporalLayer > 0:
nn = rnnet.RNNet(opts.inputDim, opts.outputDim, opts.layers,
opts.temporalLayer)
else:
nn = nnet.NNet(opts.inputDim, opts.outputDim, opts.layers)
nn.initParams()
# Load model if exists
if opts.inFile is not None:
with open(opts.inFile, 'r') as fid:
_ = pickle.load(fid)
_ = pickle.load(fid)
_ = pickle.load(fid)
nn.fromFile(fid)
SGD = sgd.SGD(nn,
alpha=opts.step,
optimizer=opts.optimizer,
momentum=opts.momentum)
# Setup some random keys for tracing
with open('randKeys.bin', 'r') as fid:
traceK = pickle.load(fid)
for k in traceK:
nn.hist[k] = []
# write initial model to disk
with open(opts.outFile, 'w') as fid:
pickle.dump(opts, fid)
pickle.dump(SGD.costt, fid)
pickle.dump(nn.hist, fid)
nn.toFile(fid)
# Training
import time
for _ in range(opts.epochs):
for i in np.random.permutation(opts.numFiles) + 1:
start = time.time()
data_dict, alis, keys, sizes = loader.loadDataFileDict(i)
SGD.run_seq(data_dict, alis, keys, sizes)
end = time.time()
print "File time %f" % (end - start)
# Save anneal after epoch
SGD.alpha = SGD.alpha / opts.anneal
with open(opts.outFile, 'w') as fid:
pickle.dump(opts, fid)
pickle.dump(SGD.costt, fid)
pickle.dump(nn.hist, fid)
nn.toFile(fid)
if __name__ == '__main__':
# gcc = gradcheckCheck(np.random.rand(10,1))
# gradcheck(gcc)
inputDim = 5
numPhones = 6
outputDim = numPhones + 1
seq_len_out = 4
seq_len_in = 10
# can't output more symbols than input times
assert seq_len_in > seq_len_out
layerSizes = [10, 10]
# make sure seq labels do not have '0' which is our blank index
label_seq = np.floor(np.random.rand(seq_len_out, 1) * numPhones)
label_seq = 1 + label_seq.astype(np.int32)
data = np.random.randn(inputDim, seq_len_in)
# make nnet
nn = nnet.NNet(inputDim, outputDim, layerSizes, seq_len_in, train=True)
nn.initParams()
# run
cost, grad = nn.costAndGrad(data, label_seq)
print cost
# run
gradcheck(nn, 1000, data, label_seq)
import numpy as np
import mnist
import nnet
if __name__ == '__main__':
trX, teX, trY, teY = mnist.mnist()
net = nnet.NNet([50], l1_reg=0.0002, dropout=0.4)
net.train(trX, trY, epochs=10)
wrong = 0
total = 0
for X, y in zip(teX, teY):
prediction = net.predict(np.array([X]))
actual = np.argmax(y)
if prediction != actual:
wrong += 1
total += 1
print(wrong, total, float(total - wrong) / float(total))
