def test_Not(tmpdir):
data1 = np.asarray([[1, 1, 0, 0], [1, 1, 1, 1]], np.float32)
model = C.element_not(data1)
verify_no_input(model, tmpdir, 'Not_0')
x = C.input_variable(np.shape(data1))
model = C.element_not(x)
verify_one_input(model, data1, tmpdir, 'Not_1')
def test_Not(tmpdir, dtype):
with C.default_options(dtype=dtype):
data1 = np.asarray([[1, 1, 0, 0], [1, 1, 1, 1]]).astype(dtype)
model = C.element_not(data1)
verify_no_input(model, tmpdir, 'Not_0')
x = C.input_variable(np.shape(data1))
model = C.element_not(x)
verify_one_input(model, data1, tmpdir, 'Not_1')
def test_Not(tmpdir, dtype):
with C.default_options(dtype = dtype):
data1 = np.asarray([[1, 1, 0, 0],[1, 1, 1, 1]]).astype(dtype)
model = C.element_not(data1)
verify_no_input(model, tmpdir, 'Not_0')
x = C.input_variable(np.shape(data1))
model = C.element_not(x)
verify_one_input(model, data1, tmpdir, 'Not_1')
def train():
# TODO: Need to add a method that reads exact sample size when
# we're loading data that's already been converted
#convertData(dataPath, 'intel', threshold, timeSteps, timeShift, seqDist)
input = cntk.sequence.input_variable((numFeatures), name='features')
label = cntk.input_variable((numClasses), name='label')
trainReader = createReader('./data/intel_train.ctf', True, numFeatures,
numClasses)
validReader = createReader('./data/intel_valid.ctf', False, numFeatures,
numClasses)
trainInputMap = {
input: trainReader.streams.features,
label: trainReader.streams.labels
}
validInputMap = {
input: validReader.streams.features,
label: validReader.streams.labels
}
model = createModel(input, numClasses, lstmLayers, lstmSize)
z = model(input)
loss = cntk.cross_entropy_with_softmax(z, label)
accy = cntk.element_not(cntk.classification_error(
z, label)) # Print accuracy %, not error!
lr = cntk.learning_parameter_schedule(0.05, batchSize)
learner = cntk.adam(
z.parameters, lr, 0.9
) #, l2_regularization_weight=0.00001, gradient_clipping_threshold_per_sample=5.0
#tbWriter = cntk.logging.TensorBoardProgressWriter(1, './Tensorboard/', model=model)
printer = cntk.logging.ProgressPrinter(100, tag='Training')
trainer = cntk.Trainer(z, (loss, accy), learner, [printer])
# TODO: These should be automatically detected!
samplesPerSeq = timeSteps
sequences = 8709
validSeqs = 968
minibatchSize = batchSize * samplesPerSeq
minibatches = sequences // batchSize
validBatches = validSeqs // batchSize
cntk.logging.log_number_of_parameters(z)
print(
"Input days: {}; Looking for +- {:.1f}% change {} days ahead;".format(
samplesPerSeq, threshold * 100.0, timeShift))
print("Total Sequences: {}; {} epochs; {} minibatches per epoch;".format(
sequences + validSeqs, numEpochs, minibatches + validBatches))
# Testing out custom data reader
reader = DataReader('./data/intel_train.ctf', numFeatures, numClasses,
batchSize, timeSteps, False)
testReader = DataReader('./data/intel_valid.ctf', numFeatures, numClasses,
batchSize, timeSteps, False)
for e in range(numEpochs):
# Train network
for b in range(minibatches):
X, Y = next(reader)
trainer.train_minibatch({z.arguments[0]: X, label: Y})
trainer.summarize_training_progress()
# Look at data we've not trained on (validation)
for b in range(minibatches):
X, Y = next(testReader)
trainer.test_minibatch({z.arguments[0]: X, label: Y})
trainer.summarize_test_progress()
def trainNet(args):
# Crash doesn't seem to occur with this flag,
# unfortunatly, it reduces training speed by about 35%
#os.environ["CUDA_LAUNCH_BLOCKING"] = "1"
# Instantiate generators for both training and
# validation datasets. Grab their generator functions
# TODO: Command line args
# TODO: Better system for using files testing/validation than ranges?
tFileShp = (1, 598)
vFileShp = (0, 1)
gen = Generator(featurePath, labelPath, tFileShp, batchSize, loadSize=3)
valGen = Generator(featurePath, labelPath, vFileShp, batchSize, loadSize=1)
g = gen.generator()
vg = valGen.generator()
inputVar = cntk.ops.input_variable((BoardDepth, BoardLength, BoardLength),
name='features')
policyVar = cntk.ops.input_variable((BoardSize))
valueVar = cntk.ops.input_variable((2))
if args.fp16:
cntk.cast(inputVar, dtype=np.float16)
cntk.cast(policyVar, dtype=np.float16)
cntk.cast(valueVar, dtype=np.float16)
net, epochOffset = loadModel(args, inputVar, netFilters, resBlockCount)
# Show a heatmap of network outputs
# over an input board state
if args.heatMap:
hmap = NetHeatMap(net, g)
hmap.genHeatmap(args.heatMap)
# Loss and accuracy
policyLoss = cntk.cross_entropy_with_softmax(net.outputs[0], policyVar)
valueLoss = cntk.cross_entropy_with_softmax(net.outputs[1], valueVar)
loss = policyLoss + valueLoss
# TODO: Figure out how to display/report both errors
policyError = cntk.element_not(
cntk.classification_error(net.outputs[0], policyVar))
valueError = cntk.element_not(
cntk.classification_error(net.outputs[1], valueVar))
#error = (valueError + policyError) / 2
#error = valueError
error = policyError
if args.fp16:
loss = cntk.cast(loss, dtype=np.float32)
error = cntk.cast(error, dtype=np.float32)
lrc = args.lr
if args.cycleLr[0]:
lrc = learningRateCycles(*args.cycleLr, gen.stepsPerEpoch,
args.cycleMax)
lrc = lrc * maxEpochs
elif args.optLr:
lrc = findOptLr(maxEpochs, *args.optLr, gen.stepsPerEpoch)
lrc = cntk.learners.learning_parameter_schedule(lrc, batchSize, batchSize)
learner = cntk.adam(net.parameters,
lrc,
momentum=0.9,
minibatch_size=batchSize,
l2_regularization_weight=0.0001)
#learner = cntk.adadelta(net.parameters, lrc, l2_regularization_weight=0.0001) # Test adelta out!
# TODO: Figure out how to write multiple 'metrics'
tbWriter = cntk.logging.TensorBoardProgressWriter(freq=1,
log_dir='./TensorBoard/',
model=net)
progressPrinter = cntk.logging.ProgressPrinter(tag='Training',
num_epochs=maxEpochs)
trainer = cntk.Trainer(net, (loss, error), learner,
[progressPrinter, tbWriter])
# TODO: Replace model load with loading/saving checkpoints!
# So we can store learners state et al
#trainer.restore_from_checkpoint(findLatestModel('latest'))
#checkpointFreq = gen.stepsPerEpoch // checkpointFreq
ls = []
losses = []
#valueAccs = []
#policyAccs = []
for epoch in range(maxEpochs):
miniBatches = 0
while miniBatches < gen.stepsPerEpoch:
X, Y, W = next(g)
miniBatches += 1
trainer.train_minibatch({
net.arguments[0]: X,
policyVar: Y,
valueVar: W
})
ls.append(trainer.previous_minibatch_loss_average)
trainer.summarize_training_progress()
policyAcc, valueAcc = printAccuracy(net, 'Validation Acc %', vg,
valGen.stepsPerEpoch)
losses.append([epoch, sum(ls) / gen.stepsPerEpoch])
ls.clear()
#policyAccs.append([epoch, policyAcc])
#valueAccs.append([epoch, valueAcc])
net.save(saveDir + netName + '_{}_{}_{}_{:.3f}.dnn'.format(
epoch + 1 + epochOffset, policyAcc, valueAcc, losses[epoch][1]))