def test_Not(tmpdir): data1 = np.asarray([[1, 1, 0, 0], [1, 1, 1, 1]], np.float32) model = C.element_not(data1) verify_no_input(model, tmpdir, 'Not_0') x = C.input_variable(np.shape(data1)) model = C.element_not(x) verify_one_input(model, data1, tmpdir, 'Not_1')
def test_Not(tmpdir, dtype): with C.default_options(dtype=dtype): data1 = np.asarray([[1, 1, 0, 0], [1, 1, 1, 1]]).astype(dtype) model = C.element_not(data1) verify_no_input(model, tmpdir, 'Not_0') x = C.input_variable(np.shape(data1)) model = C.element_not(x) verify_one_input(model, data1, tmpdir, 'Not_1')
def test_Not(tmpdir, dtype): with C.default_options(dtype = dtype): data1 = np.asarray([[1, 1, 0, 0],[1, 1, 1, 1]]).astype(dtype) model = C.element_not(data1) verify_no_input(model, tmpdir, 'Not_0') x = C.input_variable(np.shape(data1)) model = C.element_not(x) verify_one_input(model, data1, tmpdir, 'Not_1')
def train(): # TODO: Need to add a method that reads exact sample size when # we're loading data that's already been converted #convertData(dataPath, 'intel', threshold, timeSteps, timeShift, seqDist) input = cntk.sequence.input_variable((numFeatures), name='features') label = cntk.input_variable((numClasses), name='label') trainReader = createReader('./data/intel_train.ctf', True, numFeatures, numClasses) validReader = createReader('./data/intel_valid.ctf', False, numFeatures, numClasses) trainInputMap = { input: trainReader.streams.features, label: trainReader.streams.labels } validInputMap = { input: validReader.streams.features, label: validReader.streams.labels } model = createModel(input, numClasses, lstmLayers, lstmSize) z = model(input) loss = cntk.cross_entropy_with_softmax(z, label) accy = cntk.element_not(cntk.classification_error( z, label)) # Print accuracy %, not error! lr = cntk.learning_parameter_schedule(0.05, batchSize) learner = cntk.adam( z.parameters, lr, 0.9 ) #, l2_regularization_weight=0.00001, gradient_clipping_threshold_per_sample=5.0 #tbWriter = cntk.logging.TensorBoardProgressWriter(1, './Tensorboard/', model=model) printer = cntk.logging.ProgressPrinter(100, tag='Training') trainer = cntk.Trainer(z, (loss, accy), learner, [printer]) # TODO: These should be automatically detected! samplesPerSeq = timeSteps sequences = 8709 validSeqs = 968 minibatchSize = batchSize * samplesPerSeq minibatches = sequences // batchSize validBatches = validSeqs // batchSize cntk.logging.log_number_of_parameters(z) print( "Input days: {}; Looking for +- {:.1f}% change {} days ahead;".format( samplesPerSeq, threshold * 100.0, timeShift)) print("Total Sequences: {}; {} epochs; {} minibatches per epoch;".format( sequences + validSeqs, numEpochs, minibatches + validBatches)) # Testing out custom data reader reader = DataReader('./data/intel_train.ctf', numFeatures, numClasses, batchSize, timeSteps, False) testReader = DataReader('./data/intel_valid.ctf', numFeatures, numClasses, batchSize, timeSteps, False) for e in range(numEpochs): # Train network for b in range(minibatches): X, Y = next(reader) trainer.train_minibatch({z.arguments[0]: X, label: Y}) trainer.summarize_training_progress() # Look at data we've not trained on (validation) for b in range(minibatches): X, Y = next(testReader) trainer.test_minibatch({z.arguments[0]: X, label: Y}) trainer.summarize_test_progress()
def trainNet(args): # Crash doesn't seem to occur with this flag, # unfortunatly, it reduces training speed by about 35% #os.environ["CUDA_LAUNCH_BLOCKING"] = "1" # Instantiate generators for both training and # validation datasets. Grab their generator functions # TODO: Command line args # TODO: Better system for using files testing/validation than ranges? tFileShp = (1, 598) vFileShp = (0, 1) gen = Generator(featurePath, labelPath, tFileShp, batchSize, loadSize=3) valGen = Generator(featurePath, labelPath, vFileShp, batchSize, loadSize=1) g = gen.generator() vg = valGen.generator() inputVar = cntk.ops.input_variable((BoardDepth, BoardLength, BoardLength), name='features') policyVar = cntk.ops.input_variable((BoardSize)) valueVar = cntk.ops.input_variable((2)) if args.fp16: cntk.cast(inputVar, dtype=np.float16) cntk.cast(policyVar, dtype=np.float16) cntk.cast(valueVar, dtype=np.float16) net, epochOffset = loadModel(args, inputVar, netFilters, resBlockCount) # Show a heatmap of network outputs # over an input board state if args.heatMap: hmap = NetHeatMap(net, g) hmap.genHeatmap(args.heatMap) # Loss and accuracy policyLoss = cntk.cross_entropy_with_softmax(net.outputs[0], policyVar) valueLoss = cntk.cross_entropy_with_softmax(net.outputs[1], valueVar) loss = policyLoss + valueLoss # TODO: Figure out how to display/report both errors policyError = cntk.element_not( cntk.classification_error(net.outputs[0], policyVar)) valueError = cntk.element_not( cntk.classification_error(net.outputs[1], valueVar)) #error = (valueError + policyError) / 2 #error = valueError error = policyError if args.fp16: loss = cntk.cast(loss, dtype=np.float32) error = cntk.cast(error, dtype=np.float32) lrc = args.lr if args.cycleLr[0]: lrc = learningRateCycles(*args.cycleLr, gen.stepsPerEpoch, args.cycleMax) lrc = lrc * maxEpochs elif args.optLr: lrc = findOptLr(maxEpochs, *args.optLr, gen.stepsPerEpoch) lrc = cntk.learners.learning_parameter_schedule(lrc, batchSize, batchSize) learner = cntk.adam(net.parameters, lrc, momentum=0.9, minibatch_size=batchSize, l2_regularization_weight=0.0001) #learner = cntk.adadelta(net.parameters, lrc, l2_regularization_weight=0.0001) # Test adelta out! # TODO: Figure out how to write multiple 'metrics' tbWriter = cntk.logging.TensorBoardProgressWriter(freq=1, log_dir='./TensorBoard/', model=net) progressPrinter = cntk.logging.ProgressPrinter(tag='Training', num_epochs=maxEpochs) trainer = cntk.Trainer(net, (loss, error), learner, [progressPrinter, tbWriter]) # TODO: Replace model load with loading/saving checkpoints! # So we can store learners state et al #trainer.restore_from_checkpoint(findLatestModel('latest')) #checkpointFreq = gen.stepsPerEpoch // checkpointFreq ls = [] losses = [] #valueAccs = [] #policyAccs = [] for epoch in range(maxEpochs): miniBatches = 0 while miniBatches < gen.stepsPerEpoch: X, Y, W = next(g) miniBatches += 1 trainer.train_minibatch({ net.arguments[0]: X, policyVar: Y, valueVar: W }) ls.append(trainer.previous_minibatch_loss_average) trainer.summarize_training_progress() policyAcc, valueAcc = printAccuracy(net, 'Validation Acc %', vg, valGen.stepsPerEpoch) losses.append([epoch, sum(ls) / gen.stepsPerEpoch]) ls.clear() #policyAccs.append([epoch, policyAcc]) #valueAccs.append([epoch, valueAcc]) net.save(saveDir + netName + '_{}_{}_{}_{:.3f}.dnn'.format( epoch + 1 + epochOffset, policyAcc, valueAcc, losses[epoch][1]))