def test_convnet_mnist_ngraph_cpu_backend():
# This *must* be run inside the test, because env. var. PYTEST_CURRENT_TEST
# only exists when inside the test function.
ngtfDir = VT.findBridgeRepoDirectory()
script = os.path.join(ngtfDir, kConvnetScriptPath)
VT.checkScript(script)
iterations = int(os.environ.get('TEST_CONVNET_MNIST_ITER', kDefaultIter))
dataDir = os.environ.get('TEST_CONVNET_MNIST_DATA_DIR', None)
VT.checkMnistData(dataDir)
compareToFile = os.environ.get('TEST_CONVNET_MNIST_COMPARE_TO', None)
# If we have a compare-file, then read JSON data into referenceResults
if compareToFile:
print
print('Comparing ngraph results with reference results in %s' %
compareToFile)
fIn = open(compareToFile, 'r')
referenceResults = json.load(fIn)
fIn.close()
# if compareToFile
else:
referenceResults = None
# Run with NGraph CPU backend, saving timing and accuracy
VT.checkNGraphEnvironment()
ngraphLog = VT.runConvnetScript(logID=' nGraph',
useNGraph=True,
script=script,
python=kPythonProg,
iterations=iterations,
batchSize=kTrainBatchSize,
dataDirectory=dataDir,
verbose=False) # log-device-placement
ngraphResults = \
VT.collect_convnet_mnist_results(runType='nGraph CPU backend',
log=ngraphLog,
date=str(kRunDateTime),
iterations=iterations,
batchSize=kTrainBatchSize)
if referenceResults:
print
print('Reference results (from JSON file) are:')
print(json.dumps(referenceResults, indent=4))
print
print('Collected results for *nGraph* in this run are:')
print(json.dumps(ngraphResults, indent=4))
lDir = None
if os.environ.has_key('TEST_CONVNET_MNIST_LOG_DIR'):
lDir = os.path.abspath(os.environ['TEST_CONVNET_MNIST_LOG_DIR'])
# Dump logs to files, for inclusion in Jenkins artifacts
VT.writeLogToFile(ngraphLog, os.path.join(lDir, kConvnetCPUNgLog))
VT.writeJsonToFile(json.dumps(ngraphResults, indent=4),
os.path.join(lDir, kConvnetCPUNgJson))
# Write Jenkins description, for quick perusal of results
if referenceResults:
VT.write_jenkins_convnet_mnist_description(
referenceResults, ngraphResults, kAcceptableAccuracyDelta,
iterations, os.path.join(lDir, kConvnetJenkinsSummaryLog))
print
print '----- CONVNET MNIST Testing Summary ----------------------------------------'
summaryLog = None
if lDir != None:
summaryLog = os.path.join(lDir, kConvnetSummaryLog)
logOut = VT.LogAndOutput(logFile=summaryLog)
if referenceResults:
# Sanity checks on ngraph results vs compare-to reference results.
# Error messages are printed here, and assertions are triggered at the
# end of this function.
if referenceResults['iterations'] != ngraphResults['iterations']:
logOut.line(
'ERROR: iterations do not match -- ref: %s iterations, ngraph: %s iterations'
% (str(referenceResults['iterations']),
str(ngraphResults['iterations'])))
if referenceResults['batch-size'] != ngraphResults['batch-size']:
logOut.line(
'ERROR: batch-sizes do not match -- ref: %s, ngraph: %s' %
(str(referenceResults['batch-size']),
str(ngraphResults['batch-size'])))
# End: if referenceResults
# Report commands
logOut.line()
if referenceResults:
logOut.line('Reference run using CPU: %s' %
referenceResults['command'])
logOut.line('Run with NGraph CPU: %s' % ngraphResults['command'])
# Report parameters
logOut.line()
logOut.line('Iterationss: %d' % iterations)
logOut.line('Batch size: %d' % kTrainBatchSize)
logOut.line('Epoch size: %d (fixed in MNIST)' % kTrainEpochSize)
logOut.line('nGraph back-end used: %s' % 'CPU')
logOut.line('Data directory: %s' % dataDir)
if referenceResults:
refAccuracy = float(referenceResults['accuracy'])
ngAccuracy = float(ngraphResults['accuracy'])
# Report accuracy
if referenceResults:
acceptableDelta = refAccuracy * kAcceptableAccuracyDelta
deltaAccuracy = abs(refAccuracy - ngAccuracy)
logOut.line()
if referenceResults:
logOut.line('Accuracy, in reference results: %7.6f' % refAccuracy)
logOut.line('Accuracy, in run with NGraph CPU: %7.6f' % ngAccuracy)
if referenceResults:
logOut.line(
'Acceptable accuracy range (from reference) is: %4.2f%% of %7.6f' %
(kAcceptableAccuracyDelta * 100, refAccuracy))
logOut.line('Acceptable accuracy delta is <= %7.6f' %
float(acceptableDelta))
logOut.line('Actual accuracy delta is %7.6f' % deltaAccuracy)
# Report on times
logOut.line()
if referenceResults:
logOut.line('Reference run took: %f seconds' %
referenceResults['wallclock'])
logOut.line('Run with NGraph CPU took: %f seconds' %
ngraphResults['wallclock'])
if referenceResults:
logOut.line(
'NGraph was %f times faster than reference (wall-clock measurement)'
% (referenceResults['wallclock'] / ngraphResults['wallclock']))
# Make sure all output has been flushed before running assertions
logOut.flush()
# All assertions are now done at the very end of the run, after all of
# the summary output has been written.
if referenceResults:
assert referenceResults['iterations'] == ngraphResults['iterations']
assert referenceResults['batch-size'] == ngraphResults['batch-size']
assert deltaAccuracy <= acceptableDelta # Assert for out-of-bounds accuracy
def test_resnet20_cifar10_ngraph_cpu_backend():
# This *must* be run inside the test, because env. var. PYTEST_CURRENT_TEST
# only exists when inside the test function.
ngtfDir = VT.findBridgeRepoDirectory()
script = os.path.join(ngtfDir, kResnet20ScriptPath)
VT.checkScript(script)
dataDir = os.environ.get('TEST_RESNET20_CIFAR10_DATA_DIR', None)
if not os.path.isdir(dataDir):
raise Exception('Directory %s does not exist' % str(dataDir))
epochs = int(os.environ.get('TEST_RESNET20_CIFAR10_EPOCHS',
kDefaultEpochs))
# Run with CPU backend, saving timing and accuracy
refLog = VT.runResnetScript(logID=' Reference',
useNGraph=False,
script=script,
python=kPythonProg,
epochs=epochs,
batchSize=kTrainBatchSize,
dataDirectory=dataDir,
verbose=False) # log-device-placement
refResults = \
VT.collect_resnet20_cifar10_results(runType='Reference CPU backend',
log=refLog,
date=str(kRunDateTime),
epochs=epochs,
batchSize=kTrainBatchSize)
print
print('Collected results for *reference* run are:')
print(json.dumps(refResults, indent=4))
lDir = None
if os.environ.has_key('TEST_RESNET20_CIFAR10_LOG_DIR'):
lDir = os.path.abspath(os.environ['TEST_RESNET20_CIFAR10_LOG_DIR'])
# Dump logs to files, for inclusion in Jenkins artifacts
VT.writeLogToFile(refLog, os.path.join(lDir, kResnet20CPURefLog))
VT.writeJsonToFile(json.dumps(refResults, indent=4),
os.path.join(lDir, kResnet20CPURefJson))
print
print '----- RESNET20 CIFAR10 Reference Testing Summary -------------------------------'
summaryLog = None
if lDir != None:
summaryLog = os.path.join(lDir, kResnet20RefSummaryLog)
logOut = VT.LogAndOutput(logFile=summaryLog)
# Report commands
logOut.line()
logOut.line('Reference run with CPU: %s' % refResults['command'])
# Report parameters
logOut.line()
logOut.line('Epochs: %d' % epochs)
logOut.line('Batch size: %d' % kTrainBatchSize)
logOut.line('Epoch size: %d (fixed in CIFAR10)' %
kTrainEpochSize)
logOut.line('Reference back-end used: %s' % 'CPU')
logOut.line('Data directory: %s' % dataDir)
refAccuracy = float(refResults['accuracy'])
# Report accuracy
logOut.line()
logOut.line('Accuracy, in reference run using CPU: %7.6f' % refAccuracy)
# Report on times
logOut.line()
logOut.line('Reference run using CPU took: %f seconds' %
refResults['wallclock'])
# Make sure all output has been flushed before running assertions
logOut.flush()
def test_resnet20_cifar10_cpu_backend():
# This *must* be run inside the test, because env. var. PYTEST_CURRENT_TEST
# only exists when inside the test function.
ngtfDir = VT.findBridgeRepoDirectory()
script = os.path.join(ngtfDir, kResnet20ScriptPath)
VT.checkScript(script)
dataDir = os.environ.get('TEST_RESNET20_CIFAR10_DATA_DIR', None)
if not os.path.isdir(dataDir):
raise Exception('Directory %s does not exist' % str(dataDir))
epochs = int(os.environ.get('TEST_RESNET20_CIFAR10_EPOCHS',
kDefaultEpochs))
# Run with Google CPU defaults, saving timing and accuracy
referenceLog = VT.runResnetScript(logID=' Reference',
useNGraph=False,
script=script,
python=kPythonProg,
epochs=epochs,
batchSize=kTrainBatchSize,
dataDirectory=dataDir,
verbose=False) # log-device-placement
referenceResults = \
VT.collect_resnet20_cifar10_results(runType='Reference CPU backend',
log=referenceLog,
date=str(kRunDateTime),
epochs=epochs,
batchSize=kTrainBatchSize)
print
print('Collected results for *reference*:')
print(json.dumps(referenceResults, indent=4))
# Run with NGraph CPU backend, saving timing and accuracy
VT.checkNGraphEnvironment()
ngraphLog = VT.runResnetScript(logID=' nGraph',
useNGraph=True,
script=script,
python=kPythonProg,
epochs=epochs,
batchSize=kTrainBatchSize,
dataDirectory=dataDir,
verbose=False) # log-device-placement
ngraphResults = \
VT.collect_resnet20_cifar10_results(runType='nGraph CPU backend',
log=ngraphLog,
date=str(kRunDateTime),
epochs=epochs,
batchSize=kTrainBatchSize)
print
print('Collected results for *nGraph*:')
print(json.dumps(ngraphResults, indent=4))
lDir = None
if os.environ.has_key('TEST_RESNET20_CIFAR10_LOG_DIR'):
lDir = os.path.abspath(os.environ['TEST_RESNET20_CIFAR10_LOG_DIR'])
# Dump logs to files, for inclusion in Jenkins artifacts
VT.writeLogToFile(referenceLog, os.path.join(lDir, kResnet20CPURefLog))
VT.writeLogToFile(ngraphLog, os.path.join(lDir, kResnet20CPUNgLog))
VT.writeJsonToFile(json.dumps(referenceResults, indent=4),
os.path.join(lDir, kResnet20CPURefJson))
VT.writeJsonToFile(json.dumps(ngraphResults, indent=4),
os.path.join(lDir, kResnet20CPUNgJson))
# Write Jenkins description, for quick perusal of results
VT.write_jenkins_resnet20_cifar10_description(
referenceResults, ngraphResults, kAcceptableAccuracyDelta, epochs,
os.path.join(lDir, kResnet20JenkinsSummaryLog))
print
print '----- RESNET20 CIFAR10 Testing Summary ----------------------------------------'
summaryLog = None
if lDir != None:
summaryLog = os.path.join(lDir, kResnet20SummaryLog)
logOut = VT.LogAndOutput(logFile=summaryLog)
# Report commands
logOut.line()
logOut.line('Reference run using CPU: %s' % referenceResults['command'])
logOut.line('Run with NGraph CPU: %s' % ngraphResults['command'])
# Report parameters
logOut.line()
logOut.line('Epochs: %d' % epochs)
logOut.line('Batch size: %d' % kTrainBatchSize)
logOut.line('Epoch size: %d (fixed in CIFAR10)' %
kTrainEpochSize)
logOut.line('nGraph back-end used: %s' % 'CPU')
logOut.line('Data directory: %s' % dataDir)
refAccuracy = float(referenceResults['accuracy'])
ngAccuracy = float(ngraphResults['accuracy'])
# Report accuracy
acceptableDelta = refAccuracy * kAcceptableAccuracyDelta
deltaAccuracy = abs(refAccuracy - ngAccuracy)
logOut.line()
logOut.line('Accuracy, in reference run using CPU: %7.6f' % refAccuracy)
logOut.line('Accuracy, in run with NGraph CPU: %7.6f' % ngAccuracy)
logOut.line(
'Acceptable accuracy range (from reference) is: %4.2f%% of %7.6f' %
(kAcceptableAccuracyDelta * 100, refAccuracy))
logOut.line('Acceptable accuracy delta is <= %7.6f' %
float(acceptableDelta))
logOut.line('Actual accuracy delta is %7.6f' % deltaAccuracy)
# Report on times
logOut.line()
logOut.line('Reference run using CPU took: %f seconds' %
referenceResults['wallclock'])
logOut.line('Run with NGraph CPU took: %f seconds' %
ngraphResults['wallclock'])
logOut.line(
'NGraph was %f times faster than reference (wall-clock measurement)' %
(referenceResults['wallclock'] / ngraphResults['wallclock']))
# Make sure all output has been flushed before running assertions
logOut.flush()
# All assertions are now done at the very end of the run, after all of
# the summary output has been written.
assert deltaAccuracy <= acceptableDelta # Assert for out-of-bounds accuracy
def test_convnet_mnist_ngraph_cpu_backend():
# This *must* be run inside the test, because env. var. PYTEST_CURRENT_TEST
# only exists when inside the test function.
ngtfDir = VT.findBridgeRepoDirectory()
script = os.path.join(ngtfDir, kConvnetScriptPath)
VT.checkScript(script)
iterations = int(os.environ.get('TEST_CONVNET_MNIST_ITER', kDefaultIter))
dataDir = os.environ.get('TEST_CONVNET_MNIST_DATA_DIR', None)
VT.checkMnistData(dataDir)
# Run with NGraph CPU backend, saving timing and accuracy
refLog = VT.runConvnetScript(logID=' Reference',
useNGraph=False,
script=script,
python=kPythonProg,
iterations=iterations,
batchSize=kTrainBatchSize,
dataDirectory=dataDir,
verbose=False) # log-device-placement
refResults = \
VT.collect_convnet_mnist_results(runType='Reference CPU backend',
log=refLog,
date=str(kRunDateTime),
iterations=iterations,
batchSize=kTrainBatchSize)
print
print('Collected results for *reference* in this run are:')
print(json.dumps(refResults, indent=4))
lDir = None
if os.environ.has_key('TEST_CONVNET_MNIST_LOG_DIR'):
lDir = os.path.abspath(os.environ['TEST_CONVNET_MNIST_LOG_DIR'])
# Dump logs to files, for inclusion in Jenkins artifacts
VT.writeLogToFile(refLog, os.path.join(lDir, kConvnetCPUNgLog))
VT.writeJsonToFile(json.dumps(refResults, indent=4),
os.path.join(lDir, kConvnetCPUNgJson))
print
print '----- CONVNET MNIST Reference Testing Summary ----------------------------------------'
summaryLog = None
if lDir != None:
summaryLog = os.path.join(lDir, kConvnetRefSummaryLog)
logOut = VT.LogAndOutput(logFile=summaryLog)
# Report commands
logOut.line()
logOut.line('Reference run with CPU: %s' % refResults['command'])
# Report parameters
logOut.line()
logOut.line('Iterationss: %d' % iterations)
logOut.line('Batch size: %d' % kTrainBatchSize)
logOut.line('Epoch size: %d (fixed in MNIST)' % kTrainEpochSize)
logOut.line('nGraph back-end used: %s' % 'CPU')
logOut.line('Data directory: %s' % dataDir)
refAccuracy = float(refResults['accuracy'])
# Report accuracy
logOut.line()
logOut.line('Accuracy, in reference run using CPU: %7.6f' % refAccuracy)
# Report on times
logOut.line()
logOut.line('Reference run using CPU took: %f seconds' %
refResults['wallclock'])
# Make sure all output has been flushed before running assertions
logOut.flush()