def run_tensorflow_pytests_from_artifacts(backend, ngraph_tf_src_dir,
tf_src_dir, xml_output):
root_pwd = os.getcwd()
ngraph_tf_src_dir = os.path.abspath(ngraph_tf_src_dir)
# Check to see if we need to apply the patch for Grappler
import ngraph_bridge
patch_file_name = "test/python/tensorflow/tf_unittest_ngraph" + (
"_with_grappler"
if ngraph_bridge.is_grappler_enabled() else "") + ".patch"
patch_file = os.path.abspath(
os.path.join(ngraph_tf_src_dir, patch_file_name))
# Next patch the TensorFlow so that the tests run using ngraph_bridge
pwd = os.getcwd()
# Go to the location of TesorFlow install directory
import tensorflow as tf
tf_dir = tf.sysconfig.get_lib()
os.chdir(os.path.join(tf_dir, '../'))
print("CURRENT DIR: " + os.getcwd())
print("Patching TensorFlow using: %s" % patch_file)
apply_patch(patch_file)
os.chdir(pwd)
# Now run the TensorFlow python tests
test_src_dir = os.path.join(ngraph_tf_src_dir, "test/python/tensorflow")
test_script = os.path.join(test_src_dir, "tf_unittest_runner.py")
if backend is not None and 'GPU' in backend:
test_manifest_file = os.path.join(test_src_dir,
"python_tests_list_gpu.txt")
else:
test_manifest_file = os.path.join(test_src_dir,
"python_tests_list.txt")
test_xml_report = './junit_tensorflow_tests.xml'
import psutil
num_cores = int(psutil.cpu_count(logical=False))
print("OMP_NUM_THREADS: %s " % str(num_cores))
os.environ['OMP_NUM_THREADS'] = str(num_cores)
os.environ['NGRAPH_TF_DISABLE_DEASSIGN_CLUSTERS'] = '1'
# should this python be sys.executable?
cmd = [
"python",
test_script,
"--tensorflow_path",
tf_src_dir,
"--run_tests_from_file",
test_manifest_file,
]
if xml_output:
cmd.extend(["--xml_report", test_xml_report])
command_executor(cmd)
os.chdir(root_pwd)
def run_tensorflow_pytests(venv_dir, build_dir, ngraph_tf_src_dir, tf_src_dir):
root_pwd = os.getcwd()
build_dir = os.path.abspath(build_dir)
venv_dir = os.path.abspath(venv_dir)
ngraph_tf_src_dir = os.path.abspath(ngraph_tf_src_dir)
patch_file = os.path.abspath(
os.path.join(ngraph_tf_src_dir,
"test/python/tensorflow/tf_unittest_ngraph.patch"))
# Load the virtual env
venv_dir_absolute = load_venv(venv_dir)
# Next patch the TensorFlow so that the tests run using ngraph_bridge
pwd = os.getcwd()
# Go to the site-packages/tensorflow_core_python/framework
os.chdir(
glob.glob(venv_dir_absolute +
"/lib/py*/site-packages/tensorflow_core/python/framework")[0])
print("CURRENT DIR: " + os.getcwd())
print("Patching TensorFlow using: %s" % patch_file)
apply_patch(patch_file)
os.chdir(pwd)
# Now run the TensorFlow python tests
test_src_dir = os.path.join(ngraph_tf_src_dir, "test/python/tensorflow")
test_script = os.path.join(test_src_dir, "tf_unittest_runner.py")
if get_os_type() == 'Darwin':
test_manifest_file = os.path.join(test_src_dir,
"python_tests_list_mac.txt")
else:
test_manifest_file = os.path.join(test_src_dir, "python_tests_list.txt")
test_xml_report = '%s/junit_tensorflow_tests.xml' % build_dir
import psutil
num_cores = int(psutil.cpu_count(logical=False))
print("OMP_NUM_THREADS: %s " % str(num_cores))
os.environ['OMP_NUM_THREADS'] = str(num_cores)
os.environ['NGRAPH_TF_DISABLE_DEASSIGN_CLUSTERS'] = '1'
# command_executor([
# "python", test_script, "--tensorflow_path", tf_src_dir,
# "--run_tests_from_file", test_manifest_file, "--xml_report",
# test_xml_report
# ], verbose=True)
command_executor([
"python", test_script, "--tensorflow_path", tf_src_dir,
"--run_tests_from_file", test_manifest_file
],
verbose=True)
os.chdir(root_pwd)
def run_resnet50_forward_pass_from_artifacts(ngraph_tf_src_dir, artifact_dir,
batch_size, iterations):
root_pwd = os.getcwd()
artifact_dir = os.path.abspath(artifact_dir)
ngraph_tf_src_dir = os.path.abspath(ngraph_tf_src_dir)
install_ngraph_bridge(artifact_dir)
# Now clone the repo and proceed
call(['git', 'clone', 'https://github.com/tensorflow/benchmarks.git'])
os.chdir('benchmarks')
call(['git', 'checkout', '4c7b09ad87bbfc4b1f89650bcee40b3fc5e7dfed'])
# Check to see if we need to patch the repo for Grappler
# benchmark_cnn.patch will only work for the CPU backend
patch_file = os.path.abspath(
os.path.join(ngraph_tf_src_dir, "test/grappler/benchmark_cnn.patch"))
import ngraph_bridge
if ngraph_bridge.is_grappler_enabled():
print("Patching repo using: %s" % patch_file)
apply_patch(patch_file)
os.chdir('scripts/tf_cnn_benchmarks/')
# junit_script = os.path.abspath('%s/test/ci/junit-wrap.sh' % root_pwd)
# Update the script by adding `import ngraph_bridge`
with open('convnet_builder.py', 'a') as outfile:
call(['echo', 'import ngraph_bridge'], stdout=outfile)
# Setup the env flags
import psutil
num_cores = int(psutil.cpu_count(logical=False))
print("OMP_NUM_THREADS: %s " % str(num_cores))
os.environ['OMP_NUM_THREADS'] = str(num_cores)
os.environ["KMP_AFFINITY"] = 'granularity=fine,compact,1,0'
cmd = [
'python',
'tf_cnn_benchmarks.py',
'--data_format',
'NCHW',
'--num_inter_threads',
'2',
'--freeze_when_forward_only=True',
'--model=resnet50',
'--batch_size=' + str(batch_size),
'--num_batches',
str(iterations),
]
command_executor(cmd, verbose=True)
os.chdir(root_pwd)
def run_resnet50_forward_pass(build_dir):
root_pwd = os.getcwd()
build_dir = os.path.abspath(build_dir)
ngraph_tf_src_dir = os.path.abspath(build_dir + '/../')
os.chdir(build_dir)
call(['git', 'clone', 'https://github.com/tensorflow/benchmarks.git'])
os.chdir('benchmarks')
call(['git', 'checkout', '4c7b09ad87bbfc4b1f89650bcee40b3fc5e7dfed'])
junit_script = os.path.abspath('%s/test/ci/junit-wrap.sh' % root_pwd)
# Check to see if we need to patch the repo for Grappler
# benchmark_cnn.patch will only work for the CPU backend
patch_file = os.path.abspath(
os.path.join(ngraph_tf_src_dir, "test/grappler/benchmark_cnn.patch"))
import ngraph_bridge
if ngraph_bridge.is_grappler_enabled():
print("Patching repo using: %s" % patch_file)
apply_patch(patch_file)
os.chdir('scripts/tf_cnn_benchmarks/')
# Update the script by adding `import ngraph_bridge`
with open('convnet_builder.py', 'a') as outfile:
call(['echo', 'import ngraph_bridge'], stdout=outfile)
# Setup the env flags
import psutil
num_cores = int(psutil.cpu_count(logical=False))
print("OMP_NUM_THREADS: %s " % str(num_cores))
os.environ['OMP_NUM_THREADS'] = str(num_cores)
os.environ["KMP_AFFINITY"] = 'granularity=fine,compact,1,0'
os.environ['JUNIT_WRAP_FILE'] = "%s/junit_inference_test.xml" % build_dir
os.environ['JUNIT_WRAP_SUITE'] = 'models'
os.environ['JUNIT_WRAP_TEST'] = 'resnet50-inference'
# Run inference job
cmd = [
junit_script, 'python', 'tf_cnn_benchmarks.py', '--data_format', 'NHWC',
'--num_inter_threads', '2', '--freeze_when_forward_only=True',
'--model=resnet50', '--batch_size=1', '--num_batches', '32'
]
command_executor(cmd, verbose=True)
os.chdir(root_pwd)
def run_resnet50_from_artifacts(ngraph_tf_src_dir, artifact_dir, batch_size,
iterations):
root_pwd = os.getcwd()
artifact_dir = os.path.abspath(artifact_dir)
ngraph_tf_src_dir = os.path.abspath(ngraph_tf_src_dir)
install_ngraph_bridge(artifact_dir)
# Now clone the repo and proceed
call(['git', 'clone', 'https://github.com/tensorflow/benchmarks.git'])
os.chdir('benchmarks')
call(['git', 'checkout', '4c7b09ad87bbfc4b1f89650bcee40b3fc5e7dfed'])
# Check to see if we need to patch the repo for Grappler
patch_file = os.path.abspath(
os.path.join(ngraph_tf_src_dir, "test/grappler/benchmark_cnn.patch"))
import ngraph_bridge
if ngraph_bridge.is_grappler_enabled():
print("Patching repo using: %s" % patch_file)
apply_patch(patch_file)
os.chdir('scripts/tf_cnn_benchmarks/')
# junit_script = os.path.abspath('%s/test/ci/junit-wrap.sh' % root_pwd)
# Update the script by adding `import ngraph_bridge`
with open('convnet_builder.py', 'a') as outfile:
call(['echo', 'import ngraph_bridge'], stdout=outfile)
# Setup the env flags
import psutil
num_cores = int(psutil.cpu_count(logical=False))
print("OMP_NUM_THREADS: %s " % str(num_cores))
os.environ['OMP_NUM_THREADS'] = str(num_cores)
os.environ["KMP_AFFINITY"] = 'granularity=fine,compact,1,0'
# Delete the temporary model save directory
model_save_dir = os.getcwd() + '/modelsavepath'
if os.path.exists(model_save_dir) and os.path.isdir(model_save_dir):
shutil.rmtree(model_save_dir)
eval_eventlog_dir = os.getcwd() + '/eval_eventlog_dir'
if os.path.exists(eval_eventlog_dir) and os.path.isdir(eval_eventlog_dir):
shutil.rmtree(eval_eventlog_dir)
# os.environ['JUNIT_WRAP_FILE'] = "%s/junit_training_test.xml" % build_dir
# os.environ['JUNIT_WRAP_SUITE'] = 'models'
# os.environ['JUNIT_WRAP_TEST'] = 'resnet50-training'
# Run training job
# cmd = [
# junit_script, 'python', 'tf_cnn_benchmarks.py', '--data_format',
# 'NCHW', '--num_inter_threads', '1', '--train_dir=' + model_save_dir,
# '--num_batches', '10', '--model=resnet50', '--batch_size=128'
# ]
cmd = [
'python', 'tf_cnn_benchmarks.py', '--data_format', 'NCHW',
'--num_inter_threads', '1', '--train_dir=' + model_save_dir,
'--num_batches',
str(iterations), '--model=resnet50', '--batch_size=' + str(batch_size),
'--eval_dir=' + eval_eventlog_dir
]
command_executor(cmd, verbose=True)
# os.environ['JUNIT_WRAP_FILE'] = "%s/junit_inference_test.xml" % build_dir
# os.environ['JUNIT_WRAP_SUITE'] = 'models'
# os.environ['JUNIT_WRAP_TEST'] = 'resnet50-inference'
# Run inference job
# cmd = [
# junit_script, 'python', 'tf_cnn_benchmarks.py', '--data_format',
# 'NCHW', '--num_inter_threads', '1', '--train_dir=' + model_save_dir,
# '--model=resnet50', '--batch_size=128', '--num_batches', '10', '--eval'
# ]
cmd = [
'python', 'tf_cnn_benchmarks.py', '--data_format', 'NCHW',
'--num_inter_threads', '1', '--train_dir=' + model_save_dir,
'--model=resnet50', '--batch_size=' + str(batch_size), '--num_batches',
str(iterations), '--eval', '--eval_dir=' + eval_eventlog_dir
]
command_executor(cmd, verbose=True)
os.chdir(root_pwd)
def run_resnet50_from_artifacts(openvino_tf_src_dir, artifact_dir, batch_size,
iterations):
root_pwd = os.getcwd()
artifact_dir = os.path.abspath(artifact_dir)
openvino_tf_src_dir = os.path.abspath(openvino_tf_src_dir)
install_openvino_tensorflow(artifact_dir)
# Now clone the repo and proceed
subprocess.Popen(
shlex.split('git clone https://github.com/tensorflow/benchmarks.git'))
if not os.path.exists('benchmarks'):
raise AssertionError(
"Could not find directory: {}".format('benchmarks'))
os.chdir('benchmarks')
subprocess.Popen(
shlex.split('git checkout aef6daa90a467a1fc7ce8395cd0067e5fda1ecff'))
# Check to see if we need to patch the repo for Grappler
# benchmark_cnn.patch will only work for the CPU backend
patch_file = os.path.abspath(
os.path.join(openvino_tf_src_dir, "test/grappler/benchmark_cnn.patch"))
import openvino_tensorflow
if openvino_tensorflow.is_grappler_enabled():
print("Patching repo using: %s" % patch_file)
apply_patch(patch_file)
if not os.path.exists('scripts/tf_cnn_benchmarks/'):
raise AssertionError("Could not find directory: {}".format(
'scripts/tf_cnn_benchmarks/'))
os.chdir('scripts/tf_cnn_benchmarks/')
# junit_script = os.path.abspath('%s/test/ci/junit-wrap.sh' % root_pwd)
# Update the script by adding `import openvino_tensorflow`
with open('convnet_builder.py', 'a') as outfile:
subprocess.Popen(shlex.split('echo import openvino_tensorflow'),
stdout=outfile)
# Setup the env flags
import psutil
num_cores = int(psutil.cpu_count(logical=False))
print("OMP_NUM_THREADS: %s " % str(num_cores))
os.environ['OMP_NUM_THREADS'] = str(num_cores)
os.environ["KMP_AFFINITY"] = 'granularity=fine,compact,1,0'
# Delete the temporary model save directory
model_save_dir = os.getcwd() + '/modelsavepath'
if os.path.exists(model_save_dir) and os.path.isdir(model_save_dir):
shutil.rmtree(model_save_dir)
eval_eventlog_dir = os.getcwd() + '/eval_eventlog_dir'
if os.path.exists(eval_eventlog_dir) and os.path.isdir(eval_eventlog_dir):
shutil.rmtree(eval_eventlog_dir)
cmd = [
'python', 'tf_cnn_benchmarks.py', '--data_format', 'NHWC',
'--num_inter_threads', '1', '--train_dir=' + model_save_dir,
'--num_batches',
str(iterations), '--model=resnet50', '--batch_size=' + str(batch_size),
'--eval_dir=' + eval_eventlog_dir
]
command_executor(cmd, verbose=True)
cmd = [
'python', 'tf_cnn_benchmarks.py', '--data_format', 'NHWC',
'--num_inter_threads', '1', '--train_dir=' + model_save_dir,
'--model=resnet50', '--batch_size=' + str(batch_size), '--num_batches',
str(iterations), '--eval', '--eval_dir=' + eval_eventlog_dir
]
# Commenting the eval since it currently fails with TF2.0
command_executor(cmd, verbose=True)
if not os.path.exists(root_pwd):
raise AssertionError("Could not find the path")
os.chdir(root_pwd)