def run(path, option):
if option == "dpc":
app_folder = utils.get_sample_build_path("dpc_gemm")
app_file = os.path.join(app_folder, "dpc_gemm")
option = "cpu"
p = subprocess.Popen(["./cl_hot_kernels", app_file, option, "1024", "1"],\
cwd = path, stdout = subprocess.PIPE, stderr = subprocess.PIPE)
elif option == "omp":
app_folder = utils.get_sample_build_path("omp_gemm")
app_file = os.path.join(app_folder, "omp_gemm")
option = "gpu"
e = utils.add_env(None, "LIBOMPTARGET_PLUGIN", "OPENCL")
p = subprocess.Popen(["./cl_hot_kernels", app_file, option, "1024", "1"],\
env = e, cwd = path, stdout = subprocess.PIPE, stderr = subprocess.PIPE)
else:
app_folder = utils.get_sample_build_path("cl_gemm")
app_file = os.path.join(app_folder, "cl_gemm")
p = subprocess.Popen(["./cl_hot_kernels", app_file, option, "1024", "1"],\
cwd = path, stdout = subprocess.PIPE, stderr = subprocess.PIPE)
stdout, stderr = utils.run_process(p)
if not stdout:
return stderr
if stdout.find(" CORRECT") == -1:
return stdout
if not parse(stderr):
return stderr
return None
def run(path, option):
if option == "dpc":
app_folder = utils.get_sample_build_path("dpc_gemm")
app_file = os.path.join(app_folder, "dpc_gemm")
p = subprocess.Popen(["./cl_tracer", "-h", "-d", "-t", app_file, "cpu", "1024", "1"],\
cwd = path, stdout = subprocess.PIPE, stderr = subprocess.PIPE)
elif option == "omp":
app_folder = utils.get_sample_build_path("omp_gemm")
app_file = os.path.join(app_folder, "omp_gemm")
e = utils.add_env(None, "LIBOMPTARGET_PLUGIN", "OPENCL")
p = subprocess.Popen(["./cl_tracer", "-h", "-d", "-t", app_file, "gpu", "1024", "1"],\
env = e, cwd = path, stdout = subprocess.PIPE, stderr = subprocess.PIPE)
else:
app_folder = utils.get_sample_build_path("cl_gemm")
app_file = os.path.join(app_folder, "cl_gemm")
if option == "gpu":
p = subprocess.Popen(["./cl_tracer", "-h", "-d", "-t", app_file, "gpu", "1024", "1"],\
cwd = path, stdout = subprocess.PIPE, stderr = subprocess.PIPE)
else:
p = subprocess.Popen(["./cl_tracer", option, app_file, "cpu", "1024", "1"],\
cwd = path, stdout = subprocess.PIPE, stderr = subprocess.PIPE)
stdout, stderr = utils.run_process(p)
if not stdout:
return "stdout is empty"
if not stderr:
return "stderr is empty"
if stdout.find(" CORRECT") == -1:
return stdout
if stderr.find("WARNING") != -1:
return stderr
return None
def get_container(train_op,
train_env,
train_num_gpus,
drive='coco-headset-vol-1'):
(train_op.container.set_memory_request('56Gi').set_memory_limit(
'56Gi').set_cpu_request('7.5').set_cpu_limit('7.5').set_gpu_limit(
str(train_num_gpus)).add_volume_mount(
V1VolumeMount(
name='tensorboard',
mount_path='/shared/tensorboard')).add_volume_mount(
V1VolumeMount(name='data',
mount_path='/data/')).add_volume_mount(
V1VolumeMount(
name='shm',
mount_path='/dev/shm')))
(add_env(add_ssh_volume(train_op), train_env).add_toleration(
V1Toleration(key='nvidia.com/gpu',
operator='Exists',
effect='NoSchedule')).add_node_selector_constraint(
'beta.kubernetes.io/instance-type',
f'p3.{2 * train_num_gpus}xlarge').
add_volume(
V1Volume(name='tensorboard',
persistent_volume_claim=V1PersistentVolumeClaimVolumeSource(
'tensorboard-research-kf'))
).add_volume(
V1Volume(name='data',
persistent_volume_claim=V1PersistentVolumeClaimVolumeSource(
drive)))
# .add_volume(V1Volume(name='shm', host_path=V1HostPathVolumeSource(path='/dev/shm')))
.add_volume(
V1Volume(name='shm',
empty_dir=V1EmptyDirVolumeSource(medium='Memory'))))
def config(path):
e = utils.add_env(None, "CXX", "icpc")
p = subprocess.Popen(["cmake",\
"-DCMAKE_BUILD_TYPE=" + utils.get_build_flag(), ".."], env = e,\
cwd = path, stdout = subprocess.PIPE, stderr = subprocess.PIPE)
p.wait()
stdout, stderr = utils.run_process(p)
if stderr and stderr.find("CMake Error") != -1:
return stderr
return None
def train_eval_epic(owner,
project,
experiment,
model,
git_rev,
pretrained_s3,
mode,
train_additional_args='',
eval_additional_args=''):
train_env = {}
train_num_gpus = 1
train_op = components.load_component_from_file('components/train.yaml')(
owner=owner,
project=project,
experiment=experiment,
model=model,
git_rev=git_rev,
pretrained_s3=pretrained_s3,
mode=mode,
additional_args=train_additional_args)
(train_op.container.set_memory_request('56Gi').set_memory_limit(
'56Gi').set_cpu_request('7.5').set_cpu_limit('7.5').set_gpu_limit(
str(train_num_gpus)).add_volume_mount(
V1VolumeMount(
name='tensorboard',
mount_path='/shared/tensorboard')).add_volume_mount(
V1VolumeMount(name='data',
mount_path='/data/')).add_volume_mount(
V1VolumeMount(
name='shm',
mount_path='/dev/shm')))
(add_env(add_ssh_volume(train_op), train_env).add_toleration(
V1Toleration(key='nvidia.com/gpu',
operator='Exists',
effect='NoSchedule')).add_node_selector_constraint(
'beta.kubernetes.io/instance-type',
f'p3.{2*train_num_gpus}xlarge').
add_volume(
V1Volume(name='tensorboard',
persistent_volume_claim=V1PersistentVolumeClaimVolumeSource(
'tensorboard-research-kf'))
).add_volume(
V1Volume(name='data',
persistent_volume_claim=V1PersistentVolumeClaimVolumeSource(
'dataset-epic-kitchen')))
# .add_volume(V1Volume(name='shm', host_path=V1HostPathVolumeSource(path='/dev/shm')))
.add_volume(
V1Volume(name='shm',
empty_dir=V1EmptyDirVolumeSource(medium='Memory'))))
def run(path, option):
app_folder = utils.get_sample_build_path("omp_gemm")
app_file = os.path.join(app_folder, "omp_gemm")
e = utils.add_env(None, "OMP_TOOL_LIBRARIES", "./libomp_hot_regions.so")
p = subprocess.Popen([app_file, option, "1024", "1"], env = e,\
cwd = path, stdout = subprocess.PIPE, stderr = subprocess.PIPE)
stdout, stderr = utils.run_process(p)
if not stderr:
return stdout
if stdout.find(" CORRECT") == -1:
return stdout
if stderr == "gpu" and stderr.find("Target") == -1:
return stderr
if stderr == "cpu" and stderr.find("Parallel") == -1:
return stderr
if not parse(stderr):
return stderr
return None