def do(i, arg):
random_name = arg.random_name
# Detect basic platform info.
ii = {'action': 'detect', 'module_uoa': 'platform', 'out': 'out'}
r = ck.access(ii)
if r['return'] > 0: return r
# Keep to prepare ReQuEST meta.
platform_dict = copy.deepcopy(r)
# Host and target OS params.
hos = r['host_os_uoa']
hosd = r['host_os_dict']
tos = r['os_uoa']
tosd = r['os_dict']
tdid = r['device_id']
# Program and command.
program = 'caffe'
cmd_key = 'time_cpu'
# Load Caffe program meta and desc to check deps.
ii = {'action': 'load', 'module_uoa': 'program', 'data_uoa': program}
rx = ck.access(ii)
if rx['return'] > 0: return rx
mm = rx['dict']
# Get compile-time and run-time deps.
cdeps = mm.get('compile_deps', {})
rdeps = mm.get('run_deps', {})
# Merge rdeps with cdeps for setting up the pipeline (which uses
# common deps), but tag them as "for_run_time".
for k in rdeps:
cdeps[k] = rdeps[k]
cdeps[k]['for_run_time'] = 'yes'
# Caffe libs.
depl = copy.deepcopy(cdeps['lib-caffe'])
if (arg.tos is not None) and (arg.did is not None):
tos = arg.tos
tdid = arg.did
ii = {
'action': 'resolve',
'module_uoa': 'env',
'host_os': hos,
'target_os': tos,
'device_id': tdid,
'out': 'con',
'quiet': 'yes',
'deps': {
'lib-caffe': copy.deepcopy(depl)
}
}
r = ck.access(ii)
if r['return'] > 0: return r
udepl = r['deps']['lib-caffe'].get('choices',
[]) # All UOAs of env for Caffe libs.
if len(udepl) == 0:
return {'return': 1, 'error': 'no installed Caffe libs'}
# Caffe models.
depm = copy.deepcopy(cdeps['caffemodel'])
ii = {
'action': 'resolve',
'module_uoa': 'env',
'host_os': hos,
'target_os': tos,
'device_id': tdid,
'out': 'con',
'quiet': 'yes',
'deps': {
'caffemodel': copy.deepcopy(depm)
}
}
r = ck.access(ii)
if r['return'] > 0: return r
udepm = r['deps']['caffemodel'].get(
'choices', []) # All UOAs of env for Caffe models.
if len(udepm) == 0:
return {'return': 1, 'error': 'no installed Caffe models'}
# ImageNet datasets.
depd = copy.deepcopy(cdeps['dataset-imagenet-lmdb'])
ii = {
'action': 'resolve',
'module_uoa': 'env',
'host_os': hos,
'target_os': tos,
'device_id': tdid,
'out': 'con',
'quiet': 'yes',
'deps': {
'dataset-imagenet-lmdb': copy.deepcopy(depd)
}
}
r = ck.access(ii)
if r['return'] > 0: return r
udepd = r['deps']['dataset-imagenet-lmdb'].get(
'choices', []) # All UOAs of env for ImageNet datasets.
if len(udepd) == 0:
return {'return': 1, 'error': 'no installed ImageNet datasets'}
# Prepare pipeline.
cdeps['lib-caffe']['uoa'] = udepl[0]
cdeps['caffemodel']['uoa'] = udepm[0]
cdeps['dataset-imagenet-lmdb']['uoa'] = udepd[0]
ii = {
'action': 'pipeline',
'prepare': 'yes',
'dependencies': cdeps,
'module_uoa': 'program',
'data_uoa': program,
'cmd_key': cmd_key,
'target_os': tos,
'device_id': tdid,
'no_state_check': 'yes',
'no_compiler_description': 'yes',
'skip_calibration': 'yes',
'cpu_freq': 'max',
'gpu_freq': 'max',
'flags': '-O3',
'speed': 'no',
'energy': 'no',
'skip_print_timers': 'yes',
'out': 'con'
}
r = ck.access(ii)
if r['return'] > 0: return r
fail = r.get('fail', '')
if fail == 'yes':
return {
'return': 10,
'error': 'pipeline failed (' + r.get('fail_reason', '') + ')'
}
ready = r.get('ready', '')
if ready != 'yes':
return {'return': 11, 'error': 'pipeline not ready'}
state = r['state']
tmp_dir = state['tmp_dir']
# Remember resolved deps for this benchmarking session.
xcdeps = r.get('dependencies', {})
# Clean pipeline.
if 'ready' in r: del (r['ready'])
if 'fail' in r: del (r['fail'])
if 'return' in r: del (r['return'])
pipeline = copy.deepcopy(r)
# For each Caffe lib.*******************************************************
for lib_uoa in udepl:
# Load Caffe lib.
ii = {'action': 'load', 'module_uoa': 'env', 'data_uoa': lib_uoa}
r = ck.access(ii)
if r['return'] > 0: return r
real_tags = r['dict']['tags']
# if 'vrequest' in real_tags: continue
# Get the tags from e.g. 'BVLC Caffe framework (intel, request)'
lib_name = r['data_name']
lib_tags = re.match('BVLC Caffe framework \((?P<tags>.*)\)', lib_name)
lib_tags = lib_tags.group('tags').replace(' ', '').replace(',', '-')
# Skip some libs with "in [..]" or "not in [..]".
# Check if Intel artifact and select OpenMP tuning
tuning_dims = {
'choices_order': [['##choices#env#CK_CAFFE_BATCH_SIZE']],
'choices_selection': [{
'type': 'loop',
'default': bs['default'],
'choice': bs['choice']
}],
'features_keys_to_process': ['##choices#env#CK_CAFFE_BATCH_SIZE']
}
intel_artifact = False
if 'intel' in real_tags and 'vrequest' in real_tags:
intel_artifact = True
# Add extra tuning dimension (OpenMP)
tuning_dims['choices_order'].append(
['##choices#env#OMP_NUM_THREADS'])
tuning_dims['choices_selection'].append({
'type': 'loop',
'default': nt['default'],
'choice': nt['choice']
})
tuning_dims['features_keys_to_process'].append(
'##choices#env#OMP_NUM_THREADS')
# ReQuEST CUDA/CUDNN
cuda = False
rtags = r['dict'].get('tags', [])
if 'vcuda' in rtags:
# Detect gpgpu
r = ck.access({
'action': 'detect',
'module_uoa': 'platform.gpgpu',
'cuda': 'yes',
'select': 'yes'
})
if r['return'] > 0: return r
platform_dict['features'].update(r['features'])
cuda = True
# Check proper command line for CPU or GPU
cmd_key = 'time_cpu'
if cuda: cmd_key = 'time_gpu'
# Remark next one if you want to check other libs
# if lib_tags in [ 'intel-request' ]: continue
skip_compile = 'no'
# For each Caffe model.*************************************************
for model_uoa in udepm:
# Load Caffe model.
ii = {'action': 'load', 'module_uoa': 'env', 'data_uoa': model_uoa}
r = ck.access(ii)
if r['return'] > 0: return r
model_real_tags = r['dict']['tags']
if 'vint8' in model_real_tags and not intel_artifact:
continue
# Get the tags from e.g. 'Caffe model (net and weights) (inception-v3, fp32)'
model_name = r['data_name']
model_tags = re.match(
'Caffe model \(net and weights\) \((?P<tags>.*)\)', model_name)
model_tags = model_tags.group('tags').replace(' ', '').replace(
',', '-')
# Skip some models with "in [..]" or "not in [..]".
if model_tags not in [
'resnet50-fp32', 'resnet50-int8', 'inception-v3-fp32',
'inception-v3-int8'
]:
continue
record_repo = 'local'
record_uoa = 'ck-request-asplos18-caffe-intel-performance-' + platform_tags + '.' + lib_tags + '.' + model_tags
# Prepare pipeline.
ck.out(
'---------------------------------------------------------------------------------------'
)
ck.out('%s - %s' % (lib_name, lib_uoa))
ck.out('%s - %s' % (model_name, model_uoa))
ck.out('Experiment - %s:%s' % (record_repo, record_uoa))
# Prepare autotuning input.
cpipeline = copy.deepcopy(pipeline)
# Reset deps and change UOA.
new_deps = {
'lib-caffe': copy.deepcopy(depl),
'caffemodel': copy.deepcopy(depm)
}
new_deps['lib-caffe']['uoa'] = lib_uoa
new_deps['caffemodel']['uoa'] = model_uoa
jj = {
'action': 'resolve',
'module_uoa': 'env',
'host_os': hos,
'target_os': tos,
'device_id': tdid,
'deps': new_deps
}
r = ck.access(jj)
if r['return'] > 0: return r
cpipeline['dependencies'].update(new_deps)
cpipeline['no_clean'] = skip_compile
cpipeline['no_compile'] = skip_compile
cpipeline['cmd_key'] = cmd_key
cpipeline['extra_run_cmd'] = '-phase TEST'
# Prepare common meta for ReQuEST tournament
features = copy.deepcopy(cpipeline['features'])
platform_dict['features'].update(features)
r = ck.access({
'action': 'prepare_common_meta',
'module_uoa': 'request.asplos18',
'platform_dict': platform_dict,
'deps': cpipeline['dependencies'],
'request_dict': request_dict
})
if r['return'] > 0: return r
record_dict = r['record_dict']
meta = r['meta']
if random_name:
rx = ck.gen_uid({})
if rx['return'] > 0: return rx
record_uoa = rx['data_uid']
tags = r['tags']
tags.append('explore-batch-size-openmp-threads')
tags.append(program)
tags.append(model_tags)
tags.append(lib_tags)
tags.append(platform_tags)
ii = {
'action': 'autotune',
'module_uoa': 'pipeline',
'data_uoa': 'program',
'iterations': -1,
'repetitions': num_repetitions,
'record': 'yes',
'record_failed': 'yes',
'record_params': {
'search_point_by_features': 'yes'
},
'tags': tags,
'meta': meta,
'record_dict': record_dict,
'record_repo': record_repo,
'record_uoa': record_uoa,
'pipeline': cpipeline,
'out': 'con'
}
ii.update(copy.deepcopy(tuning_dims))
r = ck.access(ii)
if r['return'] > 0: return r
fail = r.get('fail', '')
if fail == 'yes':
return {
'return': 10,
'error':
'pipeline failed (' + r.get('fail_reason', '') + ')'
}
skip_compile = 'yes'
return {'return': 0}
def do(i, arg):
# Process arguments.
if (arg.accuracy):
experiment_type = 'accuracy'
num_repetitions = 1
else:
experiment_type = 'performance'
num_repetitions = arg.repetitions
random_name = arg.random_name
share_platform = arg.share_platform
# Detect basic platform info.
ii={'action':'detect',
'module_uoa':'platform',
'out':'con'}
if share_platform: ii['exchange']='yes'
r=ck.access(ii)
if r['return']>0: return r
# Keep to prepare ReQuEST meta.
platform_dict=copy.deepcopy(r)
# Host and target OS params.
hos=r['host_os_uoa']
hosd=r['host_os_dict']
tos=r['os_uoa']
tosd=r['os_dict']
tdid=r['device_id']
# program='mobilenets-armcl-opencl'
program='image-classification-tf-py'
ii={'action':'show',
'module_uoa':'env',
'tags':'dataset,imagenet,raw,val'}
rx=ck.access(ii)
if len(rx['lst']) == 0: return rx
# FIXME: It's probably better to use CK_ENV_DATASET_IMAGE_DIR.
img_dir_val = rx['lst'][0]['meta']['env']['CK_CAFFE_IMAGENET_VAL']
if (arg.accuracy):
batch_count = len([f for f in os.listdir(img_dir_val)
if f.endswith('.JPEG') and os.path.isfile(os.path.join(img_dir_val, f))])
else:
batch_count = 1
ii={'action':'show',
'module_uoa':'env',
'tags':'dataset,imagenet,aux'}
rx=ck.access(ii)
if len(rx['lst']) == 0: return rx
img_dir_aux = rx['lst'][0]['meta']['env']['CK_ENV_DATASET_IMAGENET_AUX']
ii={'action':'load',
'module_uoa':'program',
'data_uoa':program}
rx=ck.access(ii)
if rx['return']>0: return rx
mm=rx['dict']
# Get compile-time and run-time deps.
cdeps=mm.get('compile_deps',{})
rdeps=mm.get('run_deps',{})
# Merge rdeps with cdeps for setting up the pipeline (which uses
# common deps), but tag them as "for_run_time".
for k in rdeps:
cdeps[k]=rdeps[k]
cdeps[k]['for_run_time']='yes'
print cdeps
depl=copy.deepcopy(cdeps['lib-tensorflow'])
if (arg.tos is not None) and (arg.did is not None):
tos=arg.tos
tdid=arg.did
ii={'action':'resolve',
'module_uoa':'env',
'host_os':hos,
'target_os':tos,
'device_id':tdid,
'out':'con',
'deps':{'library':copy.deepcopy(depl)},
'quiet':'yes'
}
r=ck.access(ii)
if r['return']>0: return r
udepl=r['deps']['library'].get('choices',[]) # All UOAs of env for TF lib
if len(udepl)==0:
return {'return':1, 'error':'no installed TensorFlow'}
cdeps['lib-tensorflow']['uoa']=udepl[0]
depm=copy.deepcopy(cdeps['model-and-weights'])
ii={'action':'resolve',
'module_uoa':'env',
'host_os':hos,
'target_os':tos,
'device_id':tdid,
'out':'con',
'deps':{'weights':copy.deepcopy(depm)},
'quiet':'yes'
}
r=ck.access(ii)
if r['return']>0: return r
udepm=r['deps']['weights'].get('choices',[])
if len(udepm)==0:
return {'return':1, 'error':'no installed Weights'}
cdeps['lib-tensorflow']['uoa']=udepl[0]
cdeps['model-and-weights']['uoa']=udepm[0]
ii={'action':'pipeline',
'prepare':'yes',
'dependencies':cdeps,
'module_uoa':'program',
'data_uoa':program,
'target_os':tos,
'device_id':tdid,
'no_state_check':'yes',
'no_compiler_description':'yes',
'skip_calibration':'yes',
'env':{
'CK_ENV_DATASET_IMAGENET_VAL':img_dir_val,
'CK_BATCH_COUNT':batch_count,
'CK_BATCHES_DIR':'../batches',
'CK_BATCH_LIST':'../batches',
'CK_IMAGE_LIST':'../images',
'CK_RESULTS_DIR':'predictions',
'CK_SKIP_IMAGES':0
},
'cpu_freq':'max',
'gpu_freq':'max',
'flags':'-O3',
'speed':'no',
'energy':'no',
'skip_print_timers':'yes',
'out':'con'
}
r=ck.access(ii)
if r['return']>0: return r
fail=r.get('fail','')
if fail=='yes':
return {'return':10, 'error':'pipeline failed ('+r.get('fail_reason','')+')'}
ready=r.get('ready','')
if ready!='yes':
return {'return':11, 'error':'pipeline not ready'}
state=r['state']
tmp_dir=state['tmp_dir']
# Remember resolved deps for this benchmarking session.
xcdeps=r.get('dependencies',{})
# Clean pipeline.
if 'ready' in r: del(r['ready'])
if 'fail' in r: del(r['fail'])
if 'return' in r: del(r['return'])
pipeline=copy.deepcopy(r)
for lib_uoa in udepl:
# Load ArmCL lib.
ii={'action':'load',
'module_uoa':'env',
'data_uoa':lib_uoa}
r=ck.access(ii)
if r['return']>0: return r
lib_name=r['data_name']
lib_tags=r['dict']['customize']['version']
# Skip some libs with "in [..]" or "not in [..]".
if arg.accuracy and lib_tags in [ ]: continue
skip_compile='no'
# For each MobileNets model.*************************************************
for model_uoa in udepm:
# Load model.
ii={'action':'load',
'module_uoa':'env',
'data_uoa':model_uoa}
r=ck.access(ii)
if r['return']>0: return r
model_name=r['data_name']
if 'mobilenet' not in r['dict']['tags']:
continue
alpha = float(r['dict']['env']['CK_ENV_TENSORFLOW_MODEL_MOBILENET_MULTIPLIER'])
rho = int(r['dict']['env']['CK_ENV_TENSORFLOW_MODEL_MOBILENET_RESOLUTION'])
record_repo='local'
record_uoa='mobilenets-'+experiment_type+'-'+str(rho)+'-'+str(alpha)+'-tensorflow-'+lib_tags
# Prepare pipeline.
ck.out('---------------------------------------------------------------------------------------')
ck.out('%s - %s' % (lib_name, lib_uoa))
ck.out('%s - %s' % (model_name, model_uoa))
ck.out('Experiment - %s:%s' % (record_repo, record_uoa))
# Prepare autotuning input.
cpipeline=copy.deepcopy(pipeline)
# Reset deps and change UOA.
new_deps={'library':copy.deepcopy(depl),
'weights':copy.deepcopy(depm)}
new_deps['library']['uoa']=lib_uoa
new_deps['weights']['uoa']=model_uoa
jj={'action':'resolve',
'module_uoa':'env',
'host_os':hos,
'target_os':tos,
'device_id':tdid,
'deps':new_deps}
r=ck.access(jj)
if r['return']>0: return r
cpipeline['dependencies'].update(new_deps)
cpipeline['no_clean']=skip_compile
cpipeline['no_compile']=skip_compile
# Prepare common meta for ReQuEST tournament
features=copy.deepcopy(cpipeline['features'])
platform_dict['features'].update(features)
r=ck.access({'action':'prepare_common_meta',
'module_uoa':'request.asplos18',
'platform_dict':platform_dict,
'deps':cpipeline['dependencies'],
'request_dict':request_dict})
if r['return']>0: return r
record_dict=r['record_dict']
meta=r['meta']
if random_name:
rx=ck.gen_uid({})
if rx['return']>0: return rx
record_uoa=rx['data_uid']
tags=r['tags']
tags.append(experiment_type)
tags.append('explore-mobilenets-'+experiment_type)
tags.append(lib_tags)
tags.append(platform_tags)
tags.append(str(rho))
tags.append(str(alpha))
ii={'action':'autotune',
'module_uoa':'pipeline',
'data_uoa':'program',
'choices_order':[
[
'##choices#env#CK_BATCH_SIZE'
],
[
'##choices#env#CK_CONVOLUTION_METHOD_HINT'
],
[
'##choices#env#CK_ENV_MOBILENET_RESOLUTION'
],
[
'##choices#env#CK_ENV_MOBILENET_WIDTH_MULTIPLIER'
]
],
'choices_selection':[
{'type':'loop', 'start':bs['start'], 'stop':bs['stop'], 'step':bs['step'], 'default':bs['default']},
{'type':'loop', 'start':ch['start'], 'stop':ch['stop'], 'step':ch['step'], 'default':ch['default']},
{'type':'loop', 'choice': [rho], 'default': 224},
{'type':'loop', 'choice': [alpha], 'default': 1.0},
],
'features_keys_to_process':['##choices#*'],
'iterations':-1,
'repetitions': num_repetitions,
'record':'yes',
'record_failed':'yes',
'record_params':{
'search_point_by_features':'yes'
},
'tags':tags,
'meta':meta,
'record_dict':record_dict,
'record_repo':record_repo,
'record_uoa':record_uoa,
'pipeline':cpipeline,
'out':'con'
}
r=ck.access(ii)
if r['return']>0: return r
fail=r.get('fail','')
if fail=='yes':
return {'return':10, 'error':'pipeline failed ('+r.get('fail_reason','')+')'}
### end pipeline
return {'return':0}
def do(i, arg):
# Process arguments.
if (arg.accuracy):
experiment_type = 'accuracy'
num_repetitions = 1
cmd_key = 'test'
else:
experiment_type = 'performance'
num_repetitions = arg.repetitions
cmd_key = 'classify'
random_name = arg.random_name
share_platform = arg.share_platform
# Detect basic platform info.
ii = {'action': 'detect', 'module_uoa': 'platform', 'out': 'con'}
if share_platform: ii['exchange'] = 'yes'
r = ck.access(ii)
if r['return'] > 0: return r
# Keep to prepare ReQuEST meta.
platform_dict = copy.deepcopy(r)
# Host and target OS params.
hos = r['host_os_uoa']
hosd = r['host_os_dict']
tos = r['os_uoa']
tosd = r['os_dict']
tdid = r['device_id']
ii = {
'action': 'show',
'module_uoa': 'env',
'tags': 'dataset,imagenet,raw,val'
}
rx = ck.access(ii)
if len(rx['lst']) == 0: return rx
# FIXME: It's probably better to use CK_ENV_DATASET_IMAGE_DIR.
img_dir_val = rx['lst'][0]['meta']['env']['CK_CAFFE_IMAGENET_VAL']
if arg.accuracy:
batch_count = len([
f for f in os.listdir(img_dir_val) if f.endswith('.JPEG')
and os.path.isfile(os.path.join(img_dir_val, f))
])
else:
batch_count = 1
ii = {
'action': 'show',
'module_uoa': 'env',
'tags': 'dataset,imagenet,aux'
}
rx = ck.access(ii)
if len(rx['lst']) == 0: return rx
img_dir_aux = rx['lst'][0]['meta']['env']['CK_ENV_DATASET_IMAGENET_AUX']
ii = {'action': 'load', 'module_uoa': 'program', 'data_uoa': program}
rx = ck.access(ii)
if rx['return'] > 0: return rx
mm = rx['dict']
# Get compile-time and run-time deps.
cdeps = mm.get('compile_deps', {})
rdeps = mm.get('run_deps', {})
rdeps.update(mm.get('run_cmds', {}).get(cmd_key, {}).get('run_deps', {}))
# Merge rdeps with cdeps for setting up the pipeline (which uses
# common deps), but tag them as "for_run_time".
for k in rdeps:
cdeps[k] = rdeps[k]
cdeps[k]['for_run_time'] = 'yes'
depl = copy.deepcopy(cdeps['lib-nnvm-tvm'])
if (arg.tos is not None) and (arg.did is not None):
tos = arg.tos
tdid = arg.did
ii = {
'action': 'resolve',
'module_uoa': 'env',
'host_os': hos,
'target_os': tos,
'device_id': tdid,
'out': 'con',
'deps': {
'lib-nnvm-tvm': copy.deepcopy(depl)
},
'quiet': 'yes'
}
r = ck.access(ii)
if r['return'] > 0: return r
udepl = r['deps']['lib-nnvm-tvm'].get('choices', [])
if len(udepl) == 0:
return {'return': 1, 'error': 'no installed TVM'}
cdeps['lib-nnvm-tvm']['uoa'] = udepl[0]
depm = copy.deepcopy(cdeps['mxnet-model'])
ii = {
'action': 'resolve',
'module_uoa': 'env',
'host_os': hos,
'target_os': tos,
'device_id': tdid,
'out': 'con',
'deps': {
'mxnet-model': copy.deepcopy(depm)
},
'quiet': 'yes'
}
r = ck.access(ii)
if r['return'] > 0: return r
udepm = r['deps']['mxnet-model'].get('choices', [])
if len(udepm) == 0:
return {'return': 1, 'error': 'no installed mxnet-model'}
cdeps['mxnet-model']['uoa'] = udepm[0]
ii = {
'action': 'pipeline',
'prepare': 'yes',
'dependencies': cdeps,
'module_uoa': 'program',
'data_uoa': program,
'target_os': tos,
'device_id': tdid,
'cmd_key': cmd_key,
'dataset_uoa': dataset,
'no_state_check': 'yes',
'no_compiler_description': 'yes',
'skip_calibration': 'yes',
'cpu_freq': 'max',
'gpu_freq': 'max',
'env': {
'STAT_REPEAT': 5
},
'flags': '-O3',
'speed': 'no',
'energy': 'no',
'skip_print_timers': 'yes',
'out': 'con'
}
r = ck.access(ii)
if r['return'] > 0: return r
fail = r.get('fail', '')
if fail == 'yes':
return {
'return': 10,
'error': 'pipeline failed (' + r.get('fail_reason', '') + ')'
}
ready = r.get('ready', '')
if ready != 'yes':
return {'return': 11, 'error': 'pipeline not ready'}
state = r['state']
tmp_dir = state['tmp_dir']
# Remember resolved deps for this benchmarking session.
xcdeps = r.get('dependencies', {})
# Clean pipeline.
if 'ready' in r: del (r['ready'])
if 'fail' in r: del (r['fail'])
if 'return' in r: del (r['return'])
pipeline = copy.deepcopy(r)
for precision in ['float32', 'float16']:
ck.out('')
ck.out('Precision: ' + precision)
ck.out('')
xprecision = 'fp32'
if precision == 'float16': xprecision = 'fp16'
for lib_uoa in udepl:
# Load lib.
ii = {'action': 'load', 'module_uoa': 'env', 'data_uoa': lib_uoa}
r = ck.access(ii)
if r['return'] > 0: return r
lib_name = r['data_name']
lib_tags = r['dict']['customize']['version']
# Skip some libs with "in [..]" or "not in [..]".
if arg.accuracy and lib_tags not in use_lib_tags: continue
skip_compile = 'no'
# For each model*************************************************
for model_uoa in udepm:
# Load model.
ii = {
'action': 'load',
'module_uoa': 'env',
'data_uoa': model_uoa
}
r = ck.access(ii)
if r['return'] > 0: return r
model_real_tags = r['dict']['tags']
# Get the tags from e.g. 'Caffe model (net and weights) (inception-v3, fp32)'
model_name = r['data_name']
model_tags = re.match(
'MXNet model \(net and weights\) \((?P<tags>.*)\)',
model_name)
model_tags = model_tags.group('tags').replace(' ', '').replace(
',', '-')
record_repo = 'local'
record_uoa = 'ck-request-asplos18-nnvm-tvm-arm-' + experiment_type + '.' + model_tags + '.' + xprecision
# Prepare pipeline.
ck.out(
'---------------------------------------------------------------------------------------'
)
ck.out('%s - %s' % (lib_name, lib_uoa))
ck.out('%s - %s' % (model_name, model_uoa))
ck.out('Experiment - %s:%s' % (record_repo, record_uoa))
# Prepare autotuning input.
cpipeline = copy.deepcopy(pipeline)
cpipeline['choices']['env']['CK_TVM_DTYPE'] = precision
# Reset deps and change UOA.
new_deps = {
'lib-nnvm-tvm': copy.deepcopy(depl),
'mxnet-model': copy.deepcopy(depm)
}
new_deps['lib-nnvm-tvm']['uoa'] = lib_uoa
new_deps['mxnet-model']['uoa'] = model_uoa
jj = {
'action': 'resolve',
'module_uoa': 'env',
'host_os': hos,
'target_os': tos,
'device_id': tdid,
'deps': new_deps
}
r = ck.access(jj)
if r['return'] > 0: return r
cpipeline['dependencies'].update(new_deps)
cpipeline['no_clean'] = skip_compile
cpipeline['no_compile'] = skip_compile
# Prepare common meta for ReQuEST tournament
features = copy.deepcopy(cpipeline['features'])
platform_dict['features'].update(features)
r = ck.access({
'action': 'prepare_common_meta',
'module_uoa': 'request.asplos18',
'platform_dict': platform_dict,
'deps': cpipeline['dependencies'],
'request_dict': request_dict
})
if r['return'] > 0: return r
record_dict = r['record_dict']
meta = r['meta']
meta['model_precision'] = xprecision
if random_name:
rx = ck.gen_uid({})
if rx['return'] > 0: return rx
record_uoa = rx['data_uid']
tags = r['tags']
tags.append(experiment_type)
tags.append('explore-mxnet-' + experiment_type)
tags.append(lib_tags)
tags.append(platform_tags)
ii = {
'action':
'autotune',
'module_uoa':
'pipeline',
'data_uoa':
'program',
'choices_order': [['##choices#env#CK_BATCH_SIZE']],
'choices_selection': [{
'type': 'loop',
'start': bs['start'],
'stop': bs['stop'],
'step': bs['step'],
'default': bs['default']
}],
'features_keys_to_process': ['##choices#*'],
'iterations':
-1,
'repetitions':
num_repetitions,
'record':
'yes',
'record_failed':
'yes',
'record_params': {
'search_point_by_features': 'yes'
},
'tags':
tags,
'meta':
meta,
'record_dict':
record_dict,
'record_repo':
record_repo,
'record_uoa':
record_uoa,
'pipeline':
cpipeline,
'out':
'con'
}
r = ck.access(ii)
if r['return'] > 0: return r
fail = r.get('fail', '')
if fail == 'yes':
return {
'return':
10,
'error':
'pipeline failed (' + r.get('fail_reason', '') + ')'
}
### end pipeline
return {'return': 0}
def do(i, arg):
random_name = arg.random_name
# Detect basic platform info.
ii={'action':'detect',
'module_uoa':'platform',
'target':target,
'out':'out'}
r=ck.access(ii)
if r['return']>0: return r
# Keep to prepare ReQuEST meta.
platform_dict=copy.deepcopy(r)
# Host and target OS params.
hos=r['host_os_uoa']
hosd=r['host_os_dict']
tos=r['os_uoa']
tosd=r['os_dict']
tdid=r['device_id']
# Program and command.
program='request-tvm-vta-pynq'
cmd_key='classify'
dset='e496fdf046e6ac13' # image-jpeg-dnn-cat
# Load program meta and desc to check deps.
ii={'action':'load',
'module_uoa':'program',
'data_uoa':program}
rx=ck.access(ii)
if rx['return']>0: return rx
mm=rx['dict']
# Get deps
deps=mm.get('run_cmds',{}).get('classify',{}).get('run_deps',{})
# Models
depm=copy.deepcopy(deps['model'])
ii={'action':'resolve',
'module_uoa':'env',
'host_os':hos,
'target_os':tos,
'device_id':tdid,
'out':'con',
'quiet':'yes',
'deps':{'model':copy.deepcopy(depm)}
}
r=ck.access(ii)
if r['return']>0: return r
udepm=r['deps']['model'].get('choices',[]) # All UOAs of env for Caffe models.
if len(udepm)==0:
return {'return':1, 'error':'no installed VTA models'}
deps['model']['uoa']=udepm[0]
ii={'action':'pipeline',
'prepare':'yes',
'dependencies':deps,
'module_uoa':'program',
'data_uoa':program,
'cmd_key':cmd_key,
'dataset_uoa':dset,
'target':target,
'target_os':tos,
'device_id':tdid,
'no_state_check':'yes',
'no_compiler_description':'yes',
'skip_calibration':'yes',
# not needed since it's not running on host ...
# 'cpu_freq':'max',
# 'gpu_freq':'max',
'flags':'-O3',
'speed':'no',
'energy':'no',
'skip_print_timers':'yes',
'out':'con'
}
r=ck.access(ii)
if r['return']>0: return r
fail=r.get('fail','')
if fail=='yes':
return {'return':10, 'error':'pipeline failed ('+r.get('fail_reason','')+')'}
ready=r.get('ready','')
if ready!='yes':
return {'return':11, 'error':'pipeline not ready'}
state=r['state']
tmp_dir=state['tmp_dir']
# Remember resolved deps for this benchmarking session.
xdeps=r.get('dependencies',{})
# Clean pipeline.
if 'ready' in r: del(r['ready'])
if 'fail' in r: del(r['fail'])
if 'return' in r: del(r['return'])
pipeline=copy.deepcopy(r)
for dummy in ['dummy']:
# For each model.*************************************************
for model_uoa in udepm:
# Load model.
ii={'action':'load',
'module_uoa':'env',
'data_uoa':model_uoa}
r=ck.access(ii)
if r['return']>0: return r
model_real_tags=r['dict']['tags']
# Get the tags from e.g. 'Caffe model (net and weights) (inception-v3, fp32)'
model_name=r['data_name']
model_tags = re.match('VTA model \(net and weights\) \((?P<tags>.*)\)', model_name)
model_tags = model_tags.group('tags').replace(' ', '').replace(',', '-')
record_repo='local'
record_uoa='ck-request-asplos18-tvm-fpga-performance-'+platform_tags+'.'+model_tags
# Prepare pipeline.
ck.out('---------------------------------------------------------------------------------------')
ck.out('%s - %s' % (model_name, model_uoa))
ck.out('Experiment - %s:%s' % (record_repo, record_uoa))
# Prepare autotuning input.
cpipeline=copy.deepcopy(pipeline)
# Reset deps and change UOA.
new_deps={'model':copy.deepcopy(depm)}
new_deps['model']['uoa']=model_uoa
jj={'action':'resolve',
'module_uoa':'env',
'host_os':hos,
'target_os':tos,
'device_id':tdid,
'deps':new_deps}
r=ck.access(jj)
if r['return']>0: return r
cpipeline['dependencies'].update(new_deps)
cpipeline['cmd_key']=cmd_key
# Prepare common meta for ReQuEST tournament
features=copy.deepcopy(cpipeline['features'])
platform_dict['features'].update(features)
r=ck.access({'action':'prepare_common_meta',
'module_uoa':'request.asplos18',
'platform_dict':platform_dict,
'deps':cpipeline['dependencies'],
'request_dict':request_dict})
if r['return']>0: return r
record_dict=r['record_dict']
meta=r['meta']
if random_name:
rx=ck.gen_uid({})
if rx['return']>0: return rx
record_uoa=rx['data_uid']
tags=r['tags']
tags.append(program)
tags.append(model_tags)
tags.append(platform_tags)
ii={'action':'autotune',
'target':target,
'module_uoa':'pipeline',
'data_uoa':'program',
'iterations':1,
'repetitions':num_repetitions,
'record':'yes',
'record_failed':'yes',
'record_params':{
'search_point_by_features':'yes'
},
'tags':tags,
'meta':meta,
'record_dict':record_dict,
'record_repo':record_repo,
'record_uoa':record_uoa,
'pipeline':cpipeline,
'out':'con'}
r=ck.access(ii)
if r['return']>0: return r
fail=r.get('fail','')
if fail=='yes':
return {'return':10, 'error':'pipeline failed ('+r.get('fail_reason','')+')'}
skip_compile='yes'
return {'return':0}
def do(i, arg):
# Process arguments.
if (arg.accuracy):
experiment_type = 'accuracy'
num_repetitions = 1
kernel_tuners = [default_kernel_tuner]
else:
experiment_type = 'performance'
num_repetitions = arg.repetitions
random_name = arg.random_name
share_platform = arg.share_platform
# Detect basic platform info.
ii = {'action': 'detect', 'module_uoa': 'platform', 'out': 'con'}
if share_platform: ii['exchange'] = 'yes'
r = ck.access(ii)
if r['return'] > 0: return r
# Keep to prepare ReQuEST meta.
platform_dict = copy.deepcopy(r)
# Host and target OS params.
hos = r['host_os_uoa']
hosd = r['host_os_dict']
tos = r['os_uoa']
tosd = r['os_dict']
tdid = r['device_id']
# Determine platform tags: if one of the known platforms, use its id; otherwise, 'unknown-platform'.
# FIXME: only works when the target platform is the same as the host platform.
platform_tags = platform_config.get(r['features']['platform']['model'],
{'id': 'unknown-platform'})['id']
# The only supported program.
program = 'mobilenets-armcl-opencl'
# Select ImageNet dataset.
r = select_ImageNet()
if r['return'] > 0: return r
imagenet_val = r['dataset']
img_dir_val = get_ImageNet_path(imagenet_val)
ck.out('ImageNet path: ' + img_dir_val)
if arg.accuracy:
# Use as many batches (of size 1), as there are JPEG images in the directory.
batch_count = len([
f for f in os.listdir(img_dir_val) if f.endswith('.JPEG')
and os.path.isfile(os.path.join(img_dir_val, f))
])
else:
# FIXME: Use 2 batches, using the first for warm up (to be excluded from the average)?
batch_count = 1
# Restrict accuracy testing to the ReQuEST fork of ArmCL. TODO: restrict to direct convolution for large datasets.
if arg.accuracy and batch_count > 500:
use_lib_tags = ['request-d8f69c13', '18.05-0acd60ed-request']
else:
use_lib_tags = [
'request-d8f69c13', '18.11-b9abeae08', '18.08-52ba29e9',
'18.05-0acd60ed-request', '18.05-b3a371bc', '18.03-e40997bb',
'18.01-f45d5a9b', '17.12-48bc34ea'
]
# On Firefly-RK3399, the version hash has only 7 characters, not 8.
if platform_tags == 'firefly-rk3399':
use_lib_tags = [tag[:-1] for tag in use_lib_tags]
ii = {
'action': 'show',
'module_uoa': 'env',
'tags': 'dataset,imagenet,aux'
}
rx = ck.access(ii)
if len(rx['lst']) == 0: return rx
img_dir_aux = rx['lst'][0]['meta']['env']['CK_ENV_DATASET_IMAGENET_AUX']
ii = {'action': 'load', 'module_uoa': 'program', 'data_uoa': program}
rx = ck.access(ii)
if rx['return'] > 0: return rx
mm = rx['dict']
# Get compile-time and run-time deps.
cdeps = mm.get('compile_deps', {})
rdeps = mm.get('run_deps', {})
# Merge rdeps with cdeps for setting up the pipeline (which uses
# common deps), but tag them as "for_run_time".
for k in rdeps:
cdeps[k] = rdeps[k]
cdeps[k]['for_run_time'] = 'yes'
depl = copy.deepcopy(cdeps['library'])
if (arg.tos is not None) and (arg.did is not None):
tos = arg.tos
tdid = arg.did
ii = {
'action': 'resolve',
'module_uoa': 'env',
'host_os': hos,
'target_os': tos,
'device_id': tdid,
'out': 'con',
'deps': {
'library': copy.deepcopy(depl)
},
'quiet': 'yes'
}
r = ck.access(ii)
if r['return'] > 0: return r
udepl = r['deps']['library'].get(
'choices', []) # All UOAs of env for Arm Compute Libraries.
if len(udepl) == 0:
return {'return': 1, 'error': 'no installed Arm Compute Libraries'}
cdeps['library']['uoa'] = udepl[0]
depm = copy.deepcopy(cdeps['weights'])
ii = {
'action': 'resolve',
'module_uoa': 'env',
'host_os': hos,
'target_os': tos,
'device_id': tdid,
'out': 'con',
'deps': {
'weights': copy.deepcopy(depm)
},
'quiet': 'yes'
}
r = ck.access(ii)
if r['return'] > 0: return r
udepm = r['deps']['weights'].get('choices', [])
if len(udepm) == 0:
return {'return': 1, 'error': 'no installed Weights'}
cdeps['library']['uoa'] = udepl[0]
cdeps['weights']['uoa'] = udepm[0]
ii = {
'action': 'pipeline',
'prepare': 'yes',
'dependencies': cdeps,
'module_uoa': 'program',
'data_uoa': program,
'target_os': tos,
'device_id': tdid,
'no_state_check': 'yes',
'no_compiler_description': 'yes',
'skip_calibration': 'yes',
'env': {
'CK_ENV_DATASET_IMAGENET_VAL': img_dir_val,
'CK_BATCH_COUNT': batch_count,
'CK_SKIP_IMAGES': 0
},
'cpu_freq': 'max',
'gpu_freq': 'max',
'flags': '-O3',
'speed': 'no',
'energy': 'no',
'skip_print_timers': 'yes',
'out': 'con'
}
r = ck.access(ii)
if r['return'] > 0: return r
fail = r.get('fail', '')
if fail == 'yes':
return {
'return': 10,
'error': 'pipeline failed (' + r.get('fail_reason', '') + ')'
}
ready = r.get('ready', '')
if ready != 'yes':
return {'return': 11, 'error': 'pipeline not ready'}
state = r['state']
tmp_dir = state['tmp_dir']
# Remember resolved deps for this benchmarking session.
xcdeps = r.get('dependencies', {})
# Clean pipeline.
if 'ready' in r: del (r['ready'])
if 'fail' in r: del (r['fail'])
if 'return' in r: del (r['return'])
experiment_count = 0
pipeline = copy.deepcopy(r)
for lib_uoa in udepl:
# Load ArmCL lib.
ii = {'action': 'load', 'module_uoa': 'env', 'data_uoa': lib_uoa}
r = ck.access(ii)
if r['return'] > 0: return r
lib_name = r['data_name']
lib_tags = r['dict']['customize']['version']
# Skip other libraries if one is explicitly specified.
if arg.library_uoa and lib_uoa != arg.library_uoa: continue
# Skip some libs with "in [..]" or "not in [..]".
if arg.accuracy and lib_tags not in use_lib_tags: continue
skip_compile = 'no'
# For each MobileNets model.*************************************************
for model_uoa in udepm:
# Load model.
ii = {'action': 'load', 'module_uoa': 'env', 'data_uoa': model_uoa}
r = ck.access(ii)
if r['return'] > 0: return r
model_name = r['data_name']
# Skip other models if one is explicitly specified.
if arg.model_uoa and model_uoa != arg.model_uoa: continue
# Skip aggregate MobileNets packages.
if 'mobilenet-all' in r['dict']['tags']: continue
batch_size = 1
version = 1
multiplier = float(r['dict']['env']['CK_ENV_MOBILENET_MULTIPLIER'])
resolution = int(r['dict']['env']['CK_ENV_MOBILENET_RESOLUTION'])
record_repo = 'local'
record_uoa = '{}-{}-{}-mobilenet-v{}-{:.2f}-{}'.format(
experiment_type, platform_tags, lib_tags, version, multiplier,
resolution)
# Skip the experiment if it already exists.
if arg.resume:
r = ck.access({
'action': 'search',
'module_uoa': 'experiment',
'repo_uoa': record_repo,
'data_uoa': record_uoa
})
if r['return'] > 0: return r
if len(r['lst']) > 0:
ck.out('Experiment "%s" already exists, skipping...' %
record_uoa)
continue
# Prepare pipeline.
ck.out(
'---------------------------------------------------------------------------------------'
)
ck.out('%s - %s' % (lib_name, lib_uoa))
ck.out('%s - %s' % (model_name, model_uoa))
ck.out('Experiment - %s:%s' % (record_repo, record_uoa))
experiment_count += 1
# Prepare autotuning input.
cpipeline = copy.deepcopy(pipeline)
# Reset deps and change UOA.
new_deps = {
'library': copy.deepcopy(depl),
'weights': copy.deepcopy(depm)
}
new_deps['library']['uoa'] = lib_uoa
new_deps['weights']['uoa'] = model_uoa
jj = {
'action': 'resolve',
'module_uoa': 'env',
'host_os': hos,
'target_os': tos,
'device_id': tdid,
'deps': new_deps
}
r = ck.access(jj)
if r['return'] > 0: return r
cpipeline['dependencies'].update(new_deps)
cpipeline['no_clean'] = skip_compile
cpipeline['no_compile'] = skip_compile
# Prepare common meta for ReQuEST tournament
features = copy.deepcopy(cpipeline['features'])
platform_dict['features'].update(features)
r = ck.access({
'action': 'prepare_common_meta',
'module_uoa': 'request.asplos18',
'platform_dict': platform_dict,
'deps': cpipeline['dependencies'],
'request_dict': request_dict
})
if r['return'] > 0: return r
record_dict = r['record_dict']
meta = r['meta']
if random_name:
rx = ck.gen_uid({})
if rx['return'] > 0: return rx
record_uoa = rx['data_uid']
tags = r['tags']
tags.append(experiment_type)
tags.append('explore-mobilenets-' + experiment_type)
tags.append(lib_tags)
tags.append(platform_tags)
tags.append(str(resolution))
tags.append(str(multiplier))
tags.append('mobilenet-v{}'.format(version))
tags.append('mobilenet-v{}-{:.2f}-{}'.format(
version, multiplier, resolution))
ii = {
'action':
'autotune',
'module_uoa':
'pipeline',
'data_uoa':
'program',
'choices_order': [
['##choices#env#CK_BATCH_SIZE'],
['##choices#env#CK_ENV_MOBILENET_VERSION'],
['##choices#env#CK_ENV_MOBILENET_MULTIPLIER'],
['##choices#env#CK_ENV_MOBILENET_RESOLUTION'],
['##choices#env#CK_CONVOLUTION_METHOD'],
['##choices#env#CK_LWS_TUNER_TYPE'],
['##choices#env#CK_DATA_LAYOUT'],
],
'choices_selection': [
{
'type': 'loop',
'choice': [batch_size],
'default': 1
}, # Only batch_size=1 is supported.
{
'type': 'loop',
'choice': [version],
'default': 1
}, # Only version=1 is supported.
{
'type': 'loop',
'choice': [multiplier],
'default': 1.0
},
{
'type': 'loop',
'choice': [resolution],
'default': 224
},
{
'type': 'loop',
'choice': convolution_methods,
'default': default_convolution_method
},
{
'type': 'loop',
'choice': kernel_tuners,
'default': default_kernel_tuner
},
{
'type': 'loop',
'choice': data_layouts,
'default': default_data_layout
},
],
'features_keys_to_process': ['##choices#*'],
'iterations':
-1,
'repetitions':
num_repetitions,
'record':
'yes',
'record_failed':
'yes',
'record_params': {
'search_point_by_features': 'yes'
},
'tags':
tags,
'meta':
meta,
'record_dict':
record_dict,
'record_repo':
record_repo,
'record_uoa':
record_uoa,
'pipeline':
cpipeline,
'out':
'con'
}
if not arg.dry_run:
r = ck.access(ii)
if r['return'] > 0: return r
fail = r.get('fail', '')
if fail == 'yes':
return {
'return':
10,
'error':
'pipeline failed (' + r.get('fail_reason', '') + ')'
}
if arg.dry_run:
ck.out(
'---------------------------------------------------------------------------------------'
)
ck.out('Experiment count: %d' % experiment_count)
### end pipeline
return {'return': 0}
def do(i, arg):
# Process arguments.
if (arg.accuracy):
experiment_type = 'accuracy'
num_repetitions = 1
else:
experiment_type = 'performance'
num_repetitions = arg.repetitions
random_name = arg.random_name
share_platform = arg.share_platform
# Detect basic platform info.
ii = {'action': 'detect', 'module_uoa': 'platform', 'out': 'con'}
if share_platform: ii['exchange'] = 'yes'
r = ck.access(ii)
if r['return'] > 0: return r
# Keep to prepare ReQuEST meta.
platform_dict = copy.deepcopy(r)
# Host and target OS params.
hos = r['host_os_uoa']
hosd = r['host_os_dict']
tos = r['os_uoa']
tosd = r['os_dict']
tdid = r['device_id']
# Determine platform tags: if one of the known platforms, use its id; otherwise, 'unknown-platform'.
# FIXME: only works when the target platform is the same as the host platform.
platform_tags = platform_config.get(r['features']['platform']['model'],
{'id': 'unknown-platform'})['id']
# Select one of `image-classification-tf*`.
r = select_program()
if r['return'] > 0: return r
program = r['program']
# `lib_type` is used to distinguish result repo records: tf-py, tf-cpp, tflite
lib_type = program[len('image-classification-'):]
# Select ImageNet dataset.
r = select_ImageNet()
if r['return'] > 0: return r
imagenet_val = r['dataset']
img_dir_val = get_ImageNet_path(imagenet_val)
ck.out('ImageNet path: ' + img_dir_val)
if (arg.accuracy):
# Use as many batches (of size 1), as there are JPEG images in the directory.
batch_count = len([
f for f in os.listdir(img_dir_val) if f.endswith('.JPEG')
and os.path.isfile(os.path.join(img_dir_val, f))
])
else:
# Use 2 batches, using the first for warm up (excluded from the average).
batch_count = 2
ii = {
'action': 'show',
'module_uoa': 'env',
'tags': 'dataset,imagenet,aux'
}
rx = ck.access(ii)
if len(rx['lst']) == 0: return rx
imagenet_aux = rx['lst'][0]
img_dir_aux = imagenet_aux['meta']['env']['CK_ENV_DATASET_IMAGENET_AUX']
ii = {'action': 'load', 'module_uoa': 'program', 'data_uoa': program}
rx = ck.access(ii)
if rx['return'] > 0: return rx
mm = rx['dict']
# Get compile-time and run-time deps.
cdeps = mm.get('compile_deps', {})
rdeps = mm.get('run_deps', {})
# Merge rdeps with cdeps for setting up the pipeline (which uses
# common deps), but tag them as "for_run_time".
for k in rdeps:
cdeps[k] = rdeps[k]
cdeps[k]['for_run_time'] = 'yes'
if 'lib-tensorflow' in cdeps:
library_key = 'lib-tensorflow'
elif 'library' in cdeps:
library_key = 'library'
else:
return {'return': 1, 'error': 'no library dependency in program meta'}
depl = copy.deepcopy(cdeps[library_key])
if (arg.tos is not None) and (arg.did is not None):
tos = arg.tos
tdid = arg.did
ii = {
'action': 'resolve',
'module_uoa': 'env',
'host_os': hos,
'target_os': tos,
'device_id': tdid,
'out': 'con',
'deps': {
'library': copy.deepcopy(depl)
},
'quiet': 'yes'
}
r = ck.access(ii)
if r['return'] > 0: return r
udepl = r['deps']['library'].get(
'choices', []) # All UOAs of env for TensorFlow libs.
if len(udepl) == 0:
return {'return': 1, 'error': 'no installed TensorFlow libs'}
cdeps[library_key]['uoa'] = udepl[0]
depm = copy.deepcopy(cdeps['weights'])
ii = {
'action': 'resolve',
'module_uoa': 'env',
'host_os': hos,
'target_os': tos,
'device_id': tdid,
'out': 'con',
'deps': {
'weights': copy.deepcopy(depm)
},
'quiet': 'yes'
}
r = ck.access(ii)
if r['return'] > 0: return r
udepm = r['deps']['weights'].get(
'choices', []) # All UOAs of env for TensorFlow models.
if len(udepm) == 0:
return {'return': 1, 'error': 'no installed TensorFlow models'}
cdeps[library_key]['uoa'] = udepl[0]
cdeps['weights']['uoa'] = udepm[0]
cdeps['imagenet-val']['uoa'] = imagenet_val['data_uoa']
cdeps['imagenet-aux']['uoa'] = imagenet_aux['data_uoa']
env = {
'CK_ENV_DATASET_IMAGENET_VAL': img_dir_val,
'CK_BATCH_COUNT': batch_count,
'CK_RESULTS_DIR': 'predictions',
'CK_SKIP_IMAGES': 0
}
if arg.accuracy:
env['CK_SILENT_MODE'] = 'YES'
# Pass additional env vars from input
if 'env' in i:
for key in i['env']:
env[key] = i['env'][key]
ii = {
'action': 'pipeline',
'prepare': 'yes',
'dependencies': cdeps,
'module_uoa': 'program',
'data_uoa': program,
'target_os': tos,
'device_id': tdid,
'no_state_check': 'yes',
'no_compiler_description': 'yes',
'skip_calibration': 'yes',
'env': env,
'cpu_freq': 'max',
'gpu_freq': 'max',
'flags': '-O3',
'speed': 'no',
'energy': 'no',
'skip_print_timers': 'yes',
'out': 'con'
}
r = ck.access(ii)
if r['return'] > 0: return r
fail = r.get('fail', '')
if fail == 'yes':
return {
'return': 10,
'error': 'pipeline failed (' + r.get('fail_reason', '') + ')'
}
ready = r.get('ready', '')
if ready != 'yes':
return {'return': 11, 'error': 'pipeline not ready'}
state = r['state']
tmp_dir = state['tmp_dir']
# Remember resolved deps for this benchmarking session.
xcdeps = r.get('dependencies', {})
# Clean pipeline.
if 'ready' in r: del (r['ready'])
if 'fail' in r: del (r['fail'])
if 'return' in r: del (r['return'])
experiment_count = 0
pipeline = copy.deepcopy(r)
# For each TensorFlow lib.*************************************************
for lib_uoa in udepl:
# Load TensorFlow lib.
ii = {'action': 'load', 'module_uoa': 'env', 'data_uoa': lib_uoa}
r = ck.access(ii)
if r['return'] > 0: return r
lib_name = r['data_name']
lib_tags = lib_type + '-' + r['dict']['customize']['version']
# Skip other libraries if one is explicitly specified.
if arg.library_uoa and lib_uoa != arg.library_uoa: continue
# Skip some libs with "in [..]" or "not in [..]".
if lib_uoa in []: continue
# Skip XLA libs.
if arg.accuracy and r['dict']['customize']['install_env'].get(
'CK_TF_ENABLE_XLA') == 'YES':
continue
skip_compile = 'no'
# For each TensorFlow model.*************************************************
for model_uoa in udepm:
# Load TensorFlow model.
ii = {'action': 'load', 'module_uoa': 'env', 'data_uoa': model_uoa}
r = ck.access(ii)
if r['return'] > 0: return r
model_name = r['data_name']
# Skip non-MobileNets models.
if 'mobilenet' not in r['dict']['tags']: continue
# Skip aggregate MobileNets packages.
if 'mobilenet-all' in r['dict']['tags']: continue
version = r['dict']['env'][
'CK_ENV_TENSORFLOW_MODEL_MOBILENET_VERSION']
multiplier = float(
r['dict']['env']
['CK_ENV_TENSORFLOW_MODEL_MOBILENET_MULTIPLIER'])
resolution = int(r['dict']['env']
['CK_ENV_TENSORFLOW_MODEL_MOBILENET_RESOLUTION'])
record_repo = 'local'
record_uoa = '{}-{}-{}-mobilenet-v{}-{:.2f}-{}'.format(
experiment_type, platform_tags, lib_tags, version, multiplier,
resolution)
# Skip the experiment if it already exists.
if arg.resume:
r = ck.access({
'action': 'search',
'module_uoa': 'experiment',
'repo_uoa': record_repo,
'data_uoa': record_uoa
})
if r['return'] > 0: return r
if len(r['lst']) > 0:
ck.out('Experiment "%s" already exists, skipping...' %
record_uoa)
continue
# Prepare pipeline.
ck.out(
'---------------------------------------------------------------------------------------'
)
ck.out('%s - %s' % (lib_name, lib_uoa))
ck.out('%s - %s' % (model_name, model_uoa))
ck.out('Experiment - %s:%s' % (record_repo, record_uoa))
experiment_count += 1
# Prepare autotuning input.
cpipeline = copy.deepcopy(pipeline)
# Reset deps and change UOA.
new_deps = {
'library': copy.deepcopy(depl),
'weights': copy.deepcopy(depm)
}
new_deps['library']['uoa'] = lib_uoa
new_deps['weights']['uoa'] = model_uoa
jj = {
'action': 'resolve',
'module_uoa': 'env',
'host_os': hos,
'target_os': tos,
'device_id': tdid,
'deps': new_deps
}
r = ck.access(jj)
if r['return'] > 0: return r
cpipeline['dependencies'].update(new_deps)
cpipeline['no_clean'] = skip_compile
cpipeline['no_compile'] = skip_compile
# Prepare common meta for ReQuEST tournament.
features = copy.deepcopy(cpipeline['features'])
platform_dict['features'].update(features)
r = ck.access({
'action': 'prepare_common_meta',
'module_uoa': 'request.asplos18',
'platform_dict': platform_dict,
'deps': cpipeline['dependencies'],
'request_dict': request_dict
})
if r['return'] > 0: return r
record_dict = r['record_dict']
meta = r['meta']
if random_name:
rx = ck.gen_uid({})
if rx['return'] > 0: return rx
record_uoa = rx['data_uid']
tags = r['tags']
tags.append(experiment_type)
tags.append('explore-mobilenets-' + experiment_type)
tags.append(lib_tags)
tags.append(platform_tags)
tags.append(str(resolution))
tags.append(str(multiplier))
tags.append('mobilenet-v{}'.format(version))
tags.append('mobilenet-v{}-{:.2f}-{}'.format(
version, multiplier, resolution))
ii = {
'action':
'autotune',
'module_uoa':
'pipeline',
'data_uoa':
'program',
'choices_order': [
['##choices#env#CK_BATCH_SIZE'],
[
'##choices#env#CK_ENV_TENSORFLOW_MODEL_MOBILENET_VERSION'
],
[
'##choices#env#CK_ENV_TENSORFLOW_MODEL_MOBILENET_MULTIPLIER'
],
[
'##choices#env#CK_ENV_TENSORFLOW_MODEL_MOBILENET_RESOLUTION'
],
],
'choices_selection': [
{
'type': 'loop',
'start': bs['start'],
'stop': bs['stop'],
'step': bs['step'],
'default': bs['default']
},
{
'type': 'loop',
'choice': [version],
'default': version
},
{
'type': 'loop',
'choice': [multiplier],
'default': 1.0
},
{
'type': 'loop',
'choice': [resolution],
'default': 224
},
],
'features_keys_to_process': ['##choices#*'],
'iterations':
-1,
'repetitions':
num_repetitions,
'record':
'yes',
'record_failed':
'yes',
'record_params': {
'search_point_by_features': 'yes'
},
'tags':
tags,
'meta':
meta,
'record_dict':
record_dict,
'record_repo':
record_repo,
'record_uoa':
record_uoa,
'pipeline':
cpipeline,
'out':
'con'
}
if not arg.dry_run:
r = ck.access(ii)
if r['return'] > 0: return r
fail = r.get('fail', '')
if fail == 'yes':
return {
'return':
10,
'error':
'pipeline failed (' + r.get('fail_reason', '') + ')'
}
if arg.dry_run:
ck.out(
'---------------------------------------------------------------------------------------'
)
ck.out('Experiment count: %d' % experiment_count)
### end pipeline
return {'return': 0}
def do(i, arg):
random_name = arg.random_name
target = arg.target
cmd_key = arg.cmd_key
# Detect basic platform info.
ii = {
'action': 'detect',
'module_uoa': 'platform',
'target': target,
'out': 'out'
}
r = ck.access(ii)
if r['return'] > 0: return r
# Keep to prepare ReQuEST meta.
platform_dict = copy.deepcopy(r)
# Host and target OS params.
hos = r['host_os_uoa']
hosd = r['host_os_dict']
tos = r['os_uoa']
tosd = r['os_dict']
tdid = r['device_id']
ii = {
'action': 'pipeline',
'prepare': 'yes',
'module_uoa': 'program',
'data_uoa': program,
'cmd_key': cmd_key,
'target': target,
'target_os': tos,
'device_id': tdid,
'no_state_check': 'yes',
'no_compiler_description': 'yes',
'skip_calibration': 'yes',
'cpu_freq': 'max',
'gpu_freq': 'max',
'flags': '-O3',
'speed': 'no',
'energy': 'no',
'skip_print_timers': 'yes',
'out': 'con'
}
r = ck.access(ii)
if r['return'] > 0: return r
fail = r.get('fail', '')
if fail == 'yes':
return {
'return': 10,
'error': 'pipeline failed (' + r.get('fail_reason', '') + ')'
}
ready = r.get('ready', '')
if ready != 'yes':
return {'return': 11, 'error': 'pipeline not ready'}
state = r['state']
tmp_dir = state['tmp_dir']
# Remember resolved deps for this benchmarking session.
xdeps = r.get('dependencies', {})
# Clean pipeline.
if 'ready' in r: del (r['ready'])
if 'fail' in r: del (r['fail'])
if 'return' in r: del (r['return'])
pipeline = copy.deepcopy(r)
for dummy in ['dummy']:
for dummy2 in ['dummy2']:
record_repo = 'local'
record_uoa = 'ck-request-asplos18-iot-farm-' + cmd_key
# Prepare pipeline.
ck.out(
'---------------------------------------------------------------------------------------'
)
ck.out('CMD key - ' + cmd_key)
ck.out('Experiment - %s:%s' % (record_repo, record_uoa))
# Prepare autotuning input.
cpipeline = copy.deepcopy(pipeline)
cpipeline['cmd_key'] = cmd_key
# Prepare common meta for ReQuEST tournament
features = copy.deepcopy(cpipeline['features'])
platform_dict['features'].update(features)
r = ck.access({
'action': 'prepare_common_meta',
'module_uoa': 'request.asplos18',
'platform_dict': platform_dict,
'deps': cpipeline['dependencies'],
'request_dict': request_dict
})
if r['return'] > 0: return r
record_dict = r['record_dict']
meta = r['meta']
if random_name:
rx = ck.gen_uid({})
if rx['return'] > 0: return rx
record_uoa = rx['data_uid']
tags = r['tags']
tags.append(program)
ii = {
'action': 'autotune',
'target': target,
'module_uoa': 'pipeline',
'data_uoa': 'program',
'iterations': 1,
'repetitions': num_repetitions,
'record': 'yes',
'record_failed': 'yes',
'record_params': {
'search_point_by_features': 'yes'
},
'tags': tags,
'meta': meta,
'record_dict': record_dict,
'record_repo': record_repo,
'record_uoa': record_uoa,
'pipeline': cpipeline,
'out': 'con'
}
r = ck.access(ii)
if r['return'] > 0: return r
fail = r.get('fail', '')
if fail == 'yes':
return {
'return': 10,
'error':
'pipeline failed (' + r.get('fail_reason', '') + ')'
}
skip_compile = 'yes'
return {'return': 0}
