def select_program():
res = ck.access({
'action': 'search',
'module_uoa': 'program',
'data_uoa': 'image-classification-tf*'
})
if res['return'] > 0:
return res
programs = res.get('lst', [])
if programs:
if len(programs) == 1:
return {'return': 0, 'program': programs[0]['data_uoa']}
ck.out('')
ck.out('More than one program is found suitable for this script:')
ck.out('')
res = ck.access({
'action': 'select_uoa',
'module_uoa': 'choice',
'choices': programs
})
if res['return'] > 0:
return res
for p in programs:
if p['data_uid'] == res['choice']:
return {'return': 0, 'program': p['data_uoa']}
return {'return': 1, 'error': 'No related programs found'}
def test(i):
"""
Input: {
}
Output: {
return - return code = 0, if successful
> 0, if error
(error) - error text if return > 0
}
"""
duoa = i.get('data_uoa', '')
if duoa == '':
print('Please provide data entry (ex. ck compile blis_test:make)')
ck.err(i)
muoa = i.get('module_uoa', '')
r = ck.access({'action': 'load', 'module_uoa': muoa, 'data_uoa': duoa})
if r['return'] > 0: return r # use standard error handling in the CK
path = r['path']
'''
ops = ['gemv','ger','hemv','her','her2','trmv','trsv','gemm','hemm','herk','her2k','trmm','trsm']
for op in ops:
i = {'repo_uoa':'ck-blis', 'module_uoa':'results', 'data_uoa':op + '_results', 'data_name':op, 'tags': [op + '_test']}
r = ck.add(i)
#don't return r in this case - this just lets the user know that the entry already exists
if r['return']>0: print(r['error'])
'''
'''
calling runme here because the current bach process will have all of the vars it needs (otherwise have to call source for intel compiler vars)
'''
shell_cmd = 'cd ' + path + ' && make && runme.sh'
i = {
'action': 'virtual',
'module_uoa': 'env',
'tag_groups':
'compiler,lang-cpp openblas mkl,lp64 mkl,core mkl,sequential lib,pthread lib,blis',
'shell_cmd': shell_cmd,
'verbose': 'yes'
}
r = ck.access(i)
if r['return'] > 0: return r # use standard error handling in the CK
return {'return': 0}
def get(self, resource, **kwargs):
# Return path to the APK for Workload resource.owner.name
if (resource.uiauto):
return None
# Remember if debugging/changing to use self.logger.debug as below
# self.logger.debug(dir(resource.owner))
r = ck.access({'module_uoa':'apk',
'action':'load',
'data_uoa':resource.owner.package})
path = r['path']
r_version = kwargs.get('version', getattr(resource, 'version', None))
alt_abi = None
if resource.owner.device.abi in ['arm64-v8','arm64']:
alt_abi = 'armeabi'
if resource.owner.device.abi in ['x86_64']:
alt_abi = 'x86'
for apk in r['dict']['apks']:
if r_version is not None and apk['version'] != r_version:
continue
if resource.owner.device.abi in apk['abis']:
return os.path.join(path,apk['apk_name'])
if alt_abi is not None and alt_abi in apk['abis']:
return os.path.join(path,apk['apk_name'])
self.logger.error("Failed to find APK for {} / {} / {}".format(
resource.owner.package,
r_version,
resource.owner.device.abi))
return None
def login(i):
"""
Input: {
}
Output: {
return [int] - return code = 0 if success or >0 if error
(error) [str] - error string if return>0
}
"""
# Get current configuration
cfg = {}
ii = {
'action': 'load',
'repo_uoa': config.CK_CFG_REPO_UOA,
'module_uoa': config.CK_CFG_MODULE_UID,
'data_uoa': config.CK_CFG_DATA_UOA
}
r = ck.access(ii)
if (r['return'] > 0 and r['return'] != 16): ck.err(r)
# If not found, setup client
if r['return'] == 16:
setup(i)
# Load again
cfg = {}
# ii={'action':'load',
# 'repo_uoa':config.CK_CFG_REPO_UOA,
# 'module_uoa':config.CK_CFG_MODULE_UID,
# 'data_uoa':config.CK_CFG_DATA_UOA}
#
# r=ck.access(ii)
# if r['return']>0: ck.err(r)
r = config.load({})
if r['return'] > 0: return r
cfg = r.get('dict', {})
# Sending request to test connection
r = comm.send({'config': cfg, 'action': 'login'})
if r['return'] > 0: ck.err(r)
# Success
ck.out('CodeReef login tested successfully!')
return 0
def load(i):
"""
Input: {
}
Output: {
return [int] - return code = 0 if success or >0 if error
(error) [str] - error string if return>0
dict [dict] - configuration dictionary
path [str] - path to CK cfg entry
}
"""
global bootstrapping
import os
# Get current configuration
cfg = {
'server_url': CR_DEFAULT_SERVER_URL # Default
}
path = ''
ii = {
'action': 'load',
'repo_uoa': CK_CFG_REPO_UOA,
'module_uoa': CK_CFG_MODULE_UID,
'data_uoa': CK_CFG_DATA_UOA
}
r = ck.access(ii)
if (r['return'] > 0 and r['return'] != 16): return r
if r['return'] == 0:
cfg = r['dict']
path = r['path']
if not bootstrapping and (r['return'] == 16
or cfg.get('bootstrapped', '') != 'yes'):
rx = update({'cfg': cfg})
if rx['return'] > 0: return rx
# Check overriding by env
v = os.environ.get(CR_ENV_USERNAME, '')
if v != '': cfg['username'] = v
v = os.environ.get(CR_ENV_API_KEY, '')
if v != '': cfg['api_key'] = v
return {'return': 0, 'dict': cfg, 'path': path}
def update(i):
"""
Input: {
(force) [bool] - if True, force update
}
Output: {
return [int] - return code = 0 if success or >0 if error
(error) [str] - error string if return>0
}
"""
global bootstrapping
bootstrapping=True
force=i.get('force')
cfg=i.get('cfg',{})
from . import obj
title='Bootstrapping'
if cfg.get('bootstrapped','')=='yes': title='Updating'
ck.out(title+' CodeReef client:')
ck.out('')
for x in CR_SOLUTION_CK_COMPONENTS:
r=obj.download({'cid':x['cid'], 'version':x.get('version',''), 'force':force})
if r['return']>0:
if r['return']!=8: return r
else: ck.out(r['error'])
ck.out('')
# Update cfg
cfg['bootstrapped']='yes'
ii={'action':'update',
'repo_uoa':CK_CFG_REPO_UOA,
'module_uoa':CK_CFG_MODULE_UID,
'data_uoa':CK_CFG_DATA_UOA,
'dict':cfg,
'sort_keys':'yes'}
r=ck.access(ii)
ck.out(title+' finished!')
ck.out('')
return r
def select_ImageNet():
res = ck.access({
'action': 'show',
'module_uoa': 'env',
'tags': 'dataset,imagenet,raw,val'
})
if res['return'] > 0:
return res
datasets = res.get('lst', [])
if datasets:
if len(datasets) == 1:
return {'return': 0, 'dataset': datasets[0]}
ck.out('')
ck.out(
'More than one ImageNet dataset is found suitable for this script:'
)
ck.out('')
dataset_choices = []
for d in datasets:
dataset_choices.append({
'data_uid': d['data_uid'],
'data_uoa': get_ImageNet_path(d)
})
res = ck.access({
'action': 'select_uoa',
'module_uoa': 'choice',
'choices': dataset_choices
})
if res['return'] > 0:
return res
for d in datasets:
if d['data_uid'] == res['choice']:
return {'return': 0, 'dataset': d}
return {'return': 1, 'error': 'No installed ImageNet dataset found'}
def get_downloaded_synsets(repo_uoa='local', module_uoa='repo', data_uoa='imagenet-*'):
r=ck.access({
'action':'search',
'repo_uoa':repo_uoa,
'module_uoa':module_uoa,
'data_uoa':data_uoa
})
if r['return']>0:
print ("CK error: %s" % r['error'])
exit(1)
synsets = []
for item in r['lst']:
synset_uoa = item['data_uoa']
synset = synset_uoa[len('imagenet-'):]
synsets.append(synset)
return synsets
def get_downloaded_synsets(repo_uoa='local',
module_uoa='repo',
data_uoa='imagenet-*'):
r = ck.access({
'action': 'search',
'repo_uoa': repo_uoa,
'module_uoa': module_uoa,
'data_uoa': data_uoa
})
if r['return'] > 0:
print("CK error: %s" % r['error'])
exit(1)
synsets = []
for item in r['lst']:
synset_uoa = item['data_uoa']
synset = synset_uoa[len('imagenet-'):]
synsets.append(synset)
return synsets
def do(i):
top1 = {}
top5 = {}
# List accuracy entries
r = ck.access({
'action': 'search',
'module_uoa': 'experiment',
'data_uoa': 'ck-request-asplos18-caffe-intel-performance-*',
'repo_uoa': 'local',
'add_meta': 'yes'
})
if r['return'] > 0: return r
lst = r['lst']
for q in lst:
duid = q['data_uid']
duoa = q['data_uoa']
path = q['path']
if 'inception-v3' in duoa:
model = 'inception-v3'
model_species = '1b339ddb13408f8f'
elif 'resnet50' in duoa:
model = 'resnet50'
model_species = 'd777f6335496db61'
if model == '':
return {'return': 1, 'error': 'model is not recognized'}
prec = ''
if '-fp32' in duoa:
prec = 'fp32'
elif '-int8' in duoa:
prec = 'int8'
if prec == '':
return {'return': 1, 'error': 'model precision is not recognized'}
ck.out('* ' + duoa + ' / ' + model + ' / ' + prec)
# Search matching accuracy entry (if intel-request)
x = 'ck-request-asplos18-caffe-intel-accuracy.*.' + model + '-' + prec
r = ck.access({
'action': 'search',
'module_uoa': 'experiment',
'data_uoa': x,
'repo_uoa': 'local'
})
if r['return'] > 0: return r
alst = r['lst']
if len(alst) != 1:
return {'return': 1, 'error': 'ambiguity when search for accuracy'}
a = alst[0]
apath = a['path']
# There is only one point normally (no model tuning)
dacc = {}
xacc = os.listdir(apath)
for f in xacc:
if f.endswith('.flat.json'):
r = ck.load_json_file({'json_file': os.path.join(apath, f)})
if r['return'] > 0: return r
dx = r['dict']
# Get only accuracy keys (convert to common format)
for k in dx:
if k.startswith('##characteristics#run#acc/top-'):
k1 = '##characteristics#run#accuracy_top' + k[30:]
dacc[k1] = dx[k]
elif k.startswith('##characteristics#run#accuracy/top-'):
k1 = '##characteristics#run#accuracy_top' + k[35:]
dacc[k1] = dx[k]
break
if len(dacc) == 0:
return {
'return': 1,
'error': 'strange - no match for accuracy entries'
}
# Iterating over points to aggregate
dperf = os.listdir(path)
for f in dperf:
if f.endswith('.flat.json'):
ck.out(' * ' + f)
# Load performance file
p1 = os.path.join(path, f)
r = ck.load_json_file({'json_file': p1})
if r['return'] > 0: return r
d = r['dict']
# Merge accuracy
for k in dacc:
d[k] = dacc[k]
# Save updated dict
r = ck.save_json_to_file({
'json_file': p1,
'dict': d,
'sort_keys': 'yes'
})
if r['return'] > 0: return r
return {'return': 0}
import ck.kernel as ck
r=ck.access('list compiler:* out=none')
if r['return']>0:
print 'Error: '+r['error']
exit(1)
for q in r['lst']:
m=q['module_uoa']
d=q['data_uoa']
ruoa=q['repo_uoa']
print d
r=ck.access({'action':'load',
'module_uoa':m,
'data_uoa':d})
if r['return']>0:
print 'Error: '+r['error']
exit(1)
dd=r['dict']
dm=r['info']
dx=r.get('desc',{})
zz=dd.get('all_compiler_flags_desc',{})
if len(zz)>0:
dx['all_compiler_flags_desc']=zz
if 'all_compiler_flags_desc' in dd: del(dd['all_compiler_flags_desc'])
rvd=dd.get('run_vars_desc',{})
import ck.kernel as ck
import time
import os
import shutil
# Get tmp address
ii={'action':'update',
'module_uoa':'tmp',
'data_uoa':'slambench-output',
'ignore_update':''}
rx=ck.access(ii)
if rx['return']>0:
ck.err(rx)
pp=rx['path']
ppt=os.path.join(pp, 'tmp')
if not os.path.isdir(ppt): os.makedirs(ppt)
print ('Starting continuous convert of images - Press Ctrl-C to stop')
files=["tmp/tmp-raw-input-rgb.rgb", "tmp/tmp-raw-depthrender.rgb", "tmp/tmp-raw-trackrender.rgb", "tmp/tmp-raw-volumerender.rgb"]
tfiles=[None,None,None,None]
width=[640,320,320,320]
height=[480,240,240,240]
types=[None,'CMYK','CMYK','CMYK']
output=['jpeg','jpeg','jpeg','jpeg']
js='tmp/tmp-ck-timer.json'
jsh=os.path.join(pp, 'tmp/tmp-ck-timer.json.html')
tjs=None
def do(i, arg):
if arg.fp is not None:
fin = arg.fp
if (os.path.isfile(fin)):
print ("File loading %s " %(fin))
#LOAD FILE and TRIPLES
else:
print("File %s not found " %(fin))
#Load Models Matrixes
alex_net = loadMatrixFromJson(alex_net_json)
google_net = loadMatrixFromJson(google_net_json)
squeeze11_net = loadMatrixFromJson(squeeze11_net_json)
dataset =[]
dataset.append({'Model_name': 'AlexNet', 'Batch_size' : 1, 'matrix' : alex_net})
dataset.append({'Model_name': 'GoogleNet', 'Batch_size' : 1, 'matrix' : google_net})
dataset.append({'Model_name': 'SqueezeNet1.1', 'Batch_size' : 1, 'matrix' : squeeze11_net})
if VERBOSE or DEBUG:
print('[Experiment] %s' % title)
print('[Preparing pipeline] Clock resolution: %d' % clock_resolution)
#print('[Preparing pipeline] Matrix sizes: m=%s, k=%s, n=%s: ' % (size_m, size_k, size_n))
print('[Preparing pipeline] Precision: %d' % precision)
print('[Preparing pipeline] Run for configuration: %d' % run)
print('[Preparing pipeline] More parms... ')
#ntrip = len(size_m)
#print ('[Experiment] Number of triple(s) %s' % (ntrip))
#size_tag = ''
#for tp in range (0, ntrip):
# if (tp == ntrip-1):
# size_tag += str((int(size_m[tp])*int(size_n[tp])*int(size_k[tp])))
# else:
# size_tag += str((int(size_m[tp])*int(size_n[tp])*int(size_k[tp])))+','
# Detect basic platform info.
ii={'action':'detect',
'module_uoa':'platform',
'con':'con'}
r=ck.access(ii)
if DEBUG: print("%s %s" %(DEBUG_STR, r))
if r['return']>0: return r
# Host and target OS params.
hos=r['host_os_uoa']
hosd=r['host_os_dict']
tos=r['os_uoa']
tosd=r['os_dict']
tdid=r['device_id']
if DEBUG: print("%s %s %s" %(DEBUG_STR, hos, hosd))
if DEBUG: print("%s %s %s %s" %( DEBUG_STR, tos, tosd, tdid))
# Load CLBLAST program meta and desc to check deps.
ii={'action':'load',
'module_uoa':'program',
'data_uoa':'clblast-tune'}
rx=ck.access(ii)
if DEBUG: print("%s %s " %(DEBUG_STR, rx))
if rx['return']>0: return rx
meta= rx['dict']
# Get compile-time and run-time deps.
cdeps=meta.get('compile_deps',{})
rdeps=meta.get('run_deps',{})
# Merge rdeps with cdeps for setting up the pipeline (which uses
# common deps), but tag them as "for_run_time".
for k in rdeps:
cdeps[k]=rdeps[k]
cdeps[k]['for_run_time']='yes'
# CLblast libs.
depl=copy.deepcopy(cdeps['lib-clblast'])
#ON LOCAL MACHINE
if ((arg.tos is not None) and (arg.did is not None) ):
tos=arg.tos
tdid=arg.did
ii={'action':'resolve',
'module_uoa':'env',
'host_os':hos,
'target_os':tos,
'device_id':tdid,
'out':'con',
'deps':{'lib-clblast':copy.deepcopy(depl)}
}
r=ck.access(ii)
if r['return']>0: return r
udepl=r['deps']['lib-clblast'].get('choices',[])
if len(udepl)==0: return {'return':1, 'error':'no installed CLBlast libs'}
cdeps['lib-clblast']['uoa']=udepl[0]
#prepare pipeline
ii={'action':'pipeline',
'module_uoa':'program',
'data_uoa':'clblast-tune',
'prepare':'yes',
'dependencies': cdeps,
'no_compiler_description':'yes',
'cmd_key':kernel[0],
"target_os":tos,
"device_id":tdid,
"out":'con',
"no_state_check":"yes",
'flags':'-O3',
'cpu_freq' : 'max',
'gpu_freq' : 'max',
}
r=ck.access(ii)
if r['return']>0: return r
fail=r.get('fail','')
if fail=='yes': return {'return':10, 'error':'pipeline failed ('+r.get('fail_reason','')+')'}
ready=r.get('ready','')
if ready!='yes': return {'return':11, 'error':'pipeline not ready'}
state=r['state']
tmp_dir=state['tmp_dir']
xcdeps=r.get('dependencies',{})
# Clean pipeline.
if 'ready' in r: del(r['ready'])
if 'fail' in r: del(r['fail'])
if 'return' in r: del(r['return'])
pipeline=copy.deepcopy(r)
print udepl
# For each Clblast lib ***********************************
for lib_uoa in udepl:
print lib_uoa
ii={'action' : 'load',
'module_uoa' : 'env',
'data_uoa' : lib_uoa}
r=ck.access(ii)
if r['return']>0 : return r
lib_name=r['data_name']
#lib_tags=re.match('BVLC Caffe framework \((?P<tags>.*)\)', lib_name)
#lib_tags=r['dict']['tags']
lib_tags=r['dict']['customize']['used_package_uoa'].split('-')
tags=''
skip_tags = ['lib','master', 'universal']
for t in lib_tags:
if t not in skip_tags:
tags+= t + '-'
# Add the extra_tags (if any)
lib_tags=r['dict']['setup']['version']
tags += lib_tags
# For each model in dataset
for model in dataset:
record_repo='local'
record_uoa='explore-matrix-size-'+tags+'-' + model['Model_name']
ck.out('---------------------------------------------------------------------------------------')
ck.out('Experiment - %s:%s' % (record_repo, record_uoa))
cpipeline=copy.deepcopy(pipeline)
ii={
'action':'autotune',
'module_uoa':'pipeline',
'data_uoa':'program',
'choices_order':[
[
'##env#CK_CLBLAST_MSIZE'
],
[
'##env#CK_CLBLAST_NSIZE',
],
[
'##env#CK_CLBLAST_KSIZE'
]
],
'choices_selection':[
{"type":"loop-with-next", "choice":model['matrix']['M'], "default":"256"},
{"type":"loop-with-next", "choice":model['matrix']['N'], "default":"256"},
{"type":"loop-with-next", "choice":model['matrix']['K'], "default":"256"}
],
'features_keys_to_process':['##choices#*'],
'iterations':-1,
'repetitions':3,
'record':'yes',
'record_failed':'yes',
'record_params':{
'search_point_by_features':'yes'
},
'record_repo':record_repo,
'record_uoa':record_uoa,
'tags':['explore-clblast-matrix-size-client', tags, model['Model_name']],
'pipeline': cpipeline,
'out':'con'
}
r=ck.access(ii)
if DEBUG > 0: print("%s %s" %(DEBUG_STR, r))
if r['return']>0: return r
fail=r.get('fail','')
if fail=='yes':
return {'return':10, 'error':'pipeline failed ('+r.get('fail_reason','')+')'}
return {'return':0}
import ck.kernel as ck
r = ck.access('list #milepost* out=none')
if r['return'] > 0:
print 'Error: ' + r['error']
exit(1)
lst = r['lst']
r = ck.load_json_file({'json_file': 'convert2.json'})
if r['return'] > 0:
print 'Error: ' + r['error']
exit(1)
d1 = r['dict']
dtags = {}
for q in lst:
m = q['module_uoa']
d = q['data_uoa']
print('***********************************')
print(d)
r = ck.access({'action': 'load', 'module_uoa': m, 'data_uoa': d})
if r['return'] > 0:
print 'Error: ' + r['error']
exit(1)
dd = r['dict']
dm = r['info']
def login(i):
"""
Input: {
(username) [str]
(api_key) [str]
(server_url) [str]
(server_user) [str]
(server_pass) [str]
(server_skip_validation) [str]
}
Output: {
return [int] - return code = 0 if success or >0 if error
(error) [str] - error string if return>0
}
"""
# Get current configuration
cfg={}
ii={'action':'load',
'repo_uoa':config.CK_CFG_REPO_UOA,
'module_uoa':config.CK_CFG_MODULE_UID,
'data_uoa':config.CK_CFG_DATA_UOA}
r=ck.access(ii)
if (r['return']>0 and r['return']!=16): ck.err(r)
# If not found, setup client
if r['return']==16:
setup(i)
# Load again
cfg={}
# ii={'action':'load',
# 'repo_uoa':config.CK_CFG_REPO_UOA,
# 'module_uoa':config.CK_CFG_MODULE_UID,
# 'data_uoa':config.CK_CFG_DATA_UOA}
#
# r=ck.access(ii)
# if r['return']>0: ck.err(r)
r=config.load({})
if r['return']>0: return r
cfg=r.get('dict',{})
# Update cfg
for k in ['username', 'api_key', 'server_url', 'server_user', 'server_pass', 'server_skip_validation']:
v=i.get(k,'')
if v==None: v=''
if v!='': cfg[k]=v
# Sending request to test connection
r=comm.send({'config':cfg,
'action':'login'
})
if r['return']>0: ck.err(r)
# Success
ck.out('cK login tested successfully!')
return 0
# ### Path to the cache
# In[ ]:
cache_name = 'mlperf-inference-v0.5-results.zip'
cache_compression = 'zip'
cache_protocol = 2 # Supported since Python 2.3
import ck.kernel as ck
repo_uoa = 'ck-mlperf'
module_uoa = 'module'
data_uoa = 'mlperf.inference'
r = ck.access({'action':'find', 'repo_uoa':repo_uoa, 'module_uoa':module_uoa, 'data_uoa':data_uoa})
if r['return']>0:
print('Error: %s' % r['error'])
exit(1)
cache_path = os.path.join(r['path'], cache_name)
cache_path
# ### Divisions
# In[ ]:
divisions = [ 'closed', 'open' ]
def main(arg):
# CUSTOMIZABLE VARIABLES!!!!
module_uoa = 'experiment'
repo_uoa = 'explore-matrix-size-xgemm-fp32-firefly-rk3399'
tags = 'explore-clblast-matrix-size'
output_filename = 'tmp-ck-clblast-tune.json'
weights_filename = 'NULL'
WEIGHTS = 0
if arg.fp is not None:
weights_filename = arg.fp
if (os.path.isfile(weights_filename)):
print("{RANKING ERROR %s not found. USE WEIGHTS=0}" %
(weights_filename))
WEIGHTS = 1
else:
print("[RANKING ERROR] %s not found. USE WEIGHTS=0" %
(weights_filename))
### END CUST
dlist = []
dblist = []
r = ck.access({
'action': 'search',
'repo_uoa': repo_uoa,
'module_uoa': module_uoa,
'tags': tags
})
if r['return'] > 0:
print("Error: %s" % r['error'])
experiments = r['lst']
if len(experiments) == 0:
print("No experiments found in repo %s with tags %s" %
(repo_uoa, tags))
exit(1)
for exp in experiments:
# print exp
data_uoa = exp['data_uoa']
r = ck.access({
'action': 'list_points',
'repo_uoa': repo_uoa,
'module_uoa': module_uoa,
'data_uoa': data_uoa
})
if r['return'] > 0:
print("Error: %s" % r['error'])
exit(1)
tags = r['dict']['tags']
# print tags
npoint = len(r['points'])
print("[RANKING] Number of matrices: %s" % (npoint))
# print npoint
for point in r['points']:
point_file = os.path.join(r['path'], 'ckp-%s.0001.json' % point)
d = get_data(point_file)
dlist.append(d)
# LOAD WEIGHTS
w = []
# WEIGHTS = os.path.isfile(weights_filename) and WEIGHTS
if (WEIGHTS == 0):
for i in range(0, npoint):
w.append(1)
else:
print("Loading weights %s" % (weights_filename))
wdict = json.loads(open(weights_filename).read())
for i in wdict:
print i.get('Execution time (%)')
w.append(float(i.get('Execution time (%)')))
output = get_data_header(point_file)
# FIND THE BEST
#configurations = dlist[0]
#best = sys.float_info.max
#configuration_best_index = len(configurations)
#configuration_count = 0
#final_score = []
#for configuration in configurations:
# res = find_it(configuration['parameters'],dlist)
# tmp_time = 0
# for ctime in res["time"]:
# tmp_time +=ctime
# if tmp_time < best:
# best= tmp_time
# configuration_best_index = configuration_count
# res["total_time"] = tmp_time
# final_score.append(res)
# configuration_count +=1
#mbest = final_score[configuration_best_index]
mbest = find_best(dlist, w)
### PREPARE OUTPUT
del mbest['time']
del mbest['total_time']
mbest['GFLOPS'] = 0.0
mbest['kernel'] = output.get('kernel')
output['data'] = 'na'
output['db'] = 'na'
output['statistics'] = {
'default_family': {},
'default_configuration': {},
'best_configuration': mbest
}
#print (json.dumps(output, indent=2))
rr = ck.save_json_to_file({'json_file': output_filename, 'dict': output})
def publish(i):
"""
Input: {
cid [str] - CK CID of format (repo UOA:)module UOA:data UOA
(can use wildcards)
(tags) [str] - search multiple CK components by these tags separated by comma
(version) [str] - assign version
}
Output: {
return [int] - return code = 0 if success or >0 if error
(error) [str] - error string if return>0
}
"""
# Get current directory (since will be changing it to get info about Git repo)
cur_dir = os.getcwd()
# Get current configuration
r = config.load({})
if r['return'] > 0: return r
cfg = r['dict']
# Check commands
# Username ##########################################################
username = cfg.get('username', '')
if i.get('username') != None: username = i['username']
if username == '' or username == None:
return {'return': 1, 'error': 'Username is not defined'}
cfg['username'] = username
# API key ###########################################################
api_key = cfg.get('api_key', '')
if i.get('api_key') != None: api_key = i['api_key']
if api_key == '' or api_key == None:
return {'return': 1, 'error': 'API key is not defined'}
cfg['api_key'] = api_key
# CID ###########################################################
cid = i.get('cid')
if cid == '' or cid == None:
return {'return': 1, 'error': 'CK entry (CID) is not defined'}
tags = i.get('tags', '')
# Check if no module and use "cr-solution" by default
if cid.find(':') < 0:
cid = 'cr-solution:' + cid
# Version ###########################################################
version = i.get('version')
if version == '' or version == None:
return {'return': 1, 'error': 'Version is not defined'}
# Extra info about authors
author = i.get('author', '')
if author == None: author = ''
author_codereef_id = i.get('author_codereef_id', '')
if author_codereef_id == None: author_codereef_id = ''
copyright = i.get('copyright', '')
if copyright == None: copyright = ''
license = i.get('license', '')
if license == None: license = ''
source = i.get('source', '')
if source == None: source = ''
quiet = i.get('quiet', False)
force = i.get('force', False)
# List CK components
r = ck.access({
'action': 'search',
'cid': cid,
'tags': tags,
'add_info': 'yes',
'add_meta': 'yes',
'common_func': 'yes'
})
if r['return'] > 0: return r
lst = r['lst']
llst = len(lst)
if llst == 0:
ck.out('No CK objects found')
num = 0
# Sort lst by modules and then data
lst1 = sorted(lst,
key=lambda x: (x.get('repo_uoa', ''), x.get(
'module_uoa', ''), x.get('data_uoa', '')))
for obj in lst1:
num += 1
# Basic info about CK object
repo_uoa = obj['repo_uoa']
repo_uid = obj['repo_uid']
module_uoa = obj['module_uoa']
module_uid = obj['module_uid']
data_uoa = obj['data_uoa']
data_uid = obj['data_uid']
# Print info
ck.out(
str(num) + ' out of ' + str(llst) + ') ' + repo_uoa + ':' +
module_uoa + ':' + data_uoa)
# Check name and date
data_name = obj.get('info', {}).get('data_name', '')
if data_name == data_uoa: data_name = ''
data_meta = obj['meta']
if data_name == '':
if data_meta.get('misc', {}).get('title', '') != '':
data_name = data_meta['misc']['title']
data_date = ''
if data_meta.get('misc', {}).get('date', '') != '':
data_date = data_meta['misc']['date']
source2 = data_meta.get('source', '')
if source2 == '': source2 = source
license2 = data_meta.get('license', '')
if license2 == '': license2 = license
copyright2 = data_meta.get('copyright', '')
if copyright2 == '': copyright2 = copyright
# Specialize per specific modules
not_digital_component = False
extra_dict = {}
extra_tags = {}
if module_uoa == 'module':
extra_dict['last_module_actions'] = []
actions = data_meta.get('actions', {})
for a in actions:
extra_dict['last_module_actions'].append(a + ' ' + data_uoa)
elif module_uoa == 'cr-lib':
not_digital_component = True
extra_tags = ['codereef-library']
if 'reproduced-papers' in data_meta.get('tags', []):
extra_tags.append('reproduced-papers')
data_meta2 = data_meta.get('meta', {})
if data_name == '':
data_name = data_meta2.get('title', '')
all_authors = data_meta2.get('authors', '')
if all_authors != '':
extra_dict['all_authors'] = []
for aa in all_authors.split(','):
if aa != '': aa = aa.strip()
if aa != '':
extra_dict['all_authors'].append(aa)
for k in [
'badge_acm_artifact_available',
'badge_acm_artifact_functional',
'badge_acm_artifact_reusable',
'badge_acm_results_replicated',
'badge_acm_results_reproduced'
]:
if data_meta2.get(k, '') == 'yes':
extra_tags.append(k)
elif module_uoa == 'cr-event' or module_uoa == 'repo':
not_digital_component = True
# Get info of the first creation
first_creation = obj['info'].get('control', {})
# Load info about repo
repo_dict = {}
if not force and repo_uoa == 'local' and module_uoa != 'repo': # Normally skip everything from local unless we publish repos themselves
ck.out(' SKIPPED')
continue
if module_uoa == 'repo':
if not force and data_uoa == 'local':
ck.out(' SKIPPED')
continue
repo_dict = obj['meta']
elif repo_uoa != 'default' and repo_uoa != 'local':
r = ck.access({
'action': 'load',
'repo_uoa': config.CK_CFG_REPO_UOA,
'module_uoa': config.CK_CFG_MODULE_REPO_UOA,
'data_uoa': repo_uid,
'common_func': 'yes'
})
if r['return'] > 0: return r
repo_dict = r['dict']
if 'path' in repo_dict:
del (repo_dict['path'])
# Generate temp file to pack
r = ck.gen_tmp_file({'prefix': 'cr-obj-', 'suffix': '.zip'})
if r['return'] > 0: return r
fn = r['file_name']
# Pack component
p = obj['path']
zip_method = zipfile.ZIP_DEFLATED
ii = {'path': p, 'all': 'yes'}
# Prune files for cr-solution
if module_uoa == 'cr-solution':
ii['ignore_names'] = ['CK', 'venv']
r = ck.list_all_files(ii)
if r['return'] > 0: return r
fl = r['list']
# Write archive
try:
f = open(fn, 'wb')
z = zipfile.ZipFile(f, 'w', zip_method)
for fx in fl:
add = True
for k in skip_words_in_files:
if k in fx:
add = False
break
if add:
p1 = os.path.join(p, fx)
z.write(p1, fx, zip_method)
z.close()
f.close()
except Exception as e:
return {
'return': 1,
'error': 'failed to prepare archive (' + format(e) + ')'
}
# Check size
statinfo = os.stat(fn)
pack_size = statinfo.st_size
# Check problems with repository or components
x = ''
if repo_dict.get('remote', '') == 'yes':
x += 'remote repo;'
if repo_dict.get('private', '') == 'yes':
x += 'private repo;'
if repo_dict.get('url', '') == '' and repo_uoa != 'default':
x += 'repo not shared;'
if pack_size > config.PACK_SIZE_WARNING:
x += 'pack size (' + str(pack_size) + ') > ' + str(
config.PACK_SIZE_WARNING) + ';'
skip_component = False
if not force and x != '':
if quiet:
skip_component = True
else:
r = ck.inp({
'text':
' This component has potential issues (' + x +
'). Skip processing (Y/n)? '
})
if r['return'] > 0: return r
s = r['string'].strip()
if s == '' or s == 'Y' or s == 'y':
skip_component = True
if skip_component:
ck.out(' SKIPPED (' + x + ')')
if os.path.isfile(fn):
os.remove(fn)
continue
# Convert to MIME to send over internet
r = ck.convert_file_to_upload_string({'filename': fn})
if r['return'] > 0: return r
pack64 = r['file_content_base64']
if os.path.isfile(fn):
os.remove(fn)
# Check workspaces
lworkspaces = []
workspaces = i.get('workspaces', '')
if workspaces != None:
lworkspaces = workspaces.strip().split(',')
# Get extra info about repo
os.chdir(p)
repo_info = {
'publish_repo_uoa': repo_uoa,
'publish_repo_uid': repo_uid
}
# Get current Git URL
r = ck.run_and_get_stdout(
{'cmd': ['git', 'config', '--get', 'remote.origin.url']})
if r['return'] == 0 and r['return_code'] == 0:
x = r['stdout'].strip()
if x != '': repo_info['remote_git_url'] = x
# Get current Git branch
r = ck.run_and_get_stdout(
{'cmd': ['git', 'rev-parse', '--abbrev-ref', 'HEAD']})
if r['return'] == 0 and r['return_code'] == 0:
x = r['stdout'].strip()
if x != '': repo_info['remote_git_branch'] = x
# Get current Git checkout
r = ck.run_and_get_stdout(
{'cmd': ['git', 'rev-parse', '--short', 'HEAD']})
if r['return'] == 0 and r['return_code'] == 0:
x = r['stdout'].strip()
if x != '': repo_info['remote_git_checkout'] = x
repo_info['dict'] = repo_dict
#TBD: owner, version, info about repo
# Sending request
r = comm.send({
'config': cfg,
'action': 'publish',
'ownership': {
'private': i.get('private', False),
'workspaces': lworkspaces
},
'dict': {
'publish_module_uoa': module_uoa,
'publish_module_uid': module_uid,
'publish_data_uoa': data_uoa,
'publish_data_uid': data_uid,
'publish_data_name': data_name,
'publish_data_date': data_date,
'publish_pack': pack64,
'publish_pack_size': pack_size,
'repo_info': repo_info,
'first_creation': first_creation,
'version': version,
'author': author,
'author_codereef_id': author_codereef_id,
'copyright': copyright2,
'license': license2,
'source': source2,
'not_digital_component': not_digital_component,
'extra_dict': extra_dict,
'extra_tags': extra_tags,
}
})
if r['return'] > 0:
ck.out(' WARNING: CodeReef API returned error: ' + r['error'])
else:
data_uid = r['data_uid']
ck.out(' CodeReef component ID: ' + data_uid)
os.chdir(cur_dir)
return {'return': 0}
#
# Collective Knowledge (CK)
#
# See CK LICENSE.txt for licensing details.
# See CK COPYRIGHT.txt for copyright details.
#
# Developer: Grigori Fursin
#
# Entry point for running CK tests for CI and coverage measurement tools
from __future__ import print_function
try:
import ck.kernel as ck
r = ck.access(['run', 'test'])
if 'return' not in r:
raise Exception('CK access function should always return key \'return\'!')
exit(int(r['return']))
except ImportError as e:
from sys import stderr
from subprocess import call
print('WARNING: CK kernel module for python is not installed & jupyter notebooks will not be supported', file=stderr)
retcode = call(['ck', 'run', 'test'])
exit(retcode)
# # Collective Knowledge script # # See CK LICENSE.txt for licensing details # See CK COPYRIGHT.txt for copyright details # # Developer: Grigori Fursin, [email protected], http://fursin.net, 2017 # max_dataset = 20 import ck.kernel as ck r = ck.access({ 'action': 'search', 'module_uoa': 'program', 'add_meta': 'yes', 'tags': 'crowd-tuning' }) if r['return'] > 0: ck.err(r) lst = r['lst'] r = ck.access({ 'action': 'search', 'module_uoa': 'program', 'add_meta': 'yes', 'tags': 'milepost' }) if r['return'] > 0: ck.err(r) for q in r['lst']: duoa = q['data_uoa'] found = False
def do(i, arg):
fp = arg.fp
if fp is not None:
fromfile = os.path.isfile(fp)
if (fromfile):
print("Loading triples %s" %(fp))
triples = json.loads(open(fp).read())
del size_m[:]
del size_n[:]
del size_k[:]
for i in triples:
size_m.append(str(i.get('bSizeM')))
size_n.append(str(i.get('bSizeN')))
size_k.append(str(i.get('bSizeK')))
if VERBOSE or DEBUG:
print('[Experiment] %s' % title)
print('[Preparing pipeline] Clock resolution: %d' % clock_resolution)
print('[Preparing pipeline] Matrix sizes: m=%s, k=%s, n=%s: ' % (size_m, size_k, size_n))
print('[Preparing pipeline] Precision: %d' % precision)
print('[Preparing pipeline] Run for configuration: %d' % run)
print('[Preparing pipeline] More parms... ')
ntrip = len(size_m)
print ('[Experiment] Number of triple(s) %s' % (ntrip))
size_tag = ''
for tp in range (0, ntrip):
if (tp == ntrip-1):
size_tag += str((int(size_m[tp])*int(size_n[tp])*int(size_k[tp])))
else:
size_tag += str((int(size_m[tp])*int(size_n[tp])*int(size_k[tp])))+','
# Detect basic platform info.
ii={'action':'detect',
'module_uoa':'platform',
'con':'con'}
r=ck.access(ii)
if DEBUG: print("%s %s" %(DEBUG_STR, r))
if r['return']>0: return r
# Host and target OS params.
hos=r['host_os_uoa']
hosd=r['host_os_dict']
tos=r['os_uoa']
tosd=r['os_dict']
tdid=r['device_id']
if DEBUG: print("%s %s %s" %(DEBUG_STR, hos, hosd))
if DEBUG: print("%s %s %s %s" %( DEBUG_STR, tos, tosd, tdid))
#PROGRAM TAGS
program_tags='sgemm,opencl'
# Search program on tags
ii={'action':'search',
'module_uoa':'program',
'tags':program_tags
}
rx=ck.access(ii)
if rx['return']>0: return rx
programs_list=rx['lst']
for p in programs_list:
# Load CLBLAST program meta and desc to check deps.
ii={'action':'load',
'module_uoa':'program',
'data_uoa': p['data_uoa']}
rx=ck.access(ii)
if DEBUG: print("%s %s " %(DEBUG_STR, rx))
if rx['return']>0: return rx
meta= rx['dict']
# Get compile-time and run-time deps.
cdeps=meta.get('compile_deps',{})
rdeps=meta.get('run_deps',{})
print(cdeps)
# Merge rdeps with cdeps for setting up the pipeline (which uses
# common deps), but tag them as "for_run_time".
for k in rdeps:
cdeps[k]=rdeps[k]
cdeps[k]['for_run_time']='yes'
# CLblast libs.
cmd=""
if 'clblast' in p['data_uoa'] :
cmd=kernel[0]
print cmd
ii={'action':'pipeline',
'module_uoa':'program',
'data_uoa': p['data_uoa'],
'prepare':'yes',
'dependencies': cdeps,
'no_compiler_description':'yes',
'cmd_key':cmd,
"target_os":tos,
"device_id":tdid,
'dvdt_prof':'yes',
"out":'con',
"no_state_check":"yes",
'flags':'-O3',
}
r=ck.access(ii)
if r['return']>0: return r
fail=r.get('fail','')
if fail=='yes': return {'return':10, 'error':'pipeline failed ('+r.get('fail_reason','')+')'}
ready=r.get('ready','')
if ready!='yes': return {'return':11, 'error':'pipeline not ready'}
state=r['state']
tmp_dir=state['tmp_dir']
xcdeps=r.get('dependencies',{})
# Clean pipeline.
if 'ready' in r: del(r['ready'])
if 'fail' in r: del(r['fail'])
if 'return' in r: del(r['return'])
pipeline=copy.deepcopy(r)
record_repo='local'
record_uoa='explore-matrix-size-gemm-libs-'+p['data_uoa']
ck.out('---------------------------------------------------------------------------------------')
ck.out('Experiment - %s:%s' % (record_repo, record_uoa))
cpipeline=copy.deepcopy(pipeline)
ii={
'action':'autotune',
'module_uoa':'pipeline',
'data_uoa':'program',
'choices_order':[
[
'##env#CK_CLBLAST_MSIZE'
],
[
'##env#CK_CLBLAST_NSIZE',
],
[
'##env#CK_CLBLAST_KSIZE'
]
],
'choices_selection':[
{"type":"loop-with-next", "choice":size_m, "default":"256"},
{"type":"loop-with-next", "choice":size_n, "default":"256"},
{"type":"loop-with-next", "choice":size_k, "default":"256"}
],
'features_keys_to_process':['##choices#*'],
'iterations':-1,
'repetitions':3,
'record':'yes',
'record_failed':'yes',
'record_params':{
'search_point_by_features':'yes'
},
'record_repo':record_repo,
'record_uoa':record_uoa,
'tags':['dvdt-prof', 'explore-matrix-size-libs-sgemm', p['data_uoa']],
'pipeline': cpipeline,
'out':'con'
}
r=ck.access(ii)
if DEBUG > 0: print("%s %s" %(DEBUG_STR, r))
if r['return']>0: return r
fail=r.get('fail','')
if fail=='yes':
return {'return':10, 'error':'pipeline failed ('+r.get('fail_reason','')+')'}
return {'return':0}
def do(i, arg):
if arg.fp is not None:
fin = arg.fp
if (os.path.isfile(fin)):
print ("File loading %s " %(fin))
#LOAD FILE and TRIPLES
else:
print("File %s not found " %(fin))
if VERBOSE or DEBUG:
print('[Experiment] %s' % title)
print('[Preparing pipeline] Clock resolution: %d' % clock_resolution)
print('[Preparing pipeline] Matrix sizes: m=%s, k=%s, n=%s: ' % (size_m, size_k, size_n))
print('[Preparing pipeline] Precision: %d' % precision)
print('[Preparing pipeline] Run for configuration: %d' % run)
print('[Preparing pipeline] More parms... ')
ntrip = len(size_m)
print ('[Experiment] Number of triple(s) %s' % (ntrip))
size_tag = ''
for tp in range (0, ntrip):
if (tp == ntrip-1):
size_tag += str((int(size_m[tp])*int(size_n[tp])*int(size_k[tp])))
else:
size_tag += str((int(size_m[tp])*int(size_n[tp])*int(size_k[tp])))+','
# Detect basic platform info.
ii={'action':'detect',
'module_uoa':'platform',
'con':'con'}
r=ck.access(ii)
if DEBUG: print("%s %s" %(DEBUG_STR, r))
if r['return']>0: return r
# Host and target OS params.
hos=r['host_os_uoa']
hosd=r['host_os_dict']
tos=r['os_uoa']
tosd=r['os_dict']
tdid=r['device_id']
if DEBUG: print("%s %s %s" %(DEBUG_STR, hos, hosd))
if DEBUG: print("%s %s %s %s" %( DEBUG_STR, tos, tosd, tdid))
# Load CLBLAST program meta and desc to check deps.
ii={'action':'load',
'module_uoa':'program',
'data_uoa':'clblast-tune'}
rx=ck.access(ii)
if DEBUG: print("%s %s " %(DEBUG_STR, rx))
if rx['return']>0: return rx
meta= rx['dict']
# Get compile-time and run-time deps.
cdeps=meta.get('compile_deps',{})
rdeps=meta.get('run_deps',{})
# Merge rdeps with cdeps for setting up the pipeline (which uses
# common deps), but tag them as "for_run_time".
for k in rdeps:
cdeps[k]=rdeps[k]
cdeps[k]['for_run_time']='yes'
# CLblast libs.
depl=copy.deepcopy(cdeps['lib-clblast'])
#ON LOCAL MACHINE
if ((arg.tos is not None) and (arg.did is not None) ):
tos=arg.tos
tdid=arg.did
ii={'action':'resolve',
'module_uoa':'env',
'host_os':hos,
'target_os':tos,
'device_id':tdid,
'out':'con',
'deps':{'lib-clblast':copy.deepcopy(depl)}
}
r=ck.access(ii)
if r['return']>0: return r
udepl=r['deps']['lib-clblast'].get('choices',[])
if len(udepl)==0: return {'return':1, 'error':'no installed CLBlast libs'}
#prepare pipeline
ii={'action':'pipeline',
'module_uoa':'program',
'data_uoa':'clblast-tune',
'prepare':'yes',
'dependencies': cdeps,
'no_compiler_description':'yes',
'cmd_key':kernel[0],
"target_os":tos,
"device_id":tdid,
"out":'con',
"no_state_check":"yes",
'flags':'-O3',
}
r=ck.access(ii)
if r['return']>0: return r
fail=r.get('fail','')
if fail=='yes': return {'return':10, 'error':'pipeline failed ('+r.get('fail_reason','')+')'}
ready=r.get('ready','')
if ready!='yes': return {'return':11, 'error':'pipeline not ready'}
state=r['state']
tmp_dir=state['tmp_dir']
xcdeps=r.get('dependencies',{})
# Clean pipeline.
if 'ready' in r: del(r['ready'])
if 'fail' in r: del(r['fail'])
if 'return' in r: del(r['return'])
pipeline=copy.deepcopy(r)
record_repo='local'
record_uoa='explore-matrix-size-'+kernel[0]
ck.out('---------------------------------------------------------------------------------------')
ck.out('Experiment - %s:%s' % (record_repo, record_uoa))
cpipeline=copy.deepcopy(pipeline)
ii={
'action':'autotune',
'module_uoa':'pipeline',
'data_uoa':'program',
'choices_order':[
[
'##env#CK_CLBLAST_MSIZE'
],
[
'##env#CK_CLBLAST_NSIZE',
],
[
'##env#CK_CLBLAST_KSIZE'
]
],
'choices_selection':[
{"type":"loop-with-next", "choice":size_m, "default":"256"},
{"type":"loop-with-next", "choice":size_n, "default":"256"},
{"type":"loop-with-next", "choice":size_k, "default":"256"}
],
'features_keys_to_process':['##choices#*'],
'iterations':-1,
'repetitions':3,
'record':'yes',
'record_failed':'yes',
'record_params':{
'search_point_by_features':'yes'
},
'record_repo':record_repo,
'record_uoa':record_uoa,
'tags':['explore-clblast-matrix-size-client', kernel[0], size_tag],
'pipeline': cpipeline,
'out':'con'
}
r=ck.access(ii)
if DEBUG > 0: print("%s %s" %(DEBUG_STR, r))
if r['return']>0: return r
fail=r.get('fail','')
if fail=='yes':
return {'return':10, 'error':'pipeline failed ('+r.get('fail_reason','')+')'}
return {'return':0}
#
# Collective Knowledge (CK)
#
# See CK LICENSE.txt for licensing details.
# See CK COPYRIGHT.txt for copyright details.
#
# Developer: Grigori Fursin
#
# CK dummy
import sys
import ck.kernel as ck
r=ck.access(sys.argv[1:])
if 'return' not in r:
raise Exception('CK access function should always return key \'return\'!')
exit(int(r['return']))
import ck.kernel as ck
r = ck.access('list compiler:* out=none')
if r['return'] > 0:
print 'Error: ' + r['error']
exit(1)
for q in r['lst']:
m = q['module_uoa']
d = q['data_uoa']
ruoa = q['repo_uoa']
print d
r = ck.access({'action': 'load', 'module_uoa': m, 'data_uoa': d})
if r['return'] > 0:
print 'Error: ' + r['error']
exit(1)
dd = r['dict']
dm = r['info']
dx = r.get('desc', {})
zz = dd.get('all_compiler_flags_desc', {})
if len(zz) > 0:
dx['all_compiler_flags_desc'] = zz
if 'all_compiler_flags_desc' in dd: del (dd['all_compiler_flags_desc'])
rvd = dd.get('run_vars_desc', {})
if len(rvd) > 0:
dx['run_vars_desc'] = rvd
def do(i):
# Detect basic platform info.
ii = {'action': 'detect', 'module_uoa': 'platform', 'out': 'out'}
r = ck.access(ii)
if r['return'] > 0: return r
# Host and target OS params.
hos = r['host_os_uoa']
hosd = r['host_os_dict']
tos = r['os_uoa']
tosd = r['os_dict']
tdid = r['device_id']
# Fix cmd key here since it may be used to get extra run-time deps.
cmd_key = 'default'
# Load TensorRT-time program meta and desc to check deps.
ii = {
'action': 'load',
'module_uoa': 'program',
'data_uoa': 'tensorrt-time'
}
rx = ck.access(ii)
if rx['return'] > 0: return rx
mm = rx['dict']
# Get compile-time and run-time deps.
cdeps = mm.get('compile_deps', {})
rdeps = mm.get('run_deps', {})
# Merge rdeps with cdeps for setting up the pipeline (which uses
# common deps), but tag them as "for_run_time".
for k in rdeps:
cdeps[k] = rdeps[k]
cdeps[k]['for_run_time'] = 'yes'
# TensorRT engines.
depl = copy.deepcopy(cdeps['lib-tensorrt'])
ii = {
'action': 'resolve',
'module_uoa': 'env',
'host_os': hos,
'target_os': tos,
'device_id': tdid,
'deps': {
'lib-tensorrt': copy.deepcopy(depl)
}
}
r = ck.access(ii)
if r['return'] > 0: return r
udepl = r['deps']['lib-tensorrt'].get(
'choices', []) # All UOAs of env for TensorRT engines.
if len(udepl) == 0:
return {'return': 1, 'error': 'no registered TensorRT engines'}
# Caffe models.
depm = copy.deepcopy(rdeps['caffemodel'])
ii = {
'action': 'resolve',
'module_uoa': 'env',
'host_os': hos,
'target_os': tos,
'device_id': tdid,
'deps': {
'caffemodel': copy.deepcopy(depm)
}
}
r = ck.access(ii)
if r['return'] > 0: return r
udepm = r['deps']['caffemodel'].get(
'choices', []) # All UOAs of env for Caffe models.
if len(udepm) == 0:
return {'return': 1, 'error': 'no registered Caffe models'}
# Prepare pipeline.
cdeps['lib-tensorrt']['uoa'] = udepl[0]
cdeps['caffemodel']['uoa'] = udepm[0]
ii = {
'action': 'pipeline',
'prepare': 'yes',
'repo_uoa': 'ck-tensorrt',
'module_uoa': 'program',
'data_uoa': 'tensorrt-time',
'cmd_key': cmd_key,
'dependencies': cdeps,
'no_compiler_description': 'yes',
'compile_only_once': 'yes',
'cpu_freq': 'max',
'gpu_freq': 'max',
'flags': '-O3',
'speed': 'no',
'energy': 'no',
'no_state_check': 'yes',
'skip_calibration': 'yes',
'skip_print_timers': 'yes',
'out': 'con',
}
r = ck.access(ii)
if r['return'] > 0: return r
fail = r.get('fail', '')
if fail == 'yes':
return {
'return': 10,
'error': 'pipeline failed (' + r.get('fail_reason', '') + ')'
}
ready = r.get('ready', '')
if ready != 'yes':
return {'return': 11, 'error': 'pipeline not ready'}
state = r['state']
tmp_dir = state['tmp_dir']
# Remember resolved deps for this benchmarking session.
xcdeps = r.get('dependencies', {})
# Clean pipeline.
if 'ready' in r: del (r['ready'])
if 'fail' in r: del (r['fail'])
if 'return' in r: del (r['return'])
pipeline = copy.deepcopy(r)
# For each TensorRT engine.
for lib_uoa in udepl:
# Load TensorRT engine.
ii = {'action': 'load', 'module_uoa': 'env', 'data_uoa': lib_uoa}
r = ck.access(ii)
if r['return'] > 0: return r
# Get the lib name e.g. 'tensorrt-3.0.4'.
lib_version = r['dict']['customize']['version']
lib_name = 'tensorrt-%s' % lib_version
lib_tags = lib_name
# Skip some libs with "in [..]" or "not in [..]".
if lib_name in []: continue
# For each Caffe model.
for model_uoa in udepm:
# Load Caffe model.
ii = {'action': 'load', 'module_uoa': 'env', 'data_uoa': model_uoa}
r = ck.access(ii)
if r['return'] > 0: return r
# Get the tags from e.g. 'Caffe model (net and weights) (deepscale, squeezenet, 1.1)'
model_name = r['data_name']
model_tags = re.match(
'Caffe model \(net and weights\) \((?P<tags>.*)\)', model_name)
if model_tags:
model_tags = model_tags.group('tags').replace(' ', '').replace(
',', '-')
else:
model_tags = ''
for tag in r['dict']['tags']:
if model_tags != '': model_tags += '-'
model_tags += tag
# Skip some models with "in [..]" or "not in [..]".
if model_tags not in [
'bvlc-alexnet', 'bvlc-googlenet',
'deepscale-squeezenet-1.1'
]:
continue
record_repo = 'local'
record_uoa = model_tags + '-' + lib_tags
# Prepare pipeline.
ck.out(
'---------------------------------------------------------------------------------------'
)
ck.out('%s - %s' % (lib_name, lib_uoa))
ck.out('%s - %s' % (model_name, model_uoa))
ck.out('Experiment - %s:%s' % (record_repo, record_uoa))
# Prepare autotuning input.
cpipeline = copy.deepcopy(pipeline)
# Reset deps and change UOA.
new_deps = {
'lib-tensorrt': copy.deepcopy(depl),
'caffemodel': copy.deepcopy(depm)
}
new_deps['lib-tensorrt']['uoa'] = lib_uoa
new_deps['caffemodel']['uoa'] = model_uoa
jj = {
'action': 'resolve',
'module_uoa': 'env',
'host_os': hos,
'target_os': tos,
'device_id': tdid,
'deps': new_deps
}
r = ck.access(jj)
if r['return'] > 0: return r
cpipeline['dependencies'].update(new_deps)
pipeline_name = '%s.json' % record_uoa
ii = {
'action':
'autotune',
'module_uoa':
'pipeline',
'data_uoa':
'program',
'choices_order': [['##choices#env#CK_TENSORRT_ENABLE_FP16'],
['##choices#env#CK_CAFFE_BATCH_SIZE']],
'choices_selection': [{
'type': 'loop',
'start': fp['start'],
'stop': fp['stop'],
'step': fp['step'],
'default': fp['default']
}, {
'type': 'loop',
'start': bs['start'],
'stop': bs['stop'],
'step': bs['step'],
'default': bs['default']
}],
'features_keys_to_process': [
'##choices#env#CK_TENSORRT_ENABLE_FP16',
'##choices#env#CK_CAFFE_BATCH_SIZE'
],
'iterations':
-1,
'repetitions':
num_repetitions,
'record':
'yes',
'record_failed':
'yes',
'record_params': {
'search_point_by_features': 'yes'
},
'record_repo':
record_repo,
'record_uoa':
record_uoa,
'tags': [
'explore-batch-size-libs-models', platform_tags,
model_tags, lib_tags
],
'pipeline':
cpipeline,
'out':
'con'
}
r = ck.access(ii)
if r['return'] > 0: return r
fail = r.get('fail', '')
if fail == 'yes':
return {
'return': 10,
'error':
'pipeline failed (' + r.get('fail_reason', '') + ')'
}
return {'return': 0}
def do(i, arg):
# Detect basic platform info.
ii = {'action': 'detect', 'module_uoa': 'platform', 'out': 'out'}
r = ck.access(ii)
if r['return'] > 0: return r
# Host and target OS params.
hos = r['host_os_uoa']
hosd = r['host_os_dict']
tos = r['os_uoa']
tosd = r['os_dict']
tdid = r['device_id']
# Program and command.
program = 'caffe-time-opencl'
cmd_key = 'default'
tp = 'opencl'
# Load Caffe program meta and desc to check deps.
ii = {'action': 'load', 'module_uoa': 'program', 'data_uoa': program}
rx = ck.access(ii)
if rx['return'] > 0: return rx
mm = rx['dict']
# Get compile-time and run-time deps.
cdeps = mm.get('compile_deps', {})
rdeps = mm.get('run_deps', {})
# Merge rdeps with cdeps for setting up the pipeline (which uses
# common deps), but tag them as "for_run_time".
for k in rdeps:
cdeps[k] = rdeps[k]
cdeps[k]['for_run_time'] = 'yes'
# Caffe libs.
depl = copy.deepcopy(cdeps['lib-caffe'])
if (arg.tos is not None) and (arg.did is not None):
tos = arg.tos
tdid = arg.did
ii = {
'action': 'resolve',
'module_uoa': 'env',
'host_os': hos,
'target_os': tos,
'device_id': tdid,
'out': 'con',
'deps': {
'lib-caffe': copy.deepcopy(depl)
}
}
r = ck.access(ii)
if r['return'] > 0: return r
udepl = r['deps']['lib-caffe'].get('choices',
[]) # All UOAs of env for Caffe libs.
if len(udepl) == 0:
return {'return': 1, 'error': 'no installed Caffe libs'}
# Caffe models.
depm = copy.deepcopy(cdeps['caffemodel'])
ii = {
'action': 'resolve',
'module_uoa': 'env',
'host_os': hos,
'target_os': tos,
'device_id': tdid,
'out': 'con',
'deps': {
'caffemodel': copy.deepcopy(depm)
}
}
r = ck.access(ii)
if r['return'] > 0: return r
udepm = r['deps']['caffemodel'].get(
'choices', []) # All UOAs of env for Caffe models.
if len(udepm) == 0:
return {'return': 1, 'error': 'no installed Caffe models'}
# Prepare pipeline.
cdeps['lib-caffe']['uoa'] = udepl[0]
cdeps['caffemodel']['uoa'] = udepm[0]
ii = {
'action': 'pipeline',
'prepare': 'yes',
'dependencies': cdeps,
'module_uoa': 'program',
'data_uoa': program,
'cmd_key': cmd_key,
'target_os': tos,
'device_id': tdid,
'no_state_check': 'yes',
'no_compiler_description': 'yes',
'skip_calibration': 'yes',
'env': {
'CK_CAFFE_SKIP_BACKWARD': 1,
'OPENBLAS_NUM_THREADS': 4
},
'cpu_freq': 'max',
'gpu_freq': 'max',
'flags': '-O3',
'speed': 'no',
'energy': 'no',
'skip_print_timers': 'yes',
'out': 'con'
}
r = ck.access(ii)
if r['return'] > 0: return r
fail = r.get('fail', '')
if fail == 'yes':
return {
'return': 10,
'error': 'pipeline failed (' + r.get('fail_reason', '') + ')'
}
ready = r.get('ready', '')
if ready != 'yes':
return {'return': 11, 'error': 'pipeline not ready'}
state = r['state']
tmp_dir = state['tmp_dir']
# Remember resolved deps for this benchmarking session.
xcdeps = r.get('dependencies', {})
# Clean pipeline.
if 'ready' in r: del (r['ready'])
if 'fail' in r: del (r['fail'])
if 'return' in r: del (r['return'])
pipeline = copy.deepcopy(r)
# For each Caffe lib.*******************************************************
for lib_uoa in udepl:
# Load Caffe lib.
ii = {'action': 'load', 'module_uoa': 'env', 'data_uoa': lib_uoa}
r = ck.access(ii)
if r['return'] > 0: return r
# Get the tags from e.g. 'BVLC Caffe framework (libdnn,viennacl)'
lib_name = r['data_name']
lib_tags = re.match('BVLC Caffe framework \((?P<tags>.*)\)', lib_name)
lib_tags = lib_tags.group('tags').replace(' ', '').replace(',', '-')
# Skip some libs with "in [..]" or "not in [..]".
if lib_tags in ['opencl-clblast-tune']: continue
# if lib_tags not in ['opencl-clblast']: continue
# if lib_tags not in ['opencl-clblast-tune', 'opencl-clblast']: continue
# if lib_tags not in ['opencl-libdnn-clblast-tune', 'opencl-libdnn-clblast']: continue
# if lib_tags not in ['opencl-libdnn-viennacl', 'opencl-viennacl']: continue
skip_compile = 'no'
# Use the 'time_cpu' command for the CPU only lib, 'time_gpu' for all the rest.
# if r['dict']['customize']['params']['cpu_only']==1:
# cmd_key='time_cpu'
# else:
# cmd_key='time_gpu'
# # FIXME: Customise cmd for NVIDIA's experimental fp16 branch.
# if lib_tags in [ 'nvidia-fp16-cuda', 'nvidia-fp16-cudnn' ]:
# cmd_key='time_gpu_fp16'
# For each Caffe model.*************************************************
for model_uoa in udepm:
# Load Caffe model.
ii = {'action': 'load', 'module_uoa': 'env', 'data_uoa': model_uoa}
r = ck.access(ii)
if r['return'] > 0: return r
# Get the tags from e.g. 'Caffe model (net and weights) (deepscale, squeezenet, 1.1)'
model_name = r['data_name']
model_tags = re.match(
'Caffe model \(net and weights\) \((?P<tags>.*)\)', model_name)
model_tags = model_tags.group('tags').replace(' ', '').replace(
',', '-')
# Skip some models with "in [..]" or "not in [..]".
if model_tags not in ['bvlc-alexnet']: continue
record_repo = 'local'
record_uoa = model_tags + '-' + lib_tags
# Prepare pipeline.
ck.out(
'---------------------------------------------------------------------------------------'
)
ck.out('%s - %s' % (lib_name, lib_uoa))
ck.out('%s - %s' % (model_name, model_uoa))
ck.out('Experiment - %s:%s' % (record_repo, record_uoa))
# Prepare autotuning input.
cpipeline = copy.deepcopy(pipeline)
# Reset deps and change UOA.
new_deps = {
'lib-caffe': copy.deepcopy(depl),
'caffemodel': copy.deepcopy(depm)
}
new_deps['lib-caffe']['uoa'] = lib_uoa
new_deps['caffemodel']['uoa'] = model_uoa
jj = {
'action': 'resolve',
'module_uoa': 'env',
'host_os': hos,
'target_os': tos,
'device_id': tdid,
'deps': new_deps
}
r = ck.access(jj)
if r['return'] > 0: return r
cpipeline['dependencies'].update(new_deps)
cpipeline['no_clean'] = skip_compile
cpipeline['no_compile'] = skip_compile
cpipeline['cmd_key'] = cmd_key
ii = {
'action':
'autotune',
'module_uoa':
'pipeline',
'data_uoa':
'program',
'choices_order': [['##choices#env#CK_CAFFE_BATCH_SIZE']],
'choices_selection': [{
'type': 'loop',
'start': bs['start'],
'stop': bs['stop'],
'step': bs['step'],
'default': bs['default']
}],
'features_keys_to_process': ['##choices#*'],
'iterations':
-1,
'repetitions':
num_repetitions,
'record':
'yes',
'record_failed':
'yes',
'record_params': {
'search_point_by_features': 'yes'
},
'record_repo':
record_repo,
'record_uoa':
record_uoa,
'tags': [
'explore-batch-size-libs-models', cmd_key, model_tags,
lib_tags, platform_tags
],
'pipeline':
cpipeline,
'out':
'con'
}
r = ck.access(ii)
if r['return'] > 0: return r
fail = r.get('fail', '')
if fail == 'yes':
return {
'return': 10,
'error':
'pipeline failed (' + r.get('fail_reason', '') + ')'
}
skip_compile = 'yes'
return {'return': 0}
def do(i, arg):
fp = arg.fp
if fp is not None:
fromfile = os.path.isfile(fp)
if (fromfile):
print("Loading triples %s" %(fp))
triples = json.loads(open(fp).read())
del size_m[:]
del size_n[:]
del size_k[:]
for i in triples:
size_m.append(str(i.get('bSizeM')))
size_n.append(str(i.get('bSizeN')))
size_k.append(str(i.get('bSizeK')))
if VERBOSE or DEBUG:
print('[Experiment] %s' % title)
print('[Preparing pipeline] Clock resolution: %d' % clock_resolution)
print('[Preparing pipeline] Matrix sizes: m=%s, k=%s, n=%s: ' % (size_m, size_k, size_n))
print('[Preparing pipeline] Precision: %d' % precision)
print('[Preparing pipeline] Run for configuration: %d' % run)
print('[Preparing pipeline] More parms... ')
ntrip = len(size_m)
print ('[Experiment] Number of triple(s) %s' % (ntrip))
size_tag = ''
for tp in range (0, ntrip):
if (tp == ntrip-1):
size_tag += str((int(size_m[tp])*int(size_n[tp])*int(size_k[tp])))
else:
size_tag += str((int(size_m[tp])*int(size_n[tp])*int(size_k[tp])))+','
inf_g_tag = 'INF_G:'
for inf in pso_inf_g:
size_tag += str(float(inf)) +','
# Detect basic platform info.
ii={'action':'detect',
'module_uoa':'platform',
'con':'con'}
r=ck.access(ii)
if DEBUG: print("%s %s" %(DEBUG_STR, r))
if r['return']>0: return r
# Host and target OS params.
hos=r['host_os_uoa']
hosd=r['host_os_dict']
tos=r['os_uoa']
tosd=r['os_dict']
tdid=r['device_id']
if DEBUG: print("%s %s %s" %(DEBUG_STR, hos, hosd))
if DEBUG: print("%s %s %s %s" %( DEBUG_STR, tos, tosd, tdid))
# Load CLBLAST program meta and desc to check deps.
ii={'action':'load',
'module_uoa':'program',
'data_uoa':'clblast-tune'}
rx=ck.access(ii)
if DEBUG: print("%s %s " %(DEBUG_STR, rx))
if rx['return']>0: return rx
meta= rx['dict']
# Get compile-time and run-time deps.
cdeps=meta.get('compile_deps',{})
rdeps=meta.get('run_deps',{})
# Merge rdeps with cdeps for setting up the pipeline (which uses
# common deps), but tag them as "for_run_time".
for k in rdeps:
cdeps[k]=rdeps[k]
cdeps[k]['for_run_time']='yes'
# CLblast libs.
depl=copy.deepcopy(cdeps['lib-clblast'])
#ON LOCAL MACHINE
if ((arg.tos is not None) and (arg.did is not None) ):
tos=arg.tos
tdid=arg.did
ii={'action':'resolve',
'module_uoa':'env',
'host_os':hos,
'target_os':tos,
'device_id':tdid,
'out':'con',
'deps':{'lib-clblast':copy.deepcopy(depl)}
}
r=ck.access(ii)
if r['return']>0: return r
udepl=r['deps']['lib-clblast'].get('choices',[])
if len(udepl)==0: return {'return':1, 'error':'no installed CLBlast libs'}
for curr_pso_inf_g in pso_inf_g:
for curr_pso_inf_l in pso_inf_l:
for curr_pso_inf_r in pso_inf_r:
#prepare pipeline
ii={'action':'pipeline',
'module_uoa':'program',
'data_uoa':'clblast-tune',
'prepare':'yes',
'dependencies': cdeps,
'no_compiler_description':'yes',
'cmd_key':kernel[0],
"target_os":tos,
"device_id":tdid,
"out":'con',
"no_state_check":"yes",
'flags':'-O3',
}
r=ck.access(ii)
if r['return']>0: return r
fail=r.get('fail','')
if fail=='yes': return {'return':10, 'error':'pipeline failed ('+r.get('fail_reason','')+')'}
ready=r.get('ready','')
if ready!='yes': return {'return':11, 'error':'pipeline not ready'}
state=r['state']
tmp_dir=state['tmp_dir']
xcdeps=r.get('dependencies',{})
# Clean pipeline.
if 'ready' in r: del(r['ready'])
if 'fail' in r: del(r['fail'])
if 'return' in r: del(r['return'])
pipeline=copy.deepcopy(r)
ck.out('PSO_INF_G : '+curr_pso_inf_g)
curr_pso_inf_g_tag="inf_g" + curr_pso_inf_g
ck.out('PSO_INF_L : '+curr_pso_inf_l)
curr_pso_inf_l_tag="inf_l" + curr_pso_inf_l
ck.out('PSO_INF_R : '+curr_pso_inf_r)
curr_pso_inf_r_tag="inf_r" + curr_pso_inf_r
record_repo='local'
record_uoa='explore-matrix-size-'+kernel[0]+'-pso-'+curr_pso_inf_g_tag + curr_pso_inf_l_tag + curr_pso_inf_r_tag
ck.out('---------------------------------------------------------------------------------------')
ck.out('Experiment - %s:%s' % (record_repo, record_uoa))
local_pso_inf_g=[]
local_pso_inf_g.append(curr_pso_inf_g)
local_pso_inf_l=[]
local_pso_inf_l.append(curr_pso_inf_l)
local_pso_inf_r=[]
local_pso_inf_r.append(curr_pso_inf_r)
cpipeline=copy.deepcopy(pipeline)
ii={
'action':'autotune',
'module_uoa':'pipeline',
'data_uoa':'program',
'choices_order':[
[
'##env#CK_CLBLAST_MSIZE'
],
[
'##env#CK_CLBLAST_NSIZE',
],
[
'##env#CK_CLBLAST_KSIZE'
],
[
'##env#CK_TUNER_NUM_OF_STRATEGIES'
],
[
'##env#CK_SEARCH_STRATEGY'
],
[
'##env#CK_PSO_SWARM_SIZE'
],
[
'##env#CK_PSO_INF_G'
],
[
'##env#CK_PSO_INF_L'
],
[
'##env#CK_PSO_INF_R'
]
],
'choices_selection':[
{"type":"loop-with-next", "choice":size_m, "default":"256"},
{"type":"loop-with-next", "choice":size_n, "default":"256"},
{"type":"loop-with-next", "choice":size_k, "default":"256"},
{"type":"loop", "choice":num_strategy, "default":"1"},
{"type":"loop", "choice":strategy, "default":"2"},
{"type":"loop", "choice":pso_swarm_size, "default":"4"},
{"type":"loop", "choice":local_pso_inf_g, "default":"0.4"},
{"type":"loop", "choice":local_pso_inf_l, "default":"0.0"},
{"type":"loop", "choice":local_pso_inf_r, "default":"0.4"},
],
'features_keys_to_process':['##choices#*'],
'iterations':-1,
'repetitions':1,
'record':'yes',
'record_failed':'yes',
'record_params':{
'search_point_by_features':'yes'
},
'record_repo':record_repo,
'record_uoa':record_uoa,
'tags':['explore-clblast-matrix-size', kernel[0], model, size_tag,curr_pso_inf_g_tag, curr_pso_inf_l_tag, curr_pso_inf_r_tag],
'pipeline': cpipeline,
'out':'con'
}
r=ck.access(ii)
if DEBUG > 0: print("%s %s" %(DEBUG_STR, r))
if r['return']>0: return r
fail=r.get('fail','')
if fail=='yes':
return {'return':10, 'error':'pipeline failed ('+r.get('fail_reason','')+')'}
return {'return':0}
def setup(i):
"""
Input: {
}
Output: {
return [int] - return code = 0 if success or >0 if error
(error) [str] - error string if return>0
}
"""
# Get current configuration
cfg={}
ii={'action':'load',
'repo_uoa':config.CK_CFG_REPO_UOA,
'module_uoa':config.CK_CFG_MODULE_UID,
'data_uoa':config.CK_CFG_DATA_UOA}
r=ck.access(ii)
if (r['return']>0 and r['return']!=16): ck.err(r)
if r['return']==0: cfg=r['dict']
# Check commands
# Username ##########################################################
username=cfg.get('username','')
if i.get('username')!=None: username=i['username']
if username=='' or username==None:
r=ck.inp({'text':'Enter cK username: '******'return']>0: ck.err(r)
username=r['string'].strip()
if username==None: username=''
cfg['username']=username
# API key ###########################################################
api_key=cfg.get('api_key','')
if i.get('api_key')!=None: api_key=i['api_key']
if api_key=='' or api_key==None:
r=ck.inp({'text':'Enter your cK API key: '})
if r['return']>0: ck.err(r)
api_key=r['string'].strip()
if api_key==None: api_key=''
cfg['api_key']=api_key
# Server URL ###########################################################
server_url=cfg.get('server_url','')
if i.get('server_url')!=None and i.get('server_url')!='': server_url=i['server_url']
if server_url==None or server_url=='': server_url=config.CR_DEFAULT_SERVER_URL
cfg['server_url']=server_url
# Server User ###########################################################
server_user=cfg.get('server_user','')
if i.get('server_user')!=None and i.get('server_user')!='': server_user=i['server_user']
if server_user!=None and server_user!='': cfg['server_user']=server_user
# Server Pass ###########################################################
server_pass=cfg.get('server_pass','')
if i.get('server_pass')!=None and i.get('server_pass')!='': server_pass=i['server_pass']
if server_pass!=None and server_pass!='': cfg['server_pass']=server_pass
# Server Skip Certificate Validation ###########################################################
server_skip_validation=cfg.get('server_skip_validation','')
if i.get('server_skip_validation')!=None and i.get('server_skip_validation')!='': server_skip_validation=i['server_skip_validation']
if server_skip_validation=='yes': cfg['server_skip_validation']=server_skip_validation
# Save configuration
r=ck.access({'action':'update',
'repo_uyoa':config.CK_CFG_REPO_UOA,
'module_uoa':config.CK_CFG_MODULE_UID,
'data_uoa':config.CK_CFG_DATA_UOA,
'dict':cfg,
'sort_keys':'yes'})
if r['return']>0: ck.err(r)
# Print (new/updated) configuration
ck.out('')
ck.out('Current cBench configuration:')
ck.out('')
ck.out(json.dumps(cfg, indent=2, sort_keys=True))
return 0
def do(i, arg):
# Process arguments.
if (arg.accuracy):
experiment_type = 'accuracy'
num_repetitions = 1
else:
experiment_type = 'performance'
num_repetitions = arg.repetitions
random_name = arg.random_name
share_platform = arg.share_platform
# Detect basic platform info.
ii={'action':'detect',
'module_uoa':'platform',
'out':'con'}
if share_platform: ii['exchange']='yes'
r=ck.access(ii)
if r['return']>0: return r
# Keep to prepare ReQuEST meta.
platform_dict=copy.deepcopy(r)
# Host and target OS params.
hos=r['host_os_uoa']
hosd=r['host_os_dict']
tos=r['os_uoa']
tosd=r['os_dict']
tdid=r['device_id']
# program='mobilenets-armcl-opencl'
program='image-classification-tf-py'
ii={'action':'show',
'module_uoa':'env',
'tags':'dataset,imagenet,raw,val'}
rx=ck.access(ii)
if len(rx['lst']) == 0: return rx
# FIXME: It's probably better to use CK_ENV_DATASET_IMAGE_DIR.
img_dir_val = rx['lst'][0]['meta']['env']['CK_CAFFE_IMAGENET_VAL']
if (arg.accuracy):
batch_count = len([f for f in os.listdir(img_dir_val)
if f.endswith('.JPEG') and os.path.isfile(os.path.join(img_dir_val, f))])
else:
batch_count = 1
ii={'action':'show',
'module_uoa':'env',
'tags':'dataset,imagenet,aux'}
rx=ck.access(ii)
if len(rx['lst']) == 0: return rx
img_dir_aux = rx['lst'][0]['meta']['env']['CK_ENV_DATASET_IMAGENET_AUX']
ii={'action':'load',
'module_uoa':'program',
'data_uoa':program}
rx=ck.access(ii)
if rx['return']>0: return rx
mm=rx['dict']
# Get compile-time and run-time deps.
cdeps=mm.get('compile_deps',{})
rdeps=mm.get('run_deps',{})
# Merge rdeps with cdeps for setting up the pipeline (which uses
# common deps), but tag them as "for_run_time".
for k in rdeps:
cdeps[k]=rdeps[k]
cdeps[k]['for_run_time']='yes'
print cdeps
depl=copy.deepcopy(cdeps['lib-tensorflow'])
if (arg.tos is not None) and (arg.did is not None):
tos=arg.tos
tdid=arg.did
ii={'action':'resolve',
'module_uoa':'env',
'host_os':hos,
'target_os':tos,
'device_id':tdid,
'out':'con',
'deps':{'library':copy.deepcopy(depl)},
'quiet':'yes'
}
r=ck.access(ii)
if r['return']>0: return r
udepl=r['deps']['library'].get('choices',[]) # All UOAs of env for TF lib
if len(udepl)==0:
return {'return':1, 'error':'no installed TensorFlow'}
cdeps['lib-tensorflow']['uoa']=udepl[0]
depm=copy.deepcopy(cdeps['model-and-weights'])
ii={'action':'resolve',
'module_uoa':'env',
'host_os':hos,
'target_os':tos,
'device_id':tdid,
'out':'con',
'deps':{'weights':copy.deepcopy(depm)},
'quiet':'yes'
}
r=ck.access(ii)
if r['return']>0: return r
udepm=r['deps']['weights'].get('choices',[])
if len(udepm)==0:
return {'return':1, 'error':'no installed Weights'}
cdeps['lib-tensorflow']['uoa']=udepl[0]
cdeps['model-and-weights']['uoa']=udepm[0]
ii={'action':'pipeline',
'prepare':'yes',
'dependencies':cdeps,
'module_uoa':'program',
'data_uoa':program,
'target_os':tos,
'device_id':tdid,
'no_state_check':'yes',
'no_compiler_description':'yes',
'skip_calibration':'yes',
'env':{
'CK_ENV_DATASET_IMAGENET_VAL':img_dir_val,
'CK_BATCH_COUNT':batch_count,
'CK_BATCHES_DIR':'../batches',
'CK_BATCH_LIST':'../batches',
'CK_IMAGE_LIST':'../images',
'CK_RESULTS_DIR':'predictions',
'CK_SKIP_IMAGES':0
},
'cpu_freq':'max',
'gpu_freq':'max',
'flags':'-O3',
'speed':'no',
'energy':'no',
'skip_print_timers':'yes',
'out':'con'
}
r=ck.access(ii)
if r['return']>0: return r
fail=r.get('fail','')
if fail=='yes':
return {'return':10, 'error':'pipeline failed ('+r.get('fail_reason','')+')'}
ready=r.get('ready','')
if ready!='yes':
return {'return':11, 'error':'pipeline not ready'}
state=r['state']
tmp_dir=state['tmp_dir']
# Remember resolved deps for this benchmarking session.
xcdeps=r.get('dependencies',{})
# Clean pipeline.
if 'ready' in r: del(r['ready'])
if 'fail' in r: del(r['fail'])
if 'return' in r: del(r['return'])
pipeline=copy.deepcopy(r)
for lib_uoa in udepl:
# Load ArmCL lib.
ii={'action':'load',
'module_uoa':'env',
'data_uoa':lib_uoa}
r=ck.access(ii)
if r['return']>0: return r
lib_name=r['data_name']
lib_tags=r['dict']['customize']['version']
# Skip some libs with "in [..]" or "not in [..]".
if arg.accuracy and lib_tags in [ ]: continue
skip_compile='no'
# For each MobileNets model.*************************************************
for model_uoa in udepm:
# Load model.
ii={'action':'load',
'module_uoa':'env',
'data_uoa':model_uoa}
r=ck.access(ii)
if r['return']>0: return r
model_name=r['data_name']
if 'mobilenet' not in r['dict']['tags']:
continue
alpha = float(r['dict']['env']['CK_ENV_TENSORFLOW_MODEL_MOBILENET_MULTIPLIER'])
rho = int(r['dict']['env']['CK_ENV_TENSORFLOW_MODEL_MOBILENET_RESOLUTION'])
record_repo='local'
record_uoa='mobilenets-'+experiment_type+'-'+str(rho)+'-'+str(alpha)+'-tensorflow-'+lib_tags
# Prepare pipeline.
ck.out('---------------------------------------------------------------------------------------')
ck.out('%s - %s' % (lib_name, lib_uoa))
ck.out('%s - %s' % (model_name, model_uoa))
ck.out('Experiment - %s:%s' % (record_repo, record_uoa))
# Prepare autotuning input.
cpipeline=copy.deepcopy(pipeline)
# Reset deps and change UOA.
new_deps={'library':copy.deepcopy(depl),
'weights':copy.deepcopy(depm)}
new_deps['library']['uoa']=lib_uoa
new_deps['weights']['uoa']=model_uoa
jj={'action':'resolve',
'module_uoa':'env',
'host_os':hos,
'target_os':tos,
'device_id':tdid,
'deps':new_deps}
r=ck.access(jj)
if r['return']>0: return r
cpipeline['dependencies'].update(new_deps)
cpipeline['no_clean']=skip_compile
cpipeline['no_compile']=skip_compile
# Prepare common meta for ReQuEST tournament
features=copy.deepcopy(cpipeline['features'])
platform_dict['features'].update(features)
r=ck.access({'action':'prepare_common_meta',
'module_uoa':'request.asplos18',
'platform_dict':platform_dict,
'deps':cpipeline['dependencies'],
'request_dict':request_dict})
if r['return']>0: return r
record_dict=r['record_dict']
meta=r['meta']
if random_name:
rx=ck.gen_uid({})
if rx['return']>0: return rx
record_uoa=rx['data_uid']
tags=r['tags']
tags.append(experiment_type)
tags.append('explore-mobilenets-'+experiment_type)
tags.append(lib_tags)
tags.append(platform_tags)
tags.append(str(rho))
tags.append(str(alpha))
ii={'action':'autotune',
'module_uoa':'pipeline',
'data_uoa':'program',
'choices_order':[
[
'##choices#env#CK_BATCH_SIZE'
],
[
'##choices#env#CK_CONVOLUTION_METHOD_HINT'
],
[
'##choices#env#CK_ENV_MOBILENET_RESOLUTION'
],
[
'##choices#env#CK_ENV_MOBILENET_WIDTH_MULTIPLIER'
]
],
'choices_selection':[
{'type':'loop', 'start':bs['start'], 'stop':bs['stop'], 'step':bs['step'], 'default':bs['default']},
{'type':'loop', 'start':ch['start'], 'stop':ch['stop'], 'step':ch['step'], 'default':ch['default']},
{'type':'loop', 'choice': [rho], 'default': 224},
{'type':'loop', 'choice': [alpha], 'default': 1.0},
],
'features_keys_to_process':['##choices#*'],
'iterations':-1,
'repetitions': num_repetitions,
'record':'yes',
'record_failed':'yes',
'record_params':{
'search_point_by_features':'yes'
},
'tags':tags,
'meta':meta,
'record_dict':record_dict,
'record_repo':record_repo,
'record_uoa':record_uoa,
'pipeline':cpipeline,
'out':'con'
}
r=ck.access(ii)
if r['return']>0: return r
fail=r.get('fail','')
if fail=='yes':
return {'return':10, 'error':'pipeline failed ('+r.get('fail_reason','')+')'}
### end pipeline
return {'return':0}
def run(i):
# Get path1
r=ck.access({'action':'load',
'module_uoa':'experiment',
'data_uoa':euoa1})
if r['return']>0: return r
p1=r['path']
# Get path1
r=ck.access({'action':'load',
'module_uoa':'experiment',
'data_uoa':euoa2})
if r['return']>0: return r
p2=r['path']
# Going through points in path 2
dirList1=os.listdir(p1)
dirList2=os.listdir(p2)
for fn2 in dirList2:
if fn2.endswith('.features.json'):
ck.out('Loading point '+fn2+' ...')
px2=os.path.join(p2, fn2)
r=ck.load_json_file({'json_file':px2})
if r['return']>0: return r
df2=r['dict'].get('features',{})
# Searching in p1
found=False
for fn1 in dirList1:
if fn1.endswith('.features.json'):
px1=os.path.join(p1, fn1)
r=ck.load_json_file({'json_file':px1})
if r['return']>0: return r
df1=r['dict'].get('features',{})
rx=ck.compare_dicts({'dict1':df1, 'dict2':df2})
if rx['return']>0: return rx
equal=rx['equal']
if equal=='yes':
found=True
break
if found:
ck.out(' Found!')
else:
# Removing point
ck.out(' Removing point ...')
fn=fn2[:-14]
for fn0 in dirList2:
if fn0.startswith(fn):
p0=os.path.join(p2,fn0)
os.remove(p0)
return {'return':0}
def consumer(i):
while not done[0]:
try:
category, index, url = entries.get(timeout=1)
except:
continue
try:
# Try adding a CK repository for this category.
repo_uoa = 'local'; module_uoa = 'repo'; data_uoa = 'imagenet-%s' % category
r=ck.access({
'action':'add',
'repo_uoa':repo_uoa,
'module_uoa':module_uoa,
'data_uoa':data_uoa
})
if r['return']>0:
# If already exists, give a warning rather than an error.
if r['return']==16:
if verbose:
sys.stdout.write ("CK info: repository for category \'%s\' already exists.\n" % category)
else:
if verbose:
sys.stderr.write ("CK error: %s\n" % r['error'])
counts_fail[i] += 1
continue
# Get the CK repository for this category.
# FIXME: "ck add --help" says that it returns
# "Output from the 'create_entry' function".
# It may be possible to extract the repo uoa for this category
# from it but it's unclear what it contains...
r=ck.access({
'action':'search',
'repo_uoa':repo_uoa,
'module_uoa':module_uoa,
'data_uoa':data_uoa
})
if r['return']>0:
if verbose:
sys.stderr.write ("CK error: %s\n" % r['error'])
counts_fail[i] += 1
continue
if len(r['lst'])!=1:
if verbose:
sys.stderr.write ("CK error: %d repositories found, expected 1\n" % len(r['lst']))
counts_fail[i] += 1
continue
# Search for an image by the given category URL.
# (Ignore the index as it may not be unique.)
repo_uoa=r['lst'][0]['data_uoa']
module_uoa='dataset'
tags='imagenet,%s,%s' % (category,url)
r=ck.access({
'action':'search',
'repo_uoa':repo_uoa,
'module_uoa':module_uoa,
'tags':tags
})
if r['return']>0:
if verbose:
sys.stderr.write ("CK error: %s\n" % r['error'])
counts_fail[i] += 1
continue
if len(r['lst'])>0:
# If already exists, give a warning rather than an error.
if verbose:
sys.stdout.write ("CK info: image at \'%s\' already downloaded\n" % url)
counts_success[i] += 1
entries.task_done()
continue
# Add the given image to the repository for this category.
data_uoa=str(index).zfill(9)
r=ck.access({
'action':'add',
'repo_uoa':repo_uoa,
'module_uoa':module_uoa,
'data_uoa':data_uoa,
'tags':tags
})
if r['return']>0:
if verbose:
sys.stderr.write ("CK error: %s\n" % r['error'])
counts_fail[i] += 1
continue
# FIXME: "ck add --help" says that it returns
# "Output from the 'create_entry' function".
# It may be possible to extract the repo uoa for this category
# from it but it's unclear what it contains...
r=ck.access({
'action':'search',
'repo_uoa':repo_uoa,
'module_uoa':module_uoa,
'data_uoa':data_uoa
})
if r['return']>0:
if verbose:
sys.stderr.write ("CK error: %s\n" % r['error'])
counts_fail[i] += 1
continue
if len(r['lst'])!=1:
if verbose:
sys.stderr.write ("CK error: %d dataset entries found, expected 1\n" % len(r['lst']))
counts_fail[i] += 1
continue
# Download the image into the image dataset directory.
directory = r['lst'][0]['path']
content = download(url, timeout, retry, sleep_after_dl)
ext = imgtype2ext(imghdr.what('', content))
name = '{0}.{1}'.format(category, ext)
path = os.path.join(directory, name)
with open(path, 'w') as f:
f.write(content)
# Download the image category description.
words_url = "http://www.image-net.org/api/text/wordnet.synset.getwords?wnid=%s" % category
content = download(words_url, timeout, retry, sleep_after_dl)
all_words = content.split("\n")
# Update the image metadata.
info={}
info['dataset_files'] = [ name ]
info['dataset_words'] = [ word for word in all_words if word != ""]
r=ck.access({
'action':'update',
'repo_uoa':repo_uoa,
'module_uoa':module_uoa,
'data_uoa':data_uoa,
'dict':info
})
if r['return']>0:
if verbose:
sys.stderr.write ("CK error: %s\n" % r['error'])
counts_fail[i] += 1
continue
counts_success[i] += 1
time.sleep(sleep_after_dl)
except Exception as e:
counts_fail[i] += 1
if verbose:
sys.stderr.write('Error: {0} / {1}: {2}\n'.format(category, url, e))
entries.task_done()
def do(i):
# List performance entries
r = ck.access({
'action': 'search',
'module_uoa': 'experiment',
'repo_uoa': 'local',
# 'repo_uoa':'ck-request-asplos18-results'})
'data_uoa': '*ck-request-asplos18-caffe-intel-performance-*'
})
if r['return'] > 0: return r
lst = r['lst']
for q in lst:
duid = q['data_uid']
duoa = q['data_uoa']
ruid = q['repo_uid']
path = q['path']
ck.out(duoa)
# Load entry
r = ck.access({
'action': 'load',
'module_uoa': 'experiment',
'data_uoa': duid,
'repo_uoa': ruid
})
if r['return'] > 0: return r
dd = r['dict']
ruid = r['repo_uid']
apath = r['path']
# Check extra info
if 'inception-v3' in duoa:
model = 'inception-v3'
model_species = '1b339ddb13408f8f'
model_size = 95533753
elif 'resnet50' in duoa:
model = 'resnet50'
model_species = 'd777f6335496db61'
model_size = 102462397
if model == '':
return {'return': 1, 'error': 'model is not recognized'}
prec = ''
if '-fp32' in duoa:
prec = 'fp32'
elif '-int8' in duoa:
prec = 'int8'
model_size = model_size / 4 # Guess
if prec == '':
return {'return': 1, 'error': 'model precision is not recognized'}
# Updating meta if needed
dd['meta'][
'scenario_module_uoa'] = 'a555738be4b65860' # module:request.asplos18
dd['meta']['model_species'] = model_species # model.species:mobilenets
dd['meta'][
'dataset_species'] = 'ImageNet' # dataset species (free format)
dd['meta']['dataset_size'] = 50000 # number of images ...
dd['meta'][
'platform_species'] = 'server' # embedded vs server (maybe other classifications such as edge)
# Unified full name for some deps
ds = dd['meta']['deps_summary']
x = ds['caffemodel']
r = ck.access({
'action': 'make_deps_full_name',
'module_uoa': 'request.asplos18',
'deps': x
})
if r['return'] > 0: return r
dd['meta']['model_design_name'] = r['full_name']
x = ds['lib-caffe']
r = ck.access({
'action': 'make_deps_full_name',
'module_uoa': 'request.asplos18',
'deps': x
})
if r['return'] > 0: return r
dd['meta']['library_name'] = r['full_name']
x = x['deps']['compiler']
r = ck.access({
'action': 'make_deps_full_name',
'module_uoa': 'request.asplos18',
'deps': x
})
if r['return'] > 0: return r
dd['meta']['compiler_name'] = r['full_name']
used_gpgpu = False
if ds.get('lib-caffe', {}).get('deps', {}).get(
'compiler-cuda', {}).get('data_name', '') != '':
used_gpgpu = True
if used_gpgpu:
# GPU used
dd['meta']['cpu_name'] = ''
dd['meta']['cpu_vendor'] = ''
dd['meta']['platform_peak_power'] = 180 #Watts
dd['meta']['platform_price'] = 700 # $
dd['meta']['platform_price_date'] = '20180101' # date
else:
dd['meta']['gpgpu_name'] = ''
dd['meta']['gpgpu_vendor'] = ''
dd['meta']['platform_peak_power'] = 105 #Watts
dd['meta']['platform_price'] = 1166 # $
dd['meta']['platform_price_date'] = '20141212' # date
dd['meta']['artifact'] = 'e7cc77d72f13441e' # artifact description
dd['meta']['model_precision'] = prec
dd['meta']['processed'] = 'yes'
# Updating entry
r = ck.access({
'action': 'update',
'module_uoa': 'experiment',
'data_uoa': duid,
'repo_uoa': ruid,
'dict': dd,
'substitute': 'yes',
'ignore_update': 'yes',
'sort_keys': 'yes'
})
if r['return'] > 0: return r
# Checking points to aggregate
os.chdir(path)
dperf = os.listdir(path)
for f in dperf:
if f.endswith('.cache.json'):
os.system('git rm -f ' + f)
elif f.endswith('.flat.json'):
ck.out(' * ' + f)
# Load performance file
p1 = os.path.join(path, f)
r = ck.load_json_file({'json_file': p1})
if r['return'] > 0: return r
d = r['dict']
d['##features#processed#min'] = 'yes'
# Clean up keys
d1 = {}
for k in d:
v = d[k]
if not k.startswith('##characteristics#run#inference_latency') and \
not k.startswith('##characteristics#run#prediction_time_avg_s') and \
not k.startswith('##characteristics#run#inference_throughput') and \
not k.startswith('##characteristics#run#usage_cost'):
d1[k] = v
d = d1
# Unify execution time + batch size
x = d.get(
'##characteristics#run#REAL_ENV_CK_CAFFE_BATCH_SIZE#min',
'')
if x != None and x != '':
batch = int(x)
d['##features#batch_size#min'] = batch
tall = d.get('##characteristics#run#time_fw_s#all', [])
if batch == 1:
# inference latency
d['##features#measuring_latency#min'] = 'yes'
r = ck.access({
'action': 'stat_analysis',
'module_uoa': 'experiment',
'dict': d,
'dict1': {
'##characteristics#run#inference_latency': tall
}
})
if r['return'] > 0: return r
tnew = []
cnew = []
for t in tall:
t1 = t / batch
tnew.append(t1)
c1 = t1 * cost / (60 * 60)
if c1 != 0:
cnew.append(c1)
r = ck.access({
'action': 'stat_analysis',
'module_uoa': 'experiment',
'dict': d,
'dict1': {
'##characteristics#run#prediction_time_avg_s': tnew
}
})
if r['return'] > 0: return r
if len(cnew) > 0:
r = ck.access({
'action': 'stat_analysis',
'module_uoa': 'experiment',
'dict': d,
'dict1': {
'##characteristics#run#usage_cost': cnew
}
})
if r['return'] > 0: return r
d['##characteristics#run#usage_cost_per_hour#min'] = cost
d['##characteristics#run#usage_cost_date'] = '20180403'
# Throughput for all batches
if len(tnew) > 0:
tall = tnew # from previous calculation
tnew = []
for t in tall:
t1 = 1 / t
tnew.append(t1)
r = ck.access({
'action': 'stat_analysis',
'module_uoa': 'experiment',
'dict': d,
'dict1': {
'##characteristics#run#inference_throughput':
tnew
}
})
if r['return'] > 0: return r
d['##features#model_size#min'] = model_size
if not used_gpgpu:
d['##features#cpu_freq#min'] = 2000
d['##features#freq#min'] = d['##features#cpu_freq#min']
else:
d['##features#gpu_freq#min'] = 1600
d['##features#freq#min'] = d['##features#gpu_freq#min']
# Save updated dict
r = ck.save_json_to_file({
'json_file': p1,
'dict': d,
'sort_keys': 'yes'
})
if r['return'] > 0: return r
return {'return': 0}
def main(i):
# Load common table file (for all models)
ddd = {}
r = ck.load_json_file({'json_file': 'save_all_model_data_tmp.json'})
if r['return'] == 0:
ddd = r['dict']
# Searching for features
for gi in range(0, len(gcc)):
g = gcc[gi]
depth = best_depth[gi]
ck.out(
'********************************************************************************'
)
ck.out('Modeling optimizations for ' + g)
ck.out('')
if g not in ddd: ddd[g] = {}
gx = g.replace(' ', '_')
gx2 = g.replace(' ', '-')
r = ck.load_json_file(
{'json_file': 'prepare_train_data_tmp.' + gx + '.json'})
if r['return'] > 0: ck.err(r)
d = r['dict']
ftable = d['ftable']
ctable = d['ctable']
s = '==============================================================\n'
s += 'Depth: ' + str(depth) + '\n\n'
ck.out(s)
# Building decision tree on all data
ck_model_entry_name = "rpi3-milepost-model-" + gx2.lower()
ii = {
'action': 'build',
'module_uoa': 'model',
'ftable': ftable,
'ctable': ctable,
'keep_temp_files': 'yes',
"model_module_uoa": "model.sklearn",
"model_name": "dtc",
"model_file": "tmp-model-sklearn-dtc",
"model_params": {
"max_depth": depth
},
"model_repo_uoa": "ck-rpi-optimization-results",
"model_data_uoa": ck_model_entry_name,
"out": ""
}
# Training
cii = copy.deepcopy(ii)
r = ck.access(ii)
if r['return'] > 0: ck.err(r)
# Validating
ii = copy.deepcopy(ii)
ii['action'] = 'validate'
r = ck.access(ii)
if r['return'] > 0: ck.err(r)
obs = r['observations']
wrong = r['mispredictions']
acc = float(obs - wrong) / float(obs)
x = ' Accuracy on all data: ' + str(acc)
s += x
ck.out(x)
# Record example of features to demo predictions (to be integrated with compiler optimization prediction (web)services)
d = {
"action": "use",
"module_uoa": "model",
"features": ftable[123], # features of some random benchmark
"model_module_uoa": "model.sklearn",
"model_name": "dtc",
"model_file": "tmp-model-sklearn-dtc",
"model_data_uoa": ck_model_entry_name
}
r = ck.save_json_to_file({
'json_file':
'process_model_using_decision_trees_and_record_to_ck_use.' + gx +
'.json',
'dict':
d
})
if r['return'] > 0: return r
return {'return': 0}
def main(i):
# Load common table file (for all models)
ddd = {}
r = ck.load_json_file({'json_file': 'save_all_model_data_tmp.json'})
if r['return'] == 0:
ddd = r['dict']
# Searching for features
for g in gcc:
random.seed(12345)
ck.out(
'********************************************************************************'
)
ck.out('Modeling optimizations for ' + g)
ck.out('')
if g not in ddd: ddd[g] = {}
gx = g.replace(' ', '_')
r = ck.load_json_file(
{'json_file': 'prepare_train_data_tmp.' + gx + '.json'})
if r['return'] > 0: ck.err(r)
d = r['dict']
ftable = d['ftable']
ctable = d['ctable']
s = ''
for iteration in range(1, 10):
start_time = time.time()
# Full cross-validation
acc = 0
obs = 0
wrong = 0
acc_min = None
acc_max = None
s = '==============================================================\n'
s += 'Iteration: ' + str(iteration) + '\n\n'
if iteration == 1:
# Default params from TF example
hu = [10, 20, 10]
ts = 1000
else:
# Generate random DNN topology and params
hu = []
nhu = random.randint(1, 5)
for k in range(0, nhu):
x = random.randint(10, 30)
hu.append(x)
ts = random.randint(1000, 3000)
s += ' Hidden units: ' + str(hu) + '\n'
s += ' Training steps: ' + str(ts) + '\n\n'
ck.out(s)
for n in range(
0, 3
): # Trying to build model N times (random - sometimes slightly different result)
# Building decision tree on all data
ii = {
'action': 'build',
'module_uoa': 'model',
'ftable': ftable,
'ctable': ctable,
'keep_temp_files': 'yes',
"model_module_uoa": "model.tf",
"model_name": "dnn_classifier",
"model_file": "process_model_using_dnn_tf/" + gx,
"model_params": {
"hidden_units": hu,
"training_steps": ts
},
"out": ""
}
# Training
cii = copy.deepcopy(ii)
r = ck.access(ii)
if r['return'] > 0: ck.err(r)
# Validating
ii = copy.deepcopy(ii)
ii['action'] = 'validate'
r = ck.access(ii)
if r['return'] > 0: ck.err(r)
obs += r['observations']
wrong += r['mispredictions']
acc = float(obs - wrong) / float(obs)
x = '\n Accuracy on all data (' + str(
n + 1) + ' out of 3): ' + str(acc) + '\n'
s += x
ck.out(x)
acc = float(obs - wrong) / float(obs)
if acc_min == None or acc < acc_min:
acc_min = acc
if acc_max == None or acc > acc_max:
acc_max = acc
stop_time = time.time() - start_time
x = '\n\nIteration: ' + str(
iteration
) + ' ; accuracy (min/max): ' + '%.2f' % acc_min + ' .. ' + '%.2f' % acc_max + '\n'
x = '\n Elapsed time: ' + '%.1f' % stop_time + ' sec.\n'
s += x
ck.out(x)
# Cross-validating (for simplicity 1 run)
cross_obs = 0
cross_wrong = 0
x = ' *************************************************\n'
x += ' Cross-validating model (leave one out)\n\n'
s += x
ck.out(x)
for bench in range(0, len(ftable)):
train_ftable = []
train_ctable = []
test_ftable = []
test_ctable = []
for k in range(0, len(ftable)):
if k != bench:
train_ftable.append(ftable[k])
train_ctable.append(ctable[k])
else:
test_ftable.append(ftable[k])
test_ctable.append(ctable[k])
# Selecting model
ii = {
'action':
'build',
'module_uoa':
'model',
'ftable':
train_ftable,
'ctable':
train_ctable,
'keep_temp_files':
'no',
"model_module_uoa":
"model.tf",
"model_name":
"dnn_classifier",
"model_file":
"tmp-model-tf-dnn-classifier-" + gx + '_' + str(iteration),
"model_params": {
"hidden_units": hu,
"training_steps": ts
},
"out":
""
}
# Training
cii = copy.deepcopy(ii)
r = ck.access(ii)
if r['return'] > 0: ck.err(r)
# Validating
ii = copy.deepcopy(ii)
ii['action'] = 'validate'
ii['ftable'] = test_ftable
ii['ctable'] = test_ctable
# ii['ftable']=ftable
# ii['ctable']=ctable
r = ck.access(ii)
if r['return'] > 0: ck.err(r)
cross_obs += r['observations']
cross_wrong += r['mispredictions']
cross_acc = float(cross_obs - cross_wrong) / float(cross_obs)
x = '\n ' + str(bench) + ' out of ' + str(
len(ftable)
) + ' ) current cross-validation accuracy: ' + '%.2f' % cross_acc + '\n'
s += x
ck.out(x)
stop_time = time.time() - start_time
x = '\nIteration: ' + str(
iteration
) + ' ; accuracy (with cross-validation): ' + '%.2f' % cross_acc + '\n'
x = '\n Elapsed time: ' + '%.1f' % stop_time + ' sec.\n'
s += x
ck.out(x)
ddd[g]['dnn_tf_with_cross_validation_iteration_' + str(iteration) +
'_ft1_ft65'] = cross_acc
ddd[g]['dnn_tf_without_cross_validation_iteration_' +
str(iteration) + '_ft1_ft65'] = acc_max
r = ck.save_text_file({
'text_file':
'process_model_using_decision_trees/log.' + gx + '.txt',
'string':
s
})
if r['return'] > 0: return r
# Save common data (continuously, not to loose data)
r = ck.save_json_to_file({
'json_file': 'save_all_model_data_tmp.json',
'dict': ddd
})
if r['return'] > 0: return r
return {'return': 0}
def main(i):
cur_dir = os.getcwd()
fas = os.path.join(cur_dir, aggregated_stats)
# Get some info about current platform
ii = {'action': 'detect', 'module_uoa': 'platform', 'out': 'con'}
r = ck.access(ii)
if r['return'] > 0: return r
hos = r['host_os_uid']
hosx = r['host_os_uoa']
hosd = r['host_os_dict']
tos = r['os_uid']
tosx = r['os_uoa']
tosd = r['os_dict']
cpu_name = r['features']['cpu']['name']
plat_name = r['features']['platform']['name']
#############################################################
ck.out(line)
ck.out('CPU name: ' + cpu_name)
ck.out('Plat name: ' + plat_name)
#############################################################
ck.out(line)
ck.out('Loading aggregated stats ...')
aa = []
if os.path.isfile(fas):
r = ck.load_json_file({'json_file': fas})
if r['return'] > 0: return r
ax = r['dict']
if 'all' not in ax: ax['all'] = []
aa = ax['all']
#############################################################
ck.out(line)
ck.out('Finding entry related to this platform ...')
found = False
for a in aa:
if a.get('cpu_name', '') == cpu_name and a.get('plat_name',
'') == plat_name:
found = True
if not found:
a = {'cpu_name': cpu_name, 'plat_name': plat_name}
aa.append(a)
if 'data' not in a: a['data'] = {}
data = a.get('data', {})
# Init pipeline
r = ck.access({
'action': 'pipeline',
'module_uoa': 'program',
'data_uoa': 'shared-matmul-c2',
'cpu_freq': 'max',
'gpu_freq': 'max',
'speed': 'yes',
'compiler_vars': {
'USE_BLOCKED_MATMUL': 'YES'
},
'no_state_check': 'yes',
'prepare': 'yes',
'out': 'con'
})
if r['return'] > 0: return r
ready = r['ready']
if ready != 'yes':
return {'return': 1, 'error': 'can\'t init pipeline'}
pipeline = r
# Compile program ones
tpipeline = copy.deepcopy(pipeline)
r = ck.access({
'action': 'autotune',
'module_uoa': 'pipeline',
'pipeline': pipeline,
'pipeline_update': {
'env': {
'CT_MATRIX_DIMENSION': 32,
'CT_BLOCK_SIZE': 32
}
},
'iterations': 1,
'repetitions': 1,
'out': 'con'
})
if r['return'] > 0: return r
lsa = r.get('last_stat_analysis', {}).get('dict_flat', {})
time_min = lsa.get('##characteristics#run#execution_time#min', None)
if time_min == None or time_min == 0.0:
return {'return': 1, 'error': 'failed to run default pipeline'}
# data is per N size
for nn in range(-1, 2):
for n in range(3, 32):
N = (2**n) + nn
SN = str(N)
if SN not in data: data[SN] = {}
xdata = data.get(SN, {})
tmin = xdata.get('tmin', None)
tmax = xdata.get('tmax', None)
gmin = xdata.get('gmin', None)
gmax = xdata.get('gmax', None)
best_tile = xdata.get('best_tile', None)
for opts in range(0, 16):
# Choose if random BS or power of two or power of two -+1
if opts == 0:
BS = 1
elif opts == 1:
BS = N
else:
BS = 2**(opts - 1)
if BS > N: continue
ck.out('Matrix size: ' + str(N))
ck.out('Tile size: ' + str(BS))
# Run pipeline
tpipeline = copy.deepcopy(pipeline)
r = ck.access({
'action': 'autotune',
'module_uoa': 'pipeline',
'pipeline': pipeline,
'pipeline_update': {
'no_compile': 'yes',
'env': {
'CT_MATRIX_DIMENSION': N,
'CT_BLOCK_SIZE': BS
}
},
'iterations': 1,
'repetitions': 3,
'out': 'con'
})
if r['return'] > 0: return r
lsa = r.get('last_stat_analysis', {}).get('dict_flat', {})
tmin = lsa.get('##characteristics#run#execution_time#min',
None)
changed = False
if tmin != None:
ops = 2 * (N * N * N)
gflops = 1.0e-9 * ops / tmin
SBS = str(BS)
xdata[SBS] = {'tmin': tmin, 'gflops': gflops}
ck.out(line)
ck.out('Saving aggregated stats ...')
r = ck.save_json_to_file({
'json_file': fas,
'dict': {
'all': aa
},
'sort_keys': 'yes'
})
if r['return'] > 0: return r
#############################################################
ck.out(line)
ck.out('Saving aggregated stats ...')
r = ck.save_json_to_file({
'json_file': fas,
'dict': {
'all': aa
},
'sort_keys': 'yes'
})
if r['return'] > 0: return r
return {'return': 0}
def download(i):
"""
Input: {
cid [str] - CK CID of format (repo UOA:)module UOA:data UOA
(can use wildcards)
(version) [str] - assign version
(force) [bool] - if True, force download even if components already exists
(tags) [str] - can search by tags (usually soft/package)
(all) [bool] - if True, download dependencies (without force!)
}
Output: {
return [int] - return code = 0 if success or >0 if error
(error) [str] - error string if return>0
}
"""
# CID ###########################################################
cid = i.get('cid')
if cid == '' or cid == None:
return {'return': 1, 'error': 'CK entry (CID) is not defined'}
version = i.get('version')
if version == None: version = ''
force = i.get('force')
al = i.get('all')
skip_module_check = i.get('skip_module_check', False)
# Parse CID
r = ck.parse_cid({'cid': cid})
if r['return'] > 0: return r
repo_uoa = r.get('repo_uoa', '')
data_uoa = r.get('data_uoa', '')
module_uoa = r.get('module_uoa', '')
tags = i.get('tags', '')
spaces = i.get('spaces', '')
# Get current configuration
r = config.load({})
if r['return'] > 0: return r
cfg = r['dict']
# Sending request to download
r = comm.send({
'config': cfg,
'action': 'download',
'dict': {
'module_uoa': module_uoa,
'data_uoa': data_uoa,
'version': version,
'tags': tags
}
})
if r['return'] > 0:
return r
lst = r['components']
for l in lst:
fpack64 = l['file_base64']
fmd5 = l['file_md5']
muoa = l['module_uoa']
muid = l['module_uid']
duoa = l['data_uoa']
duid = l['data_uid']
dependencies = l.get('dependencies', [])
xcid = muoa + ':' + duoa
# Check if module exists
if not skip_module_check:
r = ck.access({
'action': 'find',
'module_uoa': 'module',
'data_uoa': muoa,
'common_func': 'yes'
})
if r['return'] > 0:
if r['return'] != 16: return r
x = 'module:' + muoa
if repo_uoa != '': x = repo_uoa + ':' + x
# FGG: we should not add "version" for dependencies or related components since it's not the same!
# r=download({'cid':x, 'force':force, 'version':version, 'skip_module_check':True, 'all':al})
r = download({
'cid': x,
'force': force,
'skip_module_check': True,
'all': al
})
if r['return'] > 0: return r
# Check if entry already exists
path = ''
r = ck.access({
'action': 'find',
'common_func': 'yes',
'repo_uoa': repo_uoa,
'module_uoa': muid,
'data_uoa': duoa
})
if r['return'] == 0:
if not force:
return {
'return': 8,
'error': 'local entry for "' + xcid + '" already exists'
}
else:
if r['return'] != 16: return r
r = ck.access({
'action': 'add',
'common_func': 'yes',
'repo_uoa': repo_uoa,
'module_uoa': muid,
'data_uoa': duoa,
'data_uid': duid,
'ignore_update': 'yes'
})
if r['return'] > 0: return r
path = r['path']
# Prepare pack
ppz = os.path.join(path, config.PACK_FILE)
if os.path.isfile(ppz):
if not force:
return {
'return': 1,
'error': 'pack file already exists (' + ppz + ')'
}
os.remove(ppz)
# Save pack to file
rx = ck.convert_upload_string_to_file({
'file_content_base64': fpack64,
'filename': ppz
})
if rx['return'] > 0: return rx
# MD5 of the pack
rx = ck.load_text_file({'text_file': ppz, 'keep_as_bin': 'yes'})
if rx['return'] > 0: return rx
bpack = rx['bin']
import hashlib
md5 = hashlib.md5(bpack).hexdigest()
if md5 != fmd5:
return {
'return':
1,
'error':
'MD5 of the newly created pack (' + md5 +
') did not match the one from CodeReef server (' + fmd5 + ')'
}
# Unpack to src subdirectory
import zipfile
f = open(ppz, 'rb')
z = zipfile.ZipFile(f)
for d in z.namelist():
if d != '.' and d != '..' and not d.startswith(
'/') and not d.startswith('\\'):
pp = os.path.join(path, d)
if d.endswith('/'):
# create directory
if not os.path.exists(pp): os.makedirs(pp)
else:
ppd = os.path.dirname(pp)
if not os.path.exists(ppd): os.makedirs(ppd)
# extract file
fo = open(pp, 'wb')
fo.write(z.read(d))
fo.close()
if pp.endswith('.sh') or pp.endswith('.bash'):
import stat
st = os.stat(pp)
os.chmod(pp, st.st_mode | stat.S_IEXEC)
f.close()
# Remove pack file
os.remove(ppz)
# Note
ck.out(spaces + 'Successfully downloaded "' + xcid + '" to ' + path)
# Check deps
if al:
if len(dependencies) > 0:
ck.out(spaces + ' Checking dependencies ...')
# import json
# print (json.dumps(dependencies))
# input('xyz')
for dep in dependencies:
muoa = dep.get('module_uid', '')
duoa = dep.get('data_uid', '')
tags = dep.get('tags', [])
xtags = ''
if len(tags) > 0:
xtags = ','.join(tags)
muoa = 'package'
duoa = ''
cid = muoa + ':' + duoa
rx = download({
'cid': cid,
'all': al,
'tags': xtags,
'spaces': spaces + ' '
})
if rx['return'] > 0 and rx['return'] != 8 and rx[
'return'] != 16:
return rx
if rx['return'] == 16:
if xtags == '': return rx
rx = download({
'cid': 'soft:',
'all': al,
'tags': xtags,
'spaces': spaces + ' '
})
if rx['return'] > 0 and rx['return'] != 8: return rx
return r
def main(arg):
# CUSTOMIZABLE VARIABLES!!!!
module_uoa = 'experiment'
repo_uoa = 'explore-matrix-size-xgemm-fp32-firefly-rk3399'
tags='explore-clblast-matrix-size'
output_filename = 'tmp-ck-clblast-tune.json'
weights_filename='NULL'
WEIGHTS = 0
if arg.fp is not None:
weights_filename = arg.fp
if (os.path.isfile(weights_filename)):
print("{RANKING ERROR %s not found. USE WEIGHTS=0}" %(weights_filename))
WEIGHTS = 1
else:
print("[RANKING ERROR] %s not found. USE WEIGHTS=0" %(weights_filename))
### END CUST
dlist=[]
dblist=[]
r=ck.access({'action':'search', 'repo_uoa':repo_uoa, 'module_uoa':module_uoa, 'tags':tags})
if r['return']>0:
print ("Error: %s" % r['error'])
experiments=r['lst']
if len(experiments) == 0:
print("No experiments found in repo %s with tags %s" %(repo_uoa, tags))
exit(1)
for exp in experiments:
# print exp
data_uoa = exp['data_uoa']
r = ck.access({'action':'list_points', 'repo_uoa':repo_uoa, 'module_uoa':module_uoa, 'data_uoa':data_uoa})
if r['return']>0:
print ("Error: %s" % r['error'])
exit(1)
tags = r['dict']['tags']
# print tags
npoint = len(r['points'])
print ("[RANKING] Number of matrices: %s" %(npoint))
# print npoint
for point in r['points']:
point_file = os.path.join(r['path'], 'ckp-%s.0001.json' % point)
d = get_data(point_file)
dlist.append(d)
# LOAD WEIGHTS
w = []
# WEIGHTS = os.path.isfile(weights_filename) and WEIGHTS
if (WEIGHTS == 0):
for i in range(0,npoint):
w.append(1)
else:
print("Loading weights %s" %(weights_filename))
wdict = json.loads(open(weights_filename).read())
for i in wdict:
print i.get('Execution time (%)')
w.append(float(i.get('Execution time (%)')))
output = get_data_header(point_file)
# FIND THE BEST
#configurations = dlist[0]
#best = sys.float_info.max
#configuration_best_index = len(configurations)
#configuration_count = 0
#final_score = []
#for configuration in configurations:
# res = find_it(configuration['parameters'],dlist)
# tmp_time = 0
# for ctime in res["time"]:
# tmp_time +=ctime
# if tmp_time < best:
# best= tmp_time
# configuration_best_index = configuration_count
# res["total_time"] = tmp_time
# final_score.append(res)
# configuration_count +=1
#mbest = final_score[configuration_best_index]
mbest= find_best(dlist, w)
### PREPARE OUTPUT
del mbest['time']
del mbest['total_time']
mbest['GFLOPS'] = 0.0
mbest['kernel'] = output.get('kernel')
output['data'] = 'na'
output['db'] = 'na'
output['statistics'] = {'default_family':{}, 'default_configuration':{}, 'best_configuration': mbest }
#print (json.dumps(output, indent=2))
rr=ck.save_json_to_file({'json_file':output_filename, 'dict':output})
def do(i, arg):
fp = arg.fp
if fp is not None:
fromfile = os.path.isfile(fp)
if (fromfile):
print("Loading triples %s" % (fp))
triples = json.loads(open(fp).read())
del size_m[:]
del size_n[:]
del size_k[:]
for i in triples:
size_m.append(str(i.get('bSizeM')))
size_n.append(str(i.get('bSizeN')))
size_k.append(str(i.get('bSizeK')))
if VERBOSE or DEBUG:
print('[Experiment] %s' % title)
print('[Preparing pipeline] Clock resolution: %d' % clock_resolution)
print('[Preparing pipeline] Matrix sizes: m=%s, k=%s, n=%s: ' %
(size_m, size_k, size_n))
print('[Preparing pipeline] Precision: %d' % precision)
print('[Preparing pipeline] Run for configuration: %d' % run)
print('[Preparing pipeline] More parms... ')
ntrip = len(size_m)
print('[Experiment] Number of triple(s) %s' % (ntrip))
size_tag = ''
for tp in range(0, ntrip):
if (tp == ntrip - 1):
size_tag += str(
(int(size_m[tp]) * int(size_n[tp]) * int(size_k[tp])))
else:
size_tag += str(
(int(size_m[tp]) * int(size_n[tp]) * int(size_k[tp]))) + ','
# Detect basic platform info.
ii = {'action': 'detect', 'module_uoa': 'platform', 'con': 'con'}
r = ck.access(ii)
if DEBUG: print("%s %s" % (DEBUG_STR, r))
if r['return'] > 0: return r
# Host and target OS params.
hos = r['host_os_uoa']
hosd = r['host_os_dict']
tos = r['os_uoa']
tosd = r['os_dict']
tdid = r['device_id']
if DEBUG: print("%s %s %s" % (DEBUG_STR, hos, hosd))
if DEBUG: print("%s %s %s %s" % (DEBUG_STR, tos, tosd, tdid))
# Load CLBLAST program meta and desc to check deps.
ii = {
'action': 'load',
'module_uoa': 'program',
'data_uoa': 'clblast-tune'
}
rx = ck.access(ii)
if DEBUG: print("%s %s " % (DEBUG_STR, rx))
if rx['return'] > 0: return rx
meta = rx['dict']
# Get compile-time and run-time deps.
cdeps = meta.get('compile_deps', {})
rdeps = meta.get('run_deps', {})
# Merge rdeps with cdeps for setting up the pipeline (which uses
# common deps), but tag them as "for_run_time".
for k in rdeps:
cdeps[k] = rdeps[k]
cdeps[k]['for_run_time'] = 'yes'
# CLblast libs.
depl = copy.deepcopy(cdeps['lib-clblast'])
#ON LOCAL MACHINE
if ((arg.tos is not None) and (arg.did is not None)):
tos = arg.tos
tdid = arg.did
ii = {
'action': 'resolve',
'module_uoa': 'env',
'host_os': hos,
'target_os': tos,
'device_id': tdid,
'out': 'con',
'deps': {
'lib-clblast': copy.deepcopy(depl)
}
}
r = ck.access(ii)
if r['return'] > 0: return r
udepl = r['deps']['lib-clblast'].get('choices', [])
if len(udepl) == 0:
return {'return': 1, 'error': 'no installed CLBlast libs'}
#prepare pipeline
ii = {
'action': 'pipeline',
'module_uoa': 'program',
'data_uoa': 'clblast-tune',
'prepare': 'yes',
'dependencies': cdeps,
'no_compiler_description': 'yes',
'cmd_key': kernel[0],
"target_os": tos,
"device_id": tdid,
"out": 'con',
"no_state_check": "yes",
'flags': '-O3',
}
r = ck.access(ii)
if r['return'] > 0: return r
fail = r.get('fail', '')
if fail == 'yes':
return {
'return': 10,
'error': 'pipeline failed (' + r.get('fail_reason', '') + ')'
}
ready = r.get('ready', '')
if ready != 'yes': return {'return': 11, 'error': 'pipeline not ready'}
state = r['state']
tmp_dir = state['tmp_dir']
xcdeps = r.get('dependencies', {})
# Clean pipeline.
if 'ready' in r: del (r['ready'])
if 'fail' in r: del (r['fail'])
if 'return' in r: del (r['return'])
pipeline = copy.deepcopy(r)
record_repo = 'local'
record_uoa = 'explore-matrix-size-' + kernel[0]
ck.out(
'---------------------------------------------------------------------------------------'
)
ck.out('Experiment - %s:%s' % (record_repo, record_uoa))
cpipeline = copy.deepcopy(pipeline)
ii = {
'action':
'autotune',
'module_uoa':
'pipeline',
'data_uoa':
'program',
'choices_order': [['##env#CK_CLBLAST_MSIZE'],
[
'##env#CK_CLBLAST_NSIZE',
], ['##env#CK_CLBLAST_KSIZE']],
'choices_selection': [{
"type": "loop-with-next",
"choice": size_m,
"default": "256"
}, {
"type": "loop-with-next",
"choice": size_n,
"default": "256"
}, {
"type": "loop-with-next",
"choice": size_k,
"default": "256"
}],
'features_keys_to_process': ['##choices#*'],
'iterations':
-1,
'repetitions':
3,
'record':
'yes',
'record_failed':
'yes',
'record_params': {
'search_point_by_features': 'yes'
},
'record_repo':
record_repo,
'record_uoa':
record_uoa,
'tags': ['explore-clblast-matrix-size', kernel[0], model, size_tag],
'pipeline':
cpipeline,
'out':
'con'
}
r = ck.access(ii)
if DEBUG > 0: print("%s %s" % (DEBUG_STR, r))
if r['return'] > 0: return r
fail = r.get('fail', '')
if fail == 'yes':
return {
'return': 10,
'error': 'pipeline failed (' + r.get('fail_reason', '') + ')'
}
return {'return': 0}
def do(i, arg):
# Detect basic platform info.
ii={'action':'detect',
'module_uoa':'platform',
'out':'out'}
r=ck.access(ii)
if r['return']>0: return r
# Host and target OS params.
hos=r['host_os_uoa']
hosd=r['host_os_dict']
tos=r['os_uoa']
tosd=r['os_dict']
tdid=r['device_id']
# Program and command.
program='caffe'
cmd_key='test_gpu'
# Load Caffe program meta and desc to check deps.
ii={'action':'load',
'module_uoa':'program',
'data_uoa':program}
rx=ck.access(ii)
if rx['return']>0: return rx
mm=rx['dict']
# Get compile-time and run-time deps.
cdeps=mm.get('compile_deps',{})
rdeps=mm.get('run_deps',{})
# Merge rdeps with cdeps for setting up the pipeline (which uses
# common deps), but tag them as "for_run_time".
for k in rdeps:
cdeps[k]=rdeps[k]
cdeps[k]['for_run_time']='yes'
# Caffe libs.
depl=copy.deepcopy(cdeps['lib-caffe'])
if (arg.tos is not None) and (arg.did is not None):
tos=arg.tos
tdid=arg.did
ii={'action':'resolve',
'module_uoa':'env',
'host_os':hos,
'target_os':tos,
'device_id':tdid,
'out':'con',
'deps':{'lib-caffe':copy.deepcopy(depl)}
}
r=ck.access(ii)
if r['return']>0: return r
udepl=r['deps']['lib-caffe'].get('choices',[]) # All UOAs of env for Caffe libs.
if len(udepl)==0:
return {'return':1, 'error':'no installed Caffe libs'}
# Caffe models.
depm=copy.deepcopy(cdeps['caffemodel'])
ii={'action':'resolve',
'module_uoa':'env',
'host_os':hos,
'target_os':tos,
'device_id':tdid,
'out':'con',
'deps':{'caffemodel':copy.deepcopy(depm)}
}
r=ck.access(ii)
if r['return']>0: return r
udepm=r['deps']['caffemodel'].get('choices',[]) # All UOAs of env for Caffe models.
if len(udepm)==0:
return {'return':1, 'error':'no installed Caffe models'}
# Prepare pipeline.
cdeps['lib-caffe']['uoa']=udepl[0]
cdeps['caffemodel']['uoa']=udepm[0]
ii={'action':'pipeline',
'prepare':'yes',
'dependencies':cdeps,
'module_uoa':'program',
'data_uoa':program,
'cmd_key':cmd_key,
'target_os':tos,
'device_id':tdid,
'no_state_check':'yes',
'no_compiler_description':'yes',
'skip_calibration':'yes',
'env':{
'CK_CAFFE_SKIP_BACKWARD':1,
'OPENBLAS_NUM_THREADS':4
},
'cpu_freq':'max',
'gpu_freq':'max',
'flags':'-O3',
'speed':'no',
'energy':'no',
'skip_print_timers':'yes',
'out':'con'
}
r=ck.access(ii)
if r['return']>0: return r
fail=r.get('fail','')
if fail=='yes':
return {'return':10, 'error':'pipeline failed ('+r.get('fail_reason','')+')'}
ready=r.get('ready','')
if ready!='yes':
return {'return':11, 'error':'pipeline not ready'}
state=r['state']
tmp_dir=state['tmp_dir']
# Remember resolved deps for this benchmarking session.
xcdeps=r.get('dependencies',{})
# Clean pipeline.
if 'ready' in r: del(r['ready'])
if 'fail' in r: del(r['fail'])
if 'return' in r: del(r['return'])
pipeline=copy.deepcopy(r)
# For each Caffe lib.*******************************************************
for lib_uoa in udepl:
# Load Caffe lib.
ii={'action':'load',
'module_uoa':'env',
'data_uoa':lib_uoa}
r=ck.access(ii)
if r['return']>0: return r
# Get the tags from e.g. 'BVLC Caffe framework (libdnn,viennacl)'
lib_name=r['data_name']
lib_tags=re.match('BVLC Caffe framework \((?P<tags>.*)\)', lib_name)
lib_tags=lib_tags.group('tags').replace(' ', '').replace(',', '-')
# Skip some libs with "in [..]" or "not in [..]".
# if lib_tags in ['opencl-clblast-tune']: continue
if lib_tags not in ['opencl-libdnn-clblast-tune']: continue
# if lib_tags not in ['opencl-clblast-tune', 'opencl-clblast']: continue
# if lib_tags not in ['opencl-libdnn-clblast-tune', 'opencl-libdnn-clblast']: continue
# if lib_tags not in ['opencl-libdnn-viennacl', 'opencl-viennacl']: continue
skip_compile='no'
cmd_key='time_gpu_fp16'
# For each Caffe model.*************************************************
for model_uoa in udepm:
# Load Caffe model.
ii={'action':'load',
'module_uoa':'env',
'data_uoa':model_uoa}
r=ck.access(ii)
if r['return']>0: return r
# Get the tags from e.g. 'Caffe model (net and weights) (deepscale, squeezenet, 1.1)'
model_name=r['data_name']
model_tags = re.match('Caffe model \(net and weights\) \((?P<tags>.*)\)', model_name)
model_tags = model_tags.group('tags').replace(' ', '').replace(',', '-')
# Skip some models with "in [..]" or "not in [..]".
if model_tags in []: continue
record_repo='local'
record_uoa=model_tags+'-'+lib_tags
# Prepare pipeline.
ck.out('---------------------------------------------------------------------------------------')
ck.out('%s - %s' % (lib_name, lib_uoa))
ck.out('%s - %s' % (model_name, model_uoa))
ck.out('Experiment - %s:%s' % (record_repo, record_uoa))
# Prepare autotuning input.
cpipeline=copy.deepcopy(pipeline)
# Reset deps and change UOA.
new_deps={'lib-caffe':copy.deepcopy(depl),
'caffemodel':copy.deepcopy(depm)}
new_deps['lib-caffe']['uoa']=lib_uoa
new_deps['caffemodel']['uoa']=model_uoa
jj={'action':'resolve',
'module_uoa':'env',
'host_os':hos,
'target_os':tos,
'device_id':tdid,
'deps':new_deps}
r=ck.access(jj)
if r['return']>0: return r
cpipeline['dependencies'].update(new_deps)
cpipeline['no_clean']=skip_compile
cpipeline['no_compile']=skip_compile
cpipeline['cmd_key']=cmd_key
ii={'action':'autotune',
'module_uoa':'pipeline',
'data_uoa':'program',
'choices_order':[
[
'##choices#env#CK_CAFFE_BATCH_SIZE'
]
],
'choices_selection':[
{'type':'loop', 'start':bs['start'], 'stop':bs['stop'], 'step':bs['step'], 'default':bs['default']}
],
'features_keys_to_process':['##choices#*'],
'iterations':-1,
'repetitions':num_repetitions,
'record':'yes',
'record_failed':'yes',
'record_params':{
'search_point_by_features':'yes'
},
'record_repo':record_repo,
'record_uoa':record_uoa,
'tags':[ 'explore-batch-size-libs-models', cmd_key, model_tags, lib_tags, platform_tags ],
'pipeline':cpipeline,
'out':'con'}
r=ck.access(ii)
if r['return']>0: return r
fail=r.get('fail','')
if fail=='yes':
return {'return':10, 'error':'pipeline failed ('+r.get('fail_reason','')+')'}
skip_compile='yes'
return {'return':0}
def do(i):
# List performance entries
r=ck.access({'action':'search',
'module_uoa':'experiment',
'data_uoa':'ck-request-asplos18-mobilenets-tvm-arm-performance-*',
'repo_uoa':'local'})
if r['return']>0: return r
lst=r['lst']
for q in lst:
duid=q['data_uid']
duoa=q['data_uoa']
path=q['path']
ck.out(duoa)
# Search matching accuracy entry
aduoa=duoa.replace('-performance-','-accuracy-')
r=ck.access({'action':'find',
'module_uoa':'experiment',
'data_uoa':aduoa,
'repo_uoa':'local'})
if r['return']>0: return r
apath=r['path']
# Checking points to aggregate
dperf=os.listdir(path)
for f in dperf:
if f.endswith('.flat.json'):
ck.out(' * '+f)
# Load performance file
p1=os.path.join(path, f)
r=ck.load_json_file({'json_file':p1})
if r['return']>0: return r
d=r['dict']
p2=os.path.join(path, f[:-10]+'.features_flat.json') # Features
r=ck.load_json_file({'json_file':p2})
if r['return']>0: return r
df=r['dict']
# Remove batch
del(df['##choices#env#CK_BATCH_COUNT'])
# Find matching features file to merge
dacc=os.listdir(apath)
matched=False
for af in dacc:
if af.endswith('.features_flat.json'):
r=ck.load_json_file({'json_file':os.path.join(apath,af)})
if r['return']>0: return r
adf=r['dict']
# Remove batch
del(adf['##choices#env#CK_BATCH_COUNT'])
# Compare dicts
r=ck.compare_dicts({'dict1':df, 'dict2':adf})
if r['return']>0: return r
if r['equal']=='yes':
matched=True
# Load accuracy data to merge
px=os.path.join(apath,af[:-19]+'.flat.json')
r=ck.load_json_file({'json_file':px})
if r['return']>0: return r
dd=r['dict']
# Merge keys
for k in dd:
if k.startswith('##characteristics#run#accuracy_top'):
d[k]=dd[k]
break
if not matched:
return {'return':1, 'error':'no match - strange'}
# Save updated dict
r=ck.save_json_to_file({'json_file':p1, 'dict':d, 'sort_keys':'yes'})
if r['return']>0: return r
return {'return':0}
dd['##characteristics#gpu_copy_is_much_better_cpu#min']=False
if (fcpu/fgpu_only)>1.07:
dd['##characteristics#gpu_only_is_much_better_cpu#min']=True
else:
dd['##characteristics#gpu_only_is_much_better_cpu#min']=False
return {'return':0, 'changed':changed, 'dict':dd}
########################################################
ff=getattr(sys.modules[__name__], 'filter_data')
ii={'action':'filter',
'module_uoa':'experiment',
'out':'con',
'filter_func_addr':ff}
r=ck.load_json_file({'json_file':'filter-add-characteristic-gpu-copy-is-much-better-cpu.py'})
if r['return']>0:
ck.out('Error:'+r['error'])
exit(1)
ii.update(r['dict'])
r=ck.access(ii)
if r['return']>0:
ck.out('Error:'+r['error'])
exit(1)
exit(0)
def do(i):
# List performance entries
r=ck.access({'action':'search',
'module_uoa':'experiment',
'data_uoa':'ck-request-asplos18-mxnet-arm-performance*'
# 'repo_uoa':'ck-request-asplos18-results'
})
if r['return']>0: return r
lst=r['lst']
for q in lst:
duid=q['data_uid']
duoa=q['data_uoa']
ruid=q['repo_uid']
path=q['path']
ck.out(duoa)
# Search matching accuracy entry
r=ck.access({'action':'load',
'module_uoa':'experiment',
'data_uoa':duid,
'repo_uoa':ruid})
if r['return']>0: return r
dd=r['dict']
ruid=r['repo_uid']
apath=r['path']
# Updating meta if needed
dd['meta']['scenario_module_uoa']='a555738be4b65860' # module:request.asplos18
dd['meta']['dataset_species']='ImageNet' # dataset species (free format)
dd['meta']['platform_species']='embedded' # embedded vs server (maybe other classifications such as edge)
dd['meta']['platform_peak_power']=6.05 #Watts http://opensource.rock-chips.com/images/6/60/Rockchip_RK3399_Datasheet_V1.6-20170301.pdf last page
dd['meta']['platform_price']=149 # $, http://shop.t-firefly.com/goods.php?id=45
dd['meta']['platform_price_date']='20180416' # date
dd['meta']['artifact']='08da9685582866a0' # artifact description
dd['meta']['model_precision']='fp32'
dd['meta']['processed']='yes'
# Unified full name for some deps
ds=dd['meta']['deps_summary']
x=ds['mxnet-model']
r=ck.access({'action':'make_deps_full_name','module_uoa':'request.asplos18','deps':x})
if r['return']>0: return r
full_model_name=r['full_name']
dd['meta']['model_design_name']=full_model_name
# for simplicity add manually (can later automate it as in other artifacts, but just didn't have time here)
if 'mobilenet' in full_model_name:
dd['meta']['model_species']='07d4e7aa3750ddc6' # model.species:resnet18
dd['meta']['dataset_size']=50000 # number of images ...
accuracy_top1=0.66694
accuracy_top5=0.87734
elif 'resnet' in full_model_name:
dd['meta']['model_species']='d41bbf1e489ab5e0' # model.species:resnet18
dd['meta']['dataset_size']=25000 # number of images ...
accuracy_top1=0.61318
accuracy_top5=0.83702
elif 'vgg16' in full_model_name:
dd['meta']['model_species']='a3fcac86d42bdbc4' # model.species:resnet18
dd['meta']['dataset_size']=5000 # number of images ...
accuracy_top1=0.63120
accuracy_top5=0.84951
else:
return {'return':1, 'error':'unknown model ('+y+')'}
x=ds['lib-mxnet']
r=ck.access({'action':'make_deps_full_name','module_uoa':'request.asplos18','deps':x})
if r['return']>0: return r
dd['meta']['library_name']=r['full_name']
x=x['deps']['compiler']
r=ck.access({'action':'make_deps_full_name','module_uoa':'request.asplos18','deps':x})
if r['return']>0: return r
dd['meta']['compiler_name']=r['full_name']
# Updating entry
r=ck.access({'action':'update',
'module_uoa':'experiment',
'data_uoa':duid,
'repo_uoa':ruid,
'dict':dd,
'substitute':'yes',
'ignore_update':'yes',
'sort_keys':'yes'
})
if r['return']>0: return r
# Checking points to aggregate
os.chdir(path)
dperf=os.listdir(path)
for f in dperf:
if f.endswith('.cache.json'):
os.system('git rm -f '+f)
elif f.endswith('.flat.json'):
ck.out(' * '+f)
# Load performance file
p1=os.path.join(path, f)
r=ck.load_json_file({'json_file':p1})
if r['return']>0: return r
d1=r['dict']
# Prune some old value
d={}
for k in d1:
if not k.startswith('##characteristics#run#accuracy_top1') and \
not k.startswith('##characteristics#run#accuracy_top5') and \
not k.startswith('##characteristics#run#inference_throughput') and \
not k.startswith('##characteristics#run#inference_latency'):
d[k]=d1[k]
# for simplicity add manually (can later automate it as in other artifacts, but just didn't have time here)
if 'mobilenet' in full_model_name:
model_size=17024109
elif 'resnet' in full_model_name:
model_size=46803089
elif 'vgg16' in full_model_name:
model_size=553432060
else:
return {'return':1, 'error':'unknown model ('+y+')'}
d['##features#model_size#min']=model_size # Bytes
d['##features#gpu_freq#min']=''
d['##features#cpu_freq#min']=1416
d['##features#freq#min']=d['##features#cpu_freq#min']
d['##features#processed#min']='yes'
# Add throughput (images/second)
tall=d.get('##characteristics#run#execution_time_classify#all',[]) # It's internal VTA measurements
if len(tall)>0:
tnew=[]
for t in tall:
t1=1/t
tnew.append(t1)
r=ck.access({'action':'stat_analysis',
'module_uoa':'experiment',
'dict':d,
'dict1':{'##characteristics#run#inference_throughput':tnew}
})
if r['return']>0: return r
# Unify batch size
batch=1 # for now only 1 is supported in this artifact
d['##features#batch_size#min']=batch
# inference latency
d['##features#measuring_latency#min']='yes'
r=ck.access({'action':'stat_analysis',
'module_uoa':'experiment',
'dict':d,
'dict1':{'##characteristics#run#inference_latency':tall}
})
if r['return']>0: return r
r=ck.access({'action':'stat_analysis',
'module_uoa':'experiment',
'dict':d,
'dict1':{'##characteristics#run#prediction_time_avg_s':tall}
})
if r['return']>0: return r
# Add accuracy (was calculated through separate experiment)
r=ck.access({'action':'stat_analysis',
'module_uoa':'experiment',
'dict':d,
'dict1':{'##characteristics#run#accuracy_top1':[accuracy_top1]}
})
if r['return']>0: return r
# Add accuracy (was calculated through separate experiment)
r=ck.access({'action':'stat_analysis',
'module_uoa':'experiment',
'dict':d,
'dict1':{'##characteristics#run#accuracy_top5':[accuracy_top5]}
})
if r['return']>0: return r
# Save updated dict
r=ck.save_json_to_file({'json_file':p1, 'dict':d, 'sort_keys':'yes'})
if r['return']>0: return r
return {'return':0}
def do(i, arg):
# Detect basic platform info.
ii={'action':'detect',
'module_uoa':'platform',
'out':'out'}
r=ck.access(ii)
if r['return']>0: return r
# Host and target OS params.
hos=r['host_os_uoa']
hosd=r['host_os_dict']
tos=r['os_uoa']
tosd=r['os_dict']
tdid=r['device_id']
# Program and command.
program='caffe'
cmd_key='test_gpu'
# Load Caffe program meta and desc to check deps.
ii={'action':'load',
'module_uoa':'program',
'data_uoa':program}
rx=ck.access(ii)
if rx['return']>0: return rx
mm=rx['dict']
# Get compile-time and run-time deps.
cdeps=mm.get('compile_deps',{})
rdeps=mm.get('run_deps',{})
# Merge rdeps with cdeps for setting up the pipeline (which uses
# common deps), but tag them as "for_run_time".
for k in rdeps:
cdeps[k]=rdeps[k]
cdeps[k]['for_run_time']='yes'
# Caffe libs.
depl=copy.deepcopy(cdeps['lib-caffe'])
if (arg.tos is not None) and (arg.did is not None):
tos=arg.tos
tdid=arg.did
ii={'action':'resolve',
'module_uoa':'env',
'host_os':hos,
'target_os':tos,
'device_id':tdid,
'out':'con',
'deps':{'lib-caffe':copy.deepcopy(depl)}
}
r=ck.access(ii)
if r['return']>0: return r
udepl=r['deps']['lib-caffe'].get('choices',[]) # All UOAs of env for Caffe libs.
if len(udepl)==0:
return {'return':1, 'error':'no installed Caffe libs'}
# Caffe models.
depm=copy.deepcopy(cdeps['caffemodel'])
ii={'action':'resolve',
'module_uoa':'env',
'host_os':hos,
'target_os':tos,
'device_id':tdid,
'out':'con',
'deps':{'caffemodel':copy.deepcopy(depm)}
}
r=ck.access(ii)
if r['return']>0: return r
udepm=r['deps']['caffemodel'].get('choices',[]) # All UOAs of env for Caffe models.
if len(udepm)==0:
return {'return':1, 'error':'no installed Caffe models'}
# ImageNet LMDBs.
depimg=copy.deepcopy(cdeps['dataset-imagenet-lmdb'])
ii={'action':'resolve',
'module_uoa':'env',
'host_os':hos,
'target_os':tos,
'device_id':tdid,
'out':'con',
'deps':{'dataset-imagenet-lmdb':copy.deepcopy(depimg)}
}
r=ck.access(ii)
if r['return']>0: return r
c = r['deps']
udepi=r['deps']['dataset-imagenet-lmdb'].get('choices',[]) # All UOAs of env for ImageNet LMDBs.
if len(udepi)==0:
return {'return':1, 'error':'no installed ImageNet LMDB'}
# Prepare pipeline.
cdeps['lib-caffe']['uoa']=udepl[0]
cdeps['caffemodel']['uoa']=udepm[0]
cdeps['dataset-imagenet-lmdb']['uoa']=udepi[0]
ii={'action':'pipeline',
'prepare':'yes',
'dependencies':cdeps,
'module_uoa':'program',
'data_uoa':program,
'cmd_key':cmd_key,
'target_os':tos,
'device_id':tdid,
'no_state_check':'yes',
'no_compiler_description':'yes',
'skip_calibration':'yes',
'env':{
'CK_CAFFE_BATCH_SIZE':2,
'CK_CAFFE_ITERATIONS':1,
'OPENBLAS_NUM_THREADS':4
},
'cpu_freq':'max',
'gpu_freq':'max',
'flags':'-O3',
'speed':'no',
'energy':'no',
'skip_print_timers':'yes',
'out':'con'
}
r=ck.access(ii)
if r['return']>0: return r
fail=r.get('fail','')
if fail=='yes':
return {'return':10, 'error':'pipeline failed ('+r.get('fail_reason','')+')'}
ready=r.get('ready','')
if ready!='yes':
return {'return':11, 'error':'pipeline not ready'}
state=r['state']
tmp_dir=state['tmp_dir']
# Remember resolved deps for this benchmarking session.
xcdeps=r.get('dependencies',{})
# Clean pipeline.
if 'ready' in r: del(r['ready'])
if 'fail' in r: del(r['fail'])
if 'return' in r: del(r['return'])
pipeline=copy.deepcopy(r)
# For each Caffe lib.*******************************************************
for lib_uoa in udepl:
# Load Caffe lib.
ii={'action':'load',
'module_uoa':'env',
'data_uoa':lib_uoa}
r=ck.access(ii)
if r['return']>0: return r
# Get the tags from e.g. 'BVLC Caffe framework (libdnn,viennacl)'
lib_name=r['data_name']
lib_tags=re.match('BVLC Caffe framework \((?P<tags>.*)\)', lib_name)
lib_tags=lib_tags.group('tags').replace(' ', '').replace(',', '-')
# Skip some libs with "in [..]" or "not in [..]".
if lib_tags not in ['cudnn']: continue
skip_compile='no'
# For each Caffe model.*************************************************
for model_uoa in udepm:
# Load Caffe model.
ii={'action':'load',
'module_uoa':'env',
'data_uoa':model_uoa}
r=ck.access(ii)
if r['return']>0: return r
# Get the tags from e.g. 'Caffe model (net and weights) (deepscale, squeezenet, 1.1)'
model_name=r['data_name']
model_tags=re.match('Caffe model \(net and weights\) \((?P<tags>.*)\)', model_name)
model_tags=model_tags.group('tags').replace(' ', '').replace(',', '-')
# Skip some models with "in [..]" or "not in [..]".
if model_tags not in ['bvlc-alexnet', 'bvlc-googlenet', 'deepscale-squeezenet-1.1']: continue
# For each ImageNet LMDB.*******************************************
for img_uoa in udepi:
ii={'action':'load',
'module_uoa':'env',
'data_uoa':img_uoa}
r=ck.access(ii)
if r['return']>0: return r
img_name=r['data_name']
img_tags=r.get('dict',{}).get('tags',[])
if model_tags in img_tags: break
# Skip models having no compatible LMDBs.
if model_tags not in img_tags: continue
resize_tags = [ tag for tag in img_tags if tag.find('resize-')!=-1 ]
resize_tag = resize_tags[0] if resize_tags else 'resize-unknown'
img_tags = 'imagenet-val-lmdb-'+resize_tag
record_repo='local'
record_uoa=model_tags+'-'+lib_tags+'-'+img_tags
# Prepare pipeline.
ck.out('---------------------------------------------------------------------------------------')
ck.out('%s - %s' % (lib_name, lib_uoa))
ck.out('%s - %s' % (model_name, model_uoa))
ck.out('%s - %s' % (img_name, img_uoa))
ck.out('Experiment - %s:%s' % (record_repo, record_uoa))
# Prepare autotuning input.
cpipeline=copy.deepcopy(pipeline)
# Reset deps and change UOA.
new_deps={'lib-caffe':copy.deepcopy(depl),
'caffemodel':copy.deepcopy(depm),
'dataset-imagenet-lmdb':copy.deepcopy(depimg)}
### maybe here exit(1)
new_deps['lib-caffe']['uoa']=lib_uoa
new_deps['caffemodel']['uoa']=model_uoa
new_deps['dataset-imagenet-lmdb']['uoa']=img_uoa
jj={'action':'resolve',
'module_uoa':'env',
'host_os':hos,
'target_os':tos,
'device_id':tdid,
'deps':new_deps}
r=ck.access(jj)
if r['return']>0: return r
cpipeline['dependencies'].update(new_deps)
cpipeline['no_clean']=skip_compile
cpipeline['no_compile']=skip_compile
cpipeline['cmd_key']=cmd_key
ii={'action':'autotune',
'module_uoa':'pipeline',
'data_uoa':'program',
'iterations':1,
'repetitions':1,
'record':'yes',
'record_failed':'yes',
'record_params':{
'search_point_by_features':'yes'
},
'record_repo':record_repo,
'record_uoa':record_uoa,
'tags':['test-accuracy', cmd_key, model_tags, lib_tags, img_tags, platform_tags ],
'pipeline':cpipeline,
'out':'con'}
r=ck.access(ii)
if r['return']>0: return r
fail=r.get('fail','')
if fail=='yes':
return {'return':10, 'error':'pipeline failed ('+r.get('fail_reason','')+')'}
skip_compile='yes'
return {'return':0}
def do(i, arg):
# Detect basic platform info.
ii = {'action': 'detect', 'module_uoa': 'platform', 'out': 'out'}
r = ck.access(ii)
if r['return'] > 0: return r
# Host and target OS params.
hos = r['host_os_uoa']
hosd = r['host_os_dict']
tos = r['os_uoa']
tosd = r['os_dict']
tdid = r['device_id']
# Program and command.
program = 'image-classification-tf-py'
cmd_key = 'default'
# Load program meta and description to check deps.
ii = {'action': 'load', 'module_uoa': 'program', 'data_uoa': program}
rx = ck.access(ii)
if rx['return'] > 0: return rx
mm = rx['dict']
# Get compile-time and run-time deps.
cdeps = mm.get('compile_deps', {})
rdeps = mm.get('run_deps', {})
# Merge rdeps with cdeps for setting up the pipeline (which uses
# common deps), but tag them as "for_run_time".
for k in rdeps:
cdeps[k] = rdeps[k]
cdeps[k]['for_run_time'] = 'yes'
# Tensorflow libs.
depl = copy.deepcopy(cdeps['lib-tensorflow'])
if (arg.tos is not None) and (arg.did is not None):
tos = arg.tos
tdid = arg.did
ii = {
'action': 'resolve',
'module_uoa': 'env',
'host_os': hos,
'target_os': tos,
'device_id': tdid,
'out': 'con',
'deps': {
'lib-tensorflow': copy.deepcopy(depl)
},
'quiet': 'yes'
}
r = ck.access(ii)
if r['return'] > 0: return r
udepl = r['deps']['lib-tensorflow'].get(
'choices', []) # All UOAs of env for Tensorflow libs.
if len(udepl) == 0:
return {'return': 1, 'error': 'no installed Tensorflow libs'}
# Tensorflow models.
depm = copy.deepcopy(cdeps['model-and-weights'])
ii = {
'action': 'resolve',
'module_uoa': 'env',
'host_os': hos,
'target_os': tos,
'device_id': tdid,
'out': 'con',
'deps': {
'tensorflow_model': copy.deepcopy(depm)
},
'quiet': 'yes'
}
r = ck.access(ii)
if r['return'] > 0: return r
udepm = r['deps']['tensorflow_model'].get(
'choices', []) # All UOAs of env for Tensorflow models.
if len(udepm) == 0:
return {'return': 1, 'error': 'no installed TensorFlow models'}
# Load dataset path.
# FIXME: Does not have to be ImageNet val.
ii = {
'action': 'show',
'module_uoa': 'env',
'tags': 'dataset,imagenet,val,raw'
}
rx = ck.access(ii)
if len(rx['lst']) == 0: return rx
# FIXME: Can also be 'CK_ENV_DATASET_IMAGE_DIR'.
img_dir = rx['lst'][0]['meta']['env']['CK_ENV_DATASET_IMAGENET_VAL']
# Prepare pipeline.
cdeps['lib-tensorflow']['uoa'] = udepl[0]
cdeps['model-and-weights']['uoa'] = udepm[0]
ii = {
'action': 'pipeline',
'prepare': 'yes',
'dependencies': cdeps,
'module_uoa': 'program',
'data_uoa': program,
'cmd_key': cmd_key,
'target_os': tos,
'device_id': tdid,
'no_state_check': 'yes',
'no_compiler_description': 'yes',
'skip_calibration': 'yes',
'env': {
'CK_ENV_DATASET_IMAGE_DIR': img_dir,
'CK_BATCH_COUNT': num_batches
},
'cpu_freq': 'max',
'gpu_freq': 'max',
'flags': '-O3',
'speed': 'no',
'energy': 'no',
'skip_print_timers': 'yes',
'out': 'con'
}
r = ck.access(ii)
if r['return'] > 0: return r
fail = r.get('fail', '')
if fail == 'yes':
return {
'return': 10,
'error': 'pipeline failed (' + r.get('fail_reason', '') + ')'
}
ready = r.get('ready', '')
if ready != 'yes':
return {'return': 11, 'error': 'pipeline not ready'}
state = r['state']
tmp_dir = state['tmp_dir']
# Remember resolved deps for this benchmarking session.
xcdeps = r.get('dependencies', {})
# Clean pipeline.
if 'ready' in r: del (r['ready'])
if 'fail' in r: del (r['fail'])
if 'return' in r: del (r['return'])
pipeline = copy.deepcopy(r)
# For each Tensorflow lib.*******************************************************
for lib_uoa in udepl:
# Load Tensorflow lib.
ii = {'action': 'load', 'module_uoa': 'env', 'data_uoa': lib_uoa}
r = ck.access(ii)
if r['return'] > 0: return r
# Get the tags from e.g. 'TensorFlow library (from sources, cuda)'
lib_name = r['data_name']
lib_tags = re.match('TensorFlow library \((?P<tags>.*)\)', lib_name)
lib_tags = lib_tags.group('tags').replace(' ', '').replace(',', '-')
# Skip some libs with "in [..]" or "not in [..]".
if lib_tags not in [
'prebuilt-cuda', 'fromsources-cuda', 'fromsources-cuda-xla',
'prebuilt-cpu', 'fromsources-cpu', 'fromsources-cpu-xla'
]:
continue
cmd_keys = ['default']
gpu_memory_pc = [33]
# For each cmd key.*************************************************
for cmd_key in cmd_keys:
# For each TensorFlow model.*************************************************
for model_uoa in udepm:
# Load Tensorflow model.
ii = {
'action': 'load',
'module_uoa': 'env',
'data_uoa': model_uoa
}
r = ck.access(ii)
if r['return'] > 0: return r
# Get the tags from e.g. 'TensorFlow python model and weights (squeezenet)'.
model_name = r['data_name']
model_tags = re.match(
'TensorFlow python model and weights \((?P<tags>.*)\)',
model_name)
model_tags = model_tags.group('tags').replace(' ', '').replace(
',', '-').lower()
# Skip some models with "in [..]" or "not in [..]".
if model_tags not in [
'squeezenet', 'googlenet', 'mobilenet-1.0-224'
]:
continue # 'alexnet'
record_repo = 'local'
record_uoa = model_tags + '-' + lib_tags
# Prepare pipeline.
ck.out(
'---------------------------------------------------------------------------------------'
)
ck.out('%s - %s' % (lib_name, lib_uoa))
ck.out('%s - %s' % (model_name, model_uoa))
ck.out('Experiment - %s:%s' % (record_repo, record_uoa))
# Prepare autotuning input.
cpipeline = copy.deepcopy(pipeline)
# Reset deps and change UOA.
new_deps = {
'lib-tensorflow': copy.deepcopy(depl),
'squeezedet': copy.deepcopy(depm)
}
new_deps['lib-tensorflow']['uoa'] = lib_uoa
new_deps['squeezedet']['uoa'] = model_uoa
jj = {
'action': 'resolve',
'module_uoa': 'env',
'host_os': hos,
'target_os': tos,
'device_id': tdid,
'deps': new_deps
}
r = ck.access(jj)
if r['return'] > 0: return r
cpipeline['dependencies'].update(new_deps)
cpipeline['cmd_key'] = cmd_key
ii = {
'action':
'autotune',
'module_uoa':
'pipeline',
'data_uoa':
'program',
'choices_order':
[['##choices#env#CK_TF_GPU_MEMORY_PERCENT'],
['##choices#env#CK_BATCH_SIZE']],
'choices_selection': [
{
'type': 'loop',
'choice': gpu_memory_pc
},
{
'type': 'loop',
'start': bs['start'],
'stop': bs['stop'],
'step': bs['step'],
'default': bs['default']
},
],
'features_keys_to_process': ['##choices#*'],
'iterations':
-1,
'repetitions':
num_repetitions,
'record':
'yes',
'record_failed':
'yes',
'record_params': {
'search_point_by_features': 'yes'
},
'record_repo':
record_repo,
'record_uoa':
record_uoa,
'tags': [
'explore-batch-size-libs-models', model_tags, lib_tags,
platform_tags
],
'pipeline':
cpipeline,
'out':
'con'
}
r = ck.access(ii)
if r['return'] > 0: return r
fail = r.get('fail', '')
if fail == 'yes':
return {
'return':
10,
'error':
'pipeline failed (' + r.get('fail_reason', '') + ')'
}
return {'return': 0}
def do(i, arg):
if arg.fp is not None:
fin = arg.fp
if (os.path.isfile(fin)):
print("File loading %s " % (fin))
#LOAD FILE and TRIPLES
else:
print("File %s not found " % (fin))
#Load Models Matrixes
alex_net = loadMatrixFromJson(alex_net_json)
google_net = loadMatrixFromJson(google_net_json)
squeeze11_net = loadMatrixFromJson(squeeze11_net_json)
dataset = []
dataset.append({
'Model_name': 'AlexNet',
'Batch_size': 1,
'matrix': alex_net
})
dataset.append({
'Model_name': 'GoogleNet',
'Batch_size': 1,
'matrix': google_net
})
dataset.append({
'Model_name': 'SqueezeNet1.1',
'Batch_size': 1,
'matrix': squeeze11_net
})
if VERBOSE or DEBUG:
print('[Experiment] %s' % title)
print('[Preparing pipeline] Clock resolution: %d' % clock_resolution)
#print('[Preparing pipeline] Matrix sizes: m=%s, k=%s, n=%s: ' % (size_m, size_k, size_n))
print('[Preparing pipeline] Precision: %d' % precision)
print('[Preparing pipeline] Run for configuration: %d' % run)
print('[Preparing pipeline] More parms... ')
#ntrip = len(size_m)
#print ('[Experiment] Number of triple(s) %s' % (ntrip))
#size_tag = ''
#for tp in range (0, ntrip):
# if (tp == ntrip-1):
# size_tag += str((int(size_m[tp])*int(size_n[tp])*int(size_k[tp])))
# else:
# size_tag += str((int(size_m[tp])*int(size_n[tp])*int(size_k[tp])))+','
# Detect basic platform info.
ii = {'action': 'detect', 'module_uoa': 'platform', 'con': 'con'}
r = ck.access(ii)
if DEBUG: print("%s %s" % (DEBUG_STR, r))
if r['return'] > 0: return r
# Host and target OS params.
hos = r['host_os_uoa']
hosd = r['host_os_dict']
tos = r['os_uoa']
tosd = r['os_dict']
tdid = r['device_id']
if DEBUG: print("%s %s %s" % (DEBUG_STR, hos, hosd))
if DEBUG: print("%s %s %s %s" % (DEBUG_STR, tos, tosd, tdid))
# Load CLBLAST program meta and desc to check deps.
ii = {
'action': 'load',
'module_uoa': 'program',
'data_uoa': 'clblast-tune'
}
rx = ck.access(ii)
if DEBUG: print("%s %s " % (DEBUG_STR, rx))
if rx['return'] > 0: return rx
meta = rx['dict']
# Get compile-time and run-time deps.
cdeps = meta.get('compile_deps', {})
rdeps = meta.get('run_deps', {})
# Merge rdeps with cdeps for setting up the pipeline (which uses
# common deps), but tag them as "for_run_time".
for k in rdeps:
cdeps[k] = rdeps[k]
cdeps[k]['for_run_time'] = 'yes'
# CLblast libs.
depl = copy.deepcopy(cdeps['lib-clblast'])
#ON LOCAL MACHINE
if ((arg.tos is not None) and (arg.did is not None)):
tos = arg.tos
tdid = arg.did
ii = {
'action': 'resolve',
'module_uoa': 'env',
'host_os': hos,
'target_os': tos,
'device_id': tdid,
'out': 'con',
'deps': {
'lib-clblast': copy.deepcopy(depl)
}
}
r = ck.access(ii)
if r['return'] > 0: return r
udepl = r['deps']['lib-clblast'].get('choices', [])
if len(udepl) == 0:
return {'return': 1, 'error': 'no installed CLBlast libs'}
cdeps['lib-clblast']['uoa'] = udepl[0]
#prepare pipeline
ii = {
'action': 'pipeline',
'module_uoa': 'program',
'data_uoa': 'clblast-tune',
'prepare': 'yes',
'dependencies': cdeps,
'no_compiler_description': 'yes',
'cmd_key': kernel[0],
"target_os": tos,
"device_id": tdid,
"out": 'con',
"no_state_check": "yes",
'flags': '-O3',
'cpu_freq': 'max',
'gpu_freq': 'max',
}
r = ck.access(ii)
if r['return'] > 0: return r
fail = r.get('fail', '')
if fail == 'yes':
return {
'return': 10,
'error': 'pipeline failed (' + r.get('fail_reason', '') + ')'
}
ready = r.get('ready', '')
if ready != 'yes': return {'return': 11, 'error': 'pipeline not ready'}
state = r['state']
tmp_dir = state['tmp_dir']
xcdeps = r.get('dependencies', {})
# Clean pipeline.
if 'ready' in r: del (r['ready'])
if 'fail' in r: del (r['fail'])
if 'return' in r: del (r['return'])
pipeline = copy.deepcopy(r)
print udepl
# For each Clblast lib ***********************************
for lib_uoa in udepl:
print lib_uoa
ii = {'action': 'load', 'module_uoa': 'env', 'data_uoa': lib_uoa}
r = ck.access(ii)
if r['return'] > 0: return r
lib_name = r['data_name']
#lib_tags=re.match('BVLC Caffe framework \((?P<tags>.*)\)', lib_name)
#lib_tags=r['dict']['tags']
lib_tags = r['dict']['customize']['used_package_uoa'].split('-')
tags = ''
skip_tags = ['lib', 'master', 'universal']
for t in lib_tags:
if t not in skip_tags:
tags += t + '-'
# Add the extra_tags (if any)
lib_tags = r['dict']['setup']['version']
tags += lib_tags
# For each model in dataset
for model in dataset:
record_repo = 'local'
record_uoa = 'explore-matrix-size-' + tags + '-' + model[
'Model_name']
ck.out(
'---------------------------------------------------------------------------------------'
)
ck.out('Experiment - %s:%s' % (record_repo, record_uoa))
cpipeline = copy.deepcopy(pipeline)
ii = {
'action':
'autotune',
'module_uoa':
'pipeline',
'data_uoa':
'program',
'choices_order': [['##env#CK_CLBLAST_MSIZE'],
[
'##env#CK_CLBLAST_NSIZE',
], ['##env#CK_CLBLAST_KSIZE']],
'choices_selection': [{
"type": "loop-with-next",
"choice": model['matrix']['M'],
"default": "256"
}, {
"type": "loop-with-next",
"choice": model['matrix']['N'],
"default": "256"
}, {
"type": "loop-with-next",
"choice": model['matrix']['K'],
"default": "256"
}],
'features_keys_to_process': ['##choices#*'],
'iterations':
-1,
'repetitions':
3,
'record':
'yes',
'record_failed':
'yes',
'record_params': {
'search_point_by_features': 'yes'
},
'record_repo':
record_repo,
'record_uoa':
record_uoa,
'tags': [
'explore-clblast-matrix-size-client', tags,
model['Model_name']
],
'pipeline':
cpipeline,
'out':
'con'
}
r = ck.access(ii)
if DEBUG > 0: print("%s %s" % (DEBUG_STR, r))
if r['return'] > 0: return r
fail = r.get('fail', '')
if fail == 'yes':
return {
'return': 10,
'error':
'pipeline failed (' + r.get('fail_reason', '') + ')'
}
return {'return': 0}