def main(i):
# Load common table file (for all models)
ddd = {}
r = ck.load_json_file({'json_file': 'save_all_model_data_tmp.json'})
if r['return'] == 0:
ddd = r['dict']
# Searching for features
for gi in range(0, len(gcc)):
g = gcc[gi]
depth = best_depth[gi]
ck.out(
'********************************************************************************'
)
ck.out('Modeling optimizations for ' + g)
ck.out('')
if g not in ddd: ddd[g] = {}
gx = g.replace(' ', '_')
gx2 = g.replace(' ', '-')
r = ck.load_json_file(
{'json_file': 'prepare_train_data_tmp.' + gx + '.json'})
if r['return'] > 0: ck.err(r)
d = r['dict']
ftable = d['ftable']
ctable = d['ctable']
s = '==============================================================\n'
s += 'Depth: ' + str(depth) + '\n\n'
ck.out(s)
# Building decision tree on all data
ck_model_entry_name = "rpi3-milepost-model-" + gx2.lower()
ii = {
'action': 'build',
'module_uoa': 'model',
'ftable': ftable,
'ctable': ctable,
'keep_temp_files': 'yes',
"model_module_uoa": "model.sklearn",
"model_name": "dtc",
"model_file": "tmp-model-sklearn-dtc",
"model_params": {
"max_depth": depth
},
"model_repo_uoa": "ck-rpi-optimization-results",
"model_data_uoa": ck_model_entry_name,
"out": ""
}
# Training
cii = copy.deepcopy(ii)
r = ck.access(ii)
if r['return'] > 0: ck.err(r)
# Validating
ii = copy.deepcopy(ii)
ii['action'] = 'validate'
r = ck.access(ii)
if r['return'] > 0: ck.err(r)
obs = r['observations']
wrong = r['mispredictions']
acc = float(obs - wrong) / float(obs)
x = ' Accuracy on all data: ' + str(acc)
s += x
ck.out(x)
# Record example of features to demo predictions (to be integrated with compiler optimization prediction (web)services)
d = {
"action": "use",
"module_uoa": "model",
"features": ftable[123], # features of some random benchmark
"model_module_uoa": "model.sklearn",
"model_name": "dtc",
"model_file": "tmp-model-sklearn-dtc",
"model_data_uoa": ck_model_entry_name
}
r = ck.save_json_to_file({
'json_file':
'process_model_using_decision_trees_and_record_to_ck_use.' + gx +
'.json',
'dict':
d
})
if r['return'] > 0: return r
return {'return': 0}
def do(i):
# List performance entries
r=ck.access({'action':'search',
'module_uoa':'experiment',
'data_uoa':'ck-request-asplos18-mxnet-arm-performance*'
# 'repo_uoa':'ck-request-asplos18-results'
})
if r['return']>0: return r
lst=r['lst']
for q in lst:
duid=q['data_uid']
duoa=q['data_uoa']
ruid=q['repo_uid']
path=q['path']
ck.out(duoa)
# Search matching accuracy entry
r=ck.access({'action':'load',
'module_uoa':'experiment',
'data_uoa':duid,
'repo_uoa':ruid})
if r['return']>0: return r
dd=r['dict']
ruid=r['repo_uid']
apath=r['path']
# Updating meta if needed
dd['meta']['scenario_module_uoa']='a555738be4b65860' # module:request.asplos18
dd['meta']['dataset_species']='ImageNet' # dataset species (free format)
dd['meta']['platform_species']='embedded' # embedded vs server (maybe other classifications such as edge)
dd['meta']['platform_peak_power']=6.05 #Watts http://opensource.rock-chips.com/images/6/60/Rockchip_RK3399_Datasheet_V1.6-20170301.pdf last page
dd['meta']['platform_price']=149 # $, http://shop.t-firefly.com/goods.php?id=45
dd['meta']['platform_price_date']='20180416' # date
dd['meta']['artifact']='08da9685582866a0' # artifact description
dd['meta']['model_precision']='fp32'
dd['meta']['processed']='yes'
# Unified full name for some deps
ds=dd['meta']['deps_summary']
x=ds['mxnet-model']
r=ck.access({'action':'make_deps_full_name','module_uoa':'request.asplos18','deps':x})
if r['return']>0: return r
full_model_name=r['full_name']
dd['meta']['model_design_name']=full_model_name
# for simplicity add manually (can later automate it as in other artifacts, but just didn't have time here)
if 'mobilenet' in full_model_name:
dd['meta']['model_species']='07d4e7aa3750ddc6' # model.species:resnet18
dd['meta']['dataset_size']=50000 # number of images ...
accuracy_top1=0.66694
accuracy_top5=0.87734
elif 'resnet' in full_model_name:
dd['meta']['model_species']='d41bbf1e489ab5e0' # model.species:resnet18
dd['meta']['dataset_size']=25000 # number of images ...
accuracy_top1=0.61318
accuracy_top5=0.83702
elif 'vgg16' in full_model_name:
dd['meta']['model_species']='a3fcac86d42bdbc4' # model.species:resnet18
dd['meta']['dataset_size']=5000 # number of images ...
accuracy_top1=0.63120
accuracy_top5=0.84951
else:
return {'return':1, 'error':'unknown model ('+y+')'}
x=ds['lib-mxnet']
r=ck.access({'action':'make_deps_full_name','module_uoa':'request.asplos18','deps':x})
if r['return']>0: return r
dd['meta']['library_name']=r['full_name']
x=x['deps']['compiler']
r=ck.access({'action':'make_deps_full_name','module_uoa':'request.asplos18','deps':x})
if r['return']>0: return r
dd['meta']['compiler_name']=r['full_name']
# Updating entry
r=ck.access({'action':'update',
'module_uoa':'experiment',
'data_uoa':duid,
'repo_uoa':ruid,
'dict':dd,
'substitute':'yes',
'ignore_update':'yes',
'sort_keys':'yes'
})
if r['return']>0: return r
# Checking points to aggregate
os.chdir(path)
dperf=os.listdir(path)
for f in dperf:
if f.endswith('.cache.json'):
os.system('git rm -f '+f)
elif f.endswith('.flat.json'):
ck.out(' * '+f)
# Load performance file
p1=os.path.join(path, f)
r=ck.load_json_file({'json_file':p1})
if r['return']>0: return r
d1=r['dict']
# Prune some old value
d={}
for k in d1:
if not k.startswith('##characteristics#run#accuracy_top1') and \
not k.startswith('##characteristics#run#accuracy_top5') and \
not k.startswith('##characteristics#run#inference_throughput') and \
not k.startswith('##characteristics#run#inference_latency'):
d[k]=d1[k]
# for simplicity add manually (can later automate it as in other artifacts, but just didn't have time here)
if 'mobilenet' in full_model_name:
model_size=17024109
elif 'resnet' in full_model_name:
model_size=46803089
elif 'vgg16' in full_model_name:
model_size=553432060
else:
return {'return':1, 'error':'unknown model ('+y+')'}
d['##features#model_size#min']=model_size # Bytes
d['##features#gpu_freq#min']=''
d['##features#cpu_freq#min']=1416
d['##features#freq#min']=d['##features#cpu_freq#min']
d['##features#processed#min']='yes'
# Add throughput (images/second)
tall=d.get('##characteristics#run#execution_time_classify#all',[]) # It's internal VTA measurements
if len(tall)>0:
tnew=[]
for t in tall:
t1=1/t
tnew.append(t1)
r=ck.access({'action':'stat_analysis',
'module_uoa':'experiment',
'dict':d,
'dict1':{'##characteristics#run#inference_throughput':tnew}
})
if r['return']>0: return r
# Unify batch size
batch=1 # for now only 1 is supported in this artifact
d['##features#batch_size#min']=batch
# inference latency
d['##features#measuring_latency#min']='yes'
r=ck.access({'action':'stat_analysis',
'module_uoa':'experiment',
'dict':d,
'dict1':{'##characteristics#run#inference_latency':tall}
})
if r['return']>0: return r
r=ck.access({'action':'stat_analysis',
'module_uoa':'experiment',
'dict':d,
'dict1':{'##characteristics#run#prediction_time_avg_s':tall}
})
if r['return']>0: return r
# Add accuracy (was calculated through separate experiment)
r=ck.access({'action':'stat_analysis',
'module_uoa':'experiment',
'dict':d,
'dict1':{'##characteristics#run#accuracy_top1':[accuracy_top1]}
})
if r['return']>0: return r
# Add accuracy (was calculated through separate experiment)
r=ck.access({'action':'stat_analysis',
'module_uoa':'experiment',
'dict':d,
'dict1':{'##characteristics#run#accuracy_top5':[accuracy_top5]}
})
if r['return']>0: return r
# Save updated dict
r=ck.save_json_to_file({'json_file':p1, 'dict':d, 'sort_keys':'yes'})
if r['return']>0: return r
return {'return':0}
def main(i):
# Load common table file (for all models)
ddd = {}
r = ck.load_json_file({'json_file': 'save_all_model_data_tmp.json'})
if r['return'] == 0:
ddd = r['dict']
# Searching for features
for g in gcc:
ck.out(
'********************************************************************************'
)
ck.out('Preparing accuracy graph for ' + g)
ck.out('')
gx = g.replace(' ', '_')
d = ddd[g]
table = {"0": [], "1": []}
for depth in range(1, 30):
key1 = 'decision_trees_with_cross_validation_depth_' + str(
depth) + '_ft1_ft65'
key2 = 'decision_trees_without_cross_validation_depth_' + str(
depth) + '_ft1_ft65'
acc1 = d[key1]
acc2 = d[key2]
table["0"].append([depth, acc1])
table["1"].append([depth, acc2])
# Graph input
ii = {
"action":
"plot",
"module_uoa":
"graph",
"table":
table,
"add_x_loop":
"no",
"ignore_point_if_none":
"yes",
"plot_type":
"mpl_2d_scatter",
"display_y_error_bar":
"no",
"title":
"Powered by Collective Knowledge",
"axis_x_desc":
"Decision tree depth",
"axis_y_desc":
"Model accuracy for " + g + " (%)",
"plot_grid":
"yes",
"mpl_image_size_x":
"12",
"mpl_image_size_y":
"6",
"mpl_image_dpi":
"100",
"font_size":
22,
"out_to_file":
'process_model_using_decision_trees_accuracy_graph_output.' + gx +
'.pdf',
"point_style": {
"0": {
"marker": "o"
},
"1": {
"marker": "x"
}
}
}
# Save common data (continuously, not to loose data)
r = ck.save_json_to_file({
'json_file':
'process_model_using_decision_trees_accuracy_graph_input.' + gx +
'.json',
'dict':
ii
})
if r['return'] > 0: return r
# Plot graph (save to pdf)
r = ck.access(ii)
if r['return'] > 0: return r
return {'return': 0}
def run(i):
# Get path1
r=ck.access({'action':'load',
'module_uoa':'experiment',
'data_uoa':euoa1})
if r['return']>0: return r
p1=r['path']
# Get path1
r=ck.access({'action':'load',
'module_uoa':'experiment',
'data_uoa':euoa2})
if r['return']>0: return r
p2=r['path']
# Going through points in path 2
dirList1=os.listdir(p1)
dirList2=os.listdir(p2)
for fn2 in dirList2:
if fn2.endswith('.features.json'):
ck.out('Loading point '+fn2+' ...')
px2=os.path.join(p2, fn2)
r=ck.load_json_file({'json_file':px2})
if r['return']>0: return r
df2=r['dict'].get('features',{})
# Searching in p1
found=False
for fn1 in dirList1:
if fn1.endswith('.features.json'):
px1=os.path.join(p1, fn1)
r=ck.load_json_file({'json_file':px1})
if r['return']>0: return r
df1=r['dict'].get('features',{})
rx=ck.compare_dicts({'dict1':df1, 'dict2':df2})
if rx['return']>0: return rx
equal=rx['equal']
if equal=='yes':
found=True
break
if found:
ck.out(' Found!')
else:
# Removing point
ck.out(' Removing point ...')
fn=fn2[:-14]
for fn0 in dirList2:
if fn0.startswith(fn):
p0=os.path.join(p2,fn0)
os.remove(p0)
return {'return':0}
def main(i):
# Load common table file (for all models)
ddd = {}
r = ck.load_json_file({'json_file': 'save_all_model_data_tmp.json'})
if r['return'] == 0:
ddd = r['dict']
# Searching for features
for g in gcc:
random.seed(12345)
ck.out(
'********************************************************************************'
)
ck.out('Modeling optimizations for ' + g)
ck.out('')
if g not in ddd: ddd[g] = {}
gx = g.replace(' ', '_')
r = ck.load_json_file(
{'json_file': 'prepare_train_data_tmp.' + gx + '.json'})
if r['return'] > 0: ck.err(r)
d = r['dict']
ftable = d['ftable']
ctable = d['ctable']
s = ''
for iteration in range(1, 10):
start_time = time.time()
# Full cross-validation
acc = 0
obs = 0
wrong = 0
acc_min = None
acc_max = None
s = '==============================================================\n'
s += 'Iteration: ' + str(iteration) + '\n\n'
if iteration == 1:
# Default params from TF example
hu = [10, 20, 10]
ts = 1000
else:
# Generate random DNN topology and params
hu = []
nhu = random.randint(1, 5)
for k in range(0, nhu):
x = random.randint(10, 30)
hu.append(x)
ts = random.randint(1000, 3000)
s += ' Hidden units: ' + str(hu) + '\n'
s += ' Training steps: ' + str(ts) + '\n\n'
ck.out(s)
for n in range(
0, 3
): # Trying to build model N times (random - sometimes slightly different result)
# Building decision tree on all data
ii = {
'action': 'build',
'module_uoa': 'model',
'ftable': ftable,
'ctable': ctable,
'keep_temp_files': 'yes',
"model_module_uoa": "model.tf",
"model_name": "dnn_classifier",
"model_file": "process_model_using_dnn_tf/" + gx,
"model_params": {
"hidden_units": hu,
"training_steps": ts
},
"out": ""
}
# Training
cii = copy.deepcopy(ii)
r = ck.access(ii)
if r['return'] > 0: ck.err(r)
# Validating
ii = copy.deepcopy(ii)
ii['action'] = 'validate'
r = ck.access(ii)
if r['return'] > 0: ck.err(r)
obs += r['observations']
wrong += r['mispredictions']
acc = float(obs - wrong) / float(obs)
x = '\n Accuracy on all data (' + str(
n + 1) + ' out of 3): ' + str(acc) + '\n'
s += x
ck.out(x)
acc = float(obs - wrong) / float(obs)
if acc_min == None or acc < acc_min:
acc_min = acc
if acc_max == None or acc > acc_max:
acc_max = acc
stop_time = time.time() - start_time
x = '\n\nIteration: ' + str(
iteration
) + ' ; accuracy (min/max): ' + '%.2f' % acc_min + ' .. ' + '%.2f' % acc_max + '\n'
x = '\n Elapsed time: ' + '%.1f' % stop_time + ' sec.\n'
s += x
ck.out(x)
# Cross-validating (for simplicity 1 run)
cross_obs = 0
cross_wrong = 0
x = ' *************************************************\n'
x += ' Cross-validating model (leave one out)\n\n'
s += x
ck.out(x)
for bench in range(0, len(ftable)):
train_ftable = []
train_ctable = []
test_ftable = []
test_ctable = []
for k in range(0, len(ftable)):
if k != bench:
train_ftable.append(ftable[k])
train_ctable.append(ctable[k])
else:
test_ftable.append(ftable[k])
test_ctable.append(ctable[k])
# Selecting model
ii = {
'action':
'build',
'module_uoa':
'model',
'ftable':
train_ftable,
'ctable':
train_ctable,
'keep_temp_files':
'no',
"model_module_uoa":
"model.tf",
"model_name":
"dnn_classifier",
"model_file":
"tmp-model-tf-dnn-classifier-" + gx + '_' + str(iteration),
"model_params": {
"hidden_units": hu,
"training_steps": ts
},
"out":
""
}
# Training
cii = copy.deepcopy(ii)
r = ck.access(ii)
if r['return'] > 0: ck.err(r)
# Validating
ii = copy.deepcopy(ii)
ii['action'] = 'validate'
ii['ftable'] = test_ftable
ii['ctable'] = test_ctable
# ii['ftable']=ftable
# ii['ctable']=ctable
r = ck.access(ii)
if r['return'] > 0: ck.err(r)
cross_obs += r['observations']
cross_wrong += r['mispredictions']
cross_acc = float(cross_obs - cross_wrong) / float(cross_obs)
x = '\n ' + str(bench) + ' out of ' + str(
len(ftable)
) + ' ) current cross-validation accuracy: ' + '%.2f' % cross_acc + '\n'
s += x
ck.out(x)
stop_time = time.time() - start_time
x = '\nIteration: ' + str(
iteration
) + ' ; accuracy (with cross-validation): ' + '%.2f' % cross_acc + '\n'
x = '\n Elapsed time: ' + '%.1f' % stop_time + ' sec.\n'
s += x
ck.out(x)
ddd[g]['dnn_tf_with_cross_validation_iteration_' + str(iteration) +
'_ft1_ft65'] = cross_acc
ddd[g]['dnn_tf_without_cross_validation_iteration_' +
str(iteration) + '_ft1_ft65'] = acc_max
r = ck.save_text_file({
'text_file':
'process_model_using_decision_trees/log.' + gx + '.txt',
'string':
s
})
if r['return'] > 0: return r
# Save common data (continuously, not to loose data)
r = ck.save_json_to_file({
'json_file': 'save_all_model_data_tmp.json',
'dict': ddd
})
if r['return'] > 0: return r
return {'return': 0}
if tp!=None:
ii['mode']=tp
r=ck.access(ii)
if r['return']==0:
shutil.copy(of2,of3)
print ' * converted'
# Record run-time state to html
if os.path.isfile(js):
tt=os.path.getmtime(js)
if tjs==None or tjs!=tt:
tjs=tt
r=ck.load_json_file({'json_file':js})
if r['return']==0:
d=r['dict']
rts=d.get('run_time_state',{})
risx=rts.get("input_size_x",'')
risy=rts.get("input_size_y",'')
rfps=rts.get("run_time_fps",'')
rtt=rts.get("run_time_total",'')
rf=rts.get("frames",'')
rocld=rts.get("opencl_device", '')
roclp=rts.get("opencl_platform",'')
roclu=rts.get("opencl_device_units",'')
def main(i):
# Load data
r = ck.load_json_file(
{'json_file': 'autotune_for_input_aware_blas_fixed.json'})
if r['return'] > 0: return r
ddd = r['dict']
table = {"0": [], "1": []}
# Find RPi3
found = False
for a in ddd.get('all', []):
if a.get('cpu_name', '') == 'BCM2709':
found = True
break
if not found:
return {'return': 1, 'error': 'RPi3 data not found'}
data = a.get('data', {})
for N in sorted(data, key=lambda x: int(x)):
results = data[N]
gflops_default = None
tmin_default = None
gflops_best = None
tmin_best = None
for BS in sorted(results, key=lambda x: int(x)):
x = results[BS]
tmin = x['tmin']
gflops = x['gflops']
if BS == N:
gflops_default = gflops
tmin_default = tmin
table['0'].append([int(N), gflops_default])
else:
if gflops_best == None or gflops > gflops_best:
gflops_best = gflops
tmin_best = tmin
if gflops_best != None:
table['1'].append([int(N), gflops_best])
# Graph input
ii = {
"action": "plot",
"module_uoa": "graph",
"table": table,
"plot_type": "mpl_2d_scatter",
# "title":"Powered by Collective Knowledge",
"axis_x_desc": "Square matrix size",
"axis_y_desc": "GFLOPS",
"plot_grid": "no",
"mpl_image_size_x": "12",
"mpl_image_size_y": "6",
"mpl_image_dpi": "100",
"font_size": 22,
# "out_to_file":'process_data_and_plot_graph_out.pdf',
"point_style": {
"0": {
"marker": "o"
},
"1": {
"marker": "x"
}
}
}
# Save common data (continuously, not to loose data)
r = ck.save_json_to_file({
'json_file': 'process_data_and_plot_graph_out.json',
'dict': ii
})
if r['return'] > 0: return r
return {'return': 0}
def main(i):
cur_dir = os.getcwd()
fas = os.path.join(cur_dir, aggregated_stats)
# Get some info about current platform
ii = {'action': 'detect', 'module_uoa': 'platform', 'out': 'con'}
r = ck.access(ii)
if r['return'] > 0: return r
hos = r['host_os_uid']
hosx = r['host_os_uoa']
hosd = r['host_os_dict']
tos = r['os_uid']
tosx = r['os_uoa']
tosd = r['os_dict']
cpu_name = r['features']['cpu']['name']
plat_name = r['features']['platform']['name']
#############################################################
ck.out(line)
ck.out('CPU name: ' + cpu_name)
ck.out('Plat name: ' + plat_name)
#############################################################
ck.out(line)
ck.out('Loading aggregated stats ...')
aa = []
if os.path.isfile(fas):
r = ck.load_json_file({'json_file': fas})
if r['return'] > 0: return r
ax = r['dict']
if 'all' not in ax: ax['all'] = []
aa = ax['all']
#############################################################
ck.out(line)
ck.out('Finding entry related to this platform ...')
found = False
for a in aa:
if a.get('cpu_name', '') == cpu_name and a.get('plat_name',
'') == plat_name:
found = True
if not found:
a = {'cpu_name': cpu_name, 'plat_name': plat_name}
aa.append(a)
if 'data' not in a: a['data'] = {}
data = a.get('data', {})
# Init pipeline
r = ck.access({
'action': 'pipeline',
'module_uoa': 'program',
'data_uoa': 'shared-matmul-c2',
'cpu_freq': 'max',
'gpu_freq': 'max',
'speed': 'yes',
'compiler_vars': {
'USE_BLOCKED_MATMUL': 'YES'
},
'no_state_check': 'yes',
'prepare': 'yes',
'out': 'con'
})
if r['return'] > 0: return r
ready = r['ready']
if ready != 'yes':
return {'return': 1, 'error': 'can\'t init pipeline'}
pipeline = r
# Compile program ones
tpipeline = copy.deepcopy(pipeline)
r = ck.access({
'action': 'autotune',
'module_uoa': 'pipeline',
'pipeline': pipeline,
'pipeline_update': {
'env': {
'CT_MATRIX_DIMENSION': 16,
'CT_BLOCK_SIZE': 16
}
},
'iterations': 1,
'repetitions': 1,
'out': 'con'
})
if r['return'] > 0: return r
lsa = r.get('last_stat_analysis', {}).get('dict_flat', {})
time_min = lsa.get('##characteristics#run#execution_time#min', None)
if time_min == None or time_min == 0.0:
return {'return': 1, 'error': 'failed to run default pipeline'}
# data is per N size
while True: # continue infinite loop until stopping
ck.out(line)
n = random.randint(0, 3) # Matrix size generator
if n == 0:
N = random.randint(4, 1024) # Matrix size
else:
NX = random.randint(2, 10)
N = 2**NX
if n == 2: N = N - 1
if n == 3: N = N + 1
SN = str(N)
if SN not in data: data[SN] = {}
xdata = data.get(SN, {})
tmin = xdata.get('tmin', None)
tmax = xdata.get('tmax', None)
gmin = xdata.get('gmin', None)
gmax = xdata.get('gmax', None)
best_tile = xdata.get('best_tile', None)
for opts in range(0, 16):
# Choose if random BS or power of two or power of two -+1
if opts == 0:
BS = 1
elif opts == 1:
BS = N
else:
b = random.randint(0, 3)
if b == 0:
BS = random.randint(1, N)
else:
B1 = math.frexp(N)[1] - 1
B2 = random.randint(0, B1)
BS = 2**B2
if b == 2 and BS > 1: BS = BS - 1
elif b == 3 and BS < N - 1: BS = BS + 1
ck.out('Matrix size: ' + str(N))
ck.out('Tile size: ' + str(BS))
# Run pipeline
tpipeline = copy.deepcopy(pipeline)
r = ck.access({
'action': 'autotune',
'module_uoa': 'pipeline',
'pipeline': pipeline,
'pipeline_update': {
'no_compile': 'yes',
'env': {
'CT_MATRIX_DIMENSION': N,
'CT_BLOCK_SIZE': BS
}
},
'iterations': 1,
'repetitions': 3,
'out': 'con'
})
if r['return'] > 0: return r
lsa = r.get('last_stat_analysis', {}).get('dict_flat', {})
time_min = lsa.get('##characteristics#run#execution_time#min',
None)
changed = False
if time_min != None:
ops = 2 * (N * N * N)
if tmin == None or time_min < tmin:
tmin = time_min
best_tile = BS
gmax = 1.0e-9 * ops / tmin
changed = True
if tmax == None or time_min > tmax:
tmax = time_min
gmin = 1.0e-9 * ops / tmax
changed = True
if changed:
xdata['tmin'] = tmin
xdata['tmax'] = tmax
xdata['gmin'] = gmin
xdata['gmax'] = gmax
xdata['best_tile'] = best_tile
if opts == 0:
xdata['tbs1'] = time_min
xdata['gbs1'] = 1.0e-9 * ops / time_min
changed = True
elif opts == 1:
xdata['tbsn'] = time_min
xdata['gbsn'] = 1.0e-9 * ops / time_min
xdata['bsn'] = N
changed = True
if changed:
ck.out(line)
ck.out('Saving aggregated stats ...')
r = ck.save_json_to_file({
'json_file': fas,
'dict': {
'all': aa
},
'sort_keys': 'yes'
})
if r['return'] > 0: return r
#############################################################
ck.out(line)
ck.out('Saving aggregated stats ...')
r = ck.save_json_to_file({
'json_file': fas,
'dict': {
'all': aa
},
'sort_keys': 'yes'
})
if r['return'] > 0: return r
return {'return': 0}
def main(i):
# Load common table file (for all models)
ddd = {}
r = ck.load_json_file({'json_file': 'save_all_model_data_tmp.json'})
if r['return'] == 0:
ddd = r['dict']
# Searching for features
for g in gcc:
ck.out(
'********************************************************************************'
)
ck.out('Modeling optimizations for ' + g)
ck.out('')
if g not in ddd: ddd[g] = {}
gx = g.replace(' ', '_')
r = ck.load_json_file(
{'json_file': 'prepare_train_data_tmp.' + gx + '.json'})
if r['return'] > 0: ck.err(r)
d = r['dict']
ftable = d['ftable']
ctable = d['ctable']
s = ''
for depth in range(1, 30):
# Full cross-validation
acc = 0
obs = 0
wrong = 0
acc_min = None
acc_max = None
s = '==============================================================\n'
s += 'Depth: ' + str(depth) + '\n\n'
ck.out(s)
for n in range(
0, 3
): # Trying to build model N times (random - sometimes slightly different result)
# Building decision tree on all data
ii = {
'action':
'build',
'module_uoa':
'model',
'ftable':
ftable,
'ctable':
ctable,
'keep_temp_files':
'yes',
"model_module_uoa":
"model.sklearn",
"model_name":
"dtc",
"model_file":
"process_model_using_decision_trees/model-sklearn-dtc-" +
gx + '-depth' + str(depth),
"model_params": {
"max_depth": depth
},
"out":
""
}
# Training
cii = copy.deepcopy(ii)
r = ck.access(ii)
if r['return'] > 0: ck.err(r)
# Validating
ii = copy.deepcopy(ii)
ii['action'] = 'validate'
r = ck.access(ii)
if r['return'] > 0: ck.err(r)
obs += r['observations']
wrong += r['mispredictions']
acc = float(obs - wrong) / float(obs)
x = ' Accuracy on all data (' + str(
n + 1) + ' out of 3): ' + str(acc)
s += x
ck.out(x)
acc = float(obs - wrong) / float(obs)
if acc_min == None or acc < acc_min:
acc_min = acc
if acc_max == None or acc > acc_max:
acc_max = acc
x = '\nDepth: ' + str(
depth
) + ' ; accuracy (min/max): ' + '%.2f' % acc_min + ' .. ' + '%.2f' % acc_max + '\n'
s += x
ck.out(x)
# Cross-validating (for simplicity 1 run)
cross_obs = 0
cross_wrong = 0
x = ' *************************************************\n'
x += ' Cross-validating model (leave one out)\n\n'
s += x
ck.out(x)
for bench in range(0, len(ftable)):
train_ftable = []
train_ctable = []
test_ftable = []
test_ctable = []
for k in range(0, len(ftable)):
if k != bench:
train_ftable.append(ftable[k])
train_ctable.append(ctable[k])
else:
test_ftable.append(ftable[k])
test_ctable.append(ctable[k])
# Selecting model
ii = {
'action': 'build',
'module_uoa': 'model',
'ftable': train_ftable,
'ctable': train_ctable,
'keep_temp_files': 'no',
"model_module_uoa": "model.sklearn",
"model_name": "dtc",
"model_file": "tmp-model-sklearn-dtc",
"model_params": {
"max_depth": depth
},
"out": ""
}
# Training
cii = copy.deepcopy(ii)
r = ck.access(ii)
if r['return'] > 0: ck.err(r)
# Validating
ii = copy.deepcopy(ii)
ii['action'] = 'validate'
ii['ftable'] = test_ftable
ii['ctable'] = test_ctable
r = ck.access(ii)
if r['return'] > 0: ck.err(r)
cross_obs += r['observations']
cross_wrong += r['mispredictions']
cross_acc = float(cross_obs - cross_wrong) / float(cross_obs)
x = ' ' + str(bench) + ' out of ' + str(
len(ftable)
) + ' ) current cross-validation accuracy: ' + '%.2f' % cross_acc
s += x
ck.out(x)
x = '\nDepth: ' + str(
depth
) + ' ; accuracy (with cross-validation): ' + '%.2f' % cross_acc + '\n'
s += x
ck.out(x)
ddd[g]['decision_trees_with_cross_validation_depth_' + str(depth) +
'_ft1_ft65'] = cross_acc
ddd[g]['decision_trees_without_cross_validation_depth_' +
str(depth) + '_ft1_ft65'] = acc_max
r = ck.save_text_file({
'text_file':
'process_model_using_decision_trees/log.' + gx + '.txt',
'string':
s
})
if r['return'] > 0: return r
# Save common data (continuously, not to loose data)
r = ck.save_json_to_file({
'json_file': 'save_all_model_data_tmp.json',
'dict': ddd
})
if r['return'] > 0: return r
return {'return': 0}
def do(i):
# List performance entries
r = ck.access({
'action': 'search',
'module_uoa': 'experiment',
'data_uoa': 'ck-request-asplos18-tvm-fpga-performance-*'
# 'repo_uoa':'ck-request-asplos18-results'
})
if r['return'] > 0: return r
lst = r['lst']
for q in lst:
duid = q['data_uid']
duoa = q['data_uoa']
ruid = q['repo_uid']
path = q['path']
ck.out(duoa)
# Search matching accuracy entry
r = ck.access({
'action': 'load',
'module_uoa': 'experiment',
'data_uoa': duid,
'repo_uoa': ruid
})
if r['return'] > 0: return r
dd = r['dict']
ruid = r['repo_uid']
apath = r['path']
# Updating meta if needed
dd['meta'][
'scenario_module_uoa'] = 'a555738be4b65860' # module:request.asplos18
dd['meta'][
'model_species'] = 'd41bbf1e489ab5e0' # model.species:resnet18
dd['meta'][
'dataset_species'] = 'ImageNet' # dataset species (free format)
dd['meta']['dataset_size'] = 2000 # number of images ...
dd['meta'][
'platform_species'] = 'fpga' # embedded vs server vs fpga (maybe other classifications such as edge)
dd['meta']['platform_peak_power'] = 2.5 #Watts
dd['meta']['platform_price'] = 229 # $
dd['meta']['platform_price_date'] = '20180404' # date
dd['meta']['artifact'] = '9375838469ad4029' # artifact description
dd['meta']['model_precision'] = 'int8'
dd['meta']['processed'] = 'yes'
# Unified full name for some deps
ds = dd['meta']['deps_summary']
x = ds['model']
r = ck.access({
'action': 'make_deps_full_name',
'module_uoa': 'request.asplos18',
'deps': x
})
if r['return'] > 0: return r
dd['meta']['model_design_name'] = r['full_name']
dd['meta']['plat_name'] = 'Xilinx PYNQ-Z1 FPGA (ZYNQ XC7Z020-1CLG400C)'
dd['meta']['os_name'] = 'Ubuntu 15.10'
dd['meta'][
'cpu_name'] = 'Programmable logic equivalent to Artix-7 FPGA'
# Updating entry
r = ck.access({
'action': 'update',
'module_uoa': 'experiment',
'data_uoa': duid,
'repo_uoa': ruid,
'dict': dd,
'substitute': 'yes',
'ignore_update': 'yes',
'sort_keys': 'yes'
})
if r['return'] > 0: return r
# Checking points to aggregate
os.chdir(path)
dperf = os.listdir(path)
for f in dperf:
if f.endswith('.cache.json'):
os.system('git rm -f ' + f)
elif f.endswith('.flat.json'):
ck.out(' * ' + f)
# Load performance file
p1 = os.path.join(path, f)
r = ck.load_json_file({'json_file': p1})
if r['return'] > 0: return r
d1 = r['dict']
# Prune some old value
d = {}
for k in d1:
if not k.startswith('##characteristics#run#accuracy_top1') and \
not k.startswith('##characteristics#run#accuracy_top5') and \
not k.startswith('##characteristics#run#inference_throughput') and \
not k.startswith('##characteristics#run#inference_latency'):
d[k] = d1[k]
d['##features#model_size#min'] = 129770000 # Bytes
d['##features#gpu_freq#min'] = 100
d['##features#cpu_freq#min'] = ''
d['##features#freq#min'] = d['##features#gpu_freq#min']
d['##features#processed#min'] = 'yes'
# Add throughput (images/second)
tall = d.get(
'##characteristics#run#execution_time_classify_internal#all',
[]) # It's internal VTA measurements
if len(tall) > 0:
tnew = []
for t in tall:
t1 = 1 / t
tnew.append(t1)
r = ck.access({
'action': 'stat_analysis',
'module_uoa': 'experiment',
'dict': d,
'dict1': {
'##characteristics#run#inference_throughput': tnew
}
})
if r['return'] > 0: return r
# Unify batch size
batch = 1 # for now only 1 is supported in this artifact
d['##features#batch_size#min'] = batch
# inference latency
d['##features#measuring_latency#min'] = 'yes'
r = ck.access({
'action': 'stat_analysis',
'module_uoa': 'experiment',
'dict': d,
'dict1': {
'##characteristics#run#inference_latency': tall
}
})
if r['return'] > 0: return r
r = ck.access({
'action': 'stat_analysis',
'module_uoa': 'experiment',
'dict': d,
'dict1': {
'##characteristics#run#prediction_time_avg_s': tall
}
})
if r['return'] > 0: return r
# Add accuracy (was calculated through separate experiment)
r = ck.access({
'action': 'stat_analysis',
'module_uoa': 'experiment',
'dict': d,
'dict1': {
'##characteristics#run#accuracy_top1': [accuracy_top1]
}
})
if r['return'] > 0: return r
# Add accuracy (was calculated through separate experiment)
r = ck.access({
'action': 'stat_analysis',
'module_uoa': 'experiment',
'dict': d,
'dict1': {
'##characteristics#run#accuracy_top5': [accuracy_top5]
}
})
if r['return'] > 0: return r
# Save updated dict
r = ck.save_json_to_file({
'json_file': p1,
'dict': d,
'sort_keys': 'yes'
})
if r['return'] > 0: return r
return {'return': 0}
for q in compilers:
s='rpi3-snapshot-'+q.lower().replace(' ','-')+'-autotuning'
# Find entry
r=ck.access({'action':'load',
'module_uoa':'slide',
'data_uoa':s})
if r['return']>0: ck.err(r)
p=r['path']
d=r['dict']
d1=d['slides'][0]
p1=os.path.join(p,d1+'.json')
r=ck.load_json_file({'json_file':p1})
if r['return']>0: ck.err(r)
d2=r['dict']
classes[q]={'-O3':[]}
uopts[q]={'-O3':0}
improvements[q]={}
individual_flags[q]={}
for t in d2['table']:
num=t['solution_num']
flags=t['best_flags'].strip()
classes[q][flags]=[]
uopts[q][flags]=num
def main(i):
# Load common table file (for all models)
r = ck.load_json_file({'json_file': 'save_all_model_data_tmp.json'})
if r['return'] > 0: return r
d = r['dict']
# Get all models/features
kk = list(d['GCC 4.9.2'])
table = []
table_milepost = []
table_short = []
ikk = []
for k in kk:
sort = 0
k1 = k
j = k.find('_depth_')
if j > 0:
j1 = k.find('_', j + 7)
if j1 > 0:
sort = int(k[j + 7:j1])
k1 = k[:j]
j = k.find('_iteration_')
if sort == '' and j > 0:
j1 = k.find('_', j + 11)
if j1 > 0:
sort = int(k[j + 11:j1])
k1 = k[:j]
ikk.append([k1, sort, k])
for k1 in sorted(ikk, key=lambda x: (x[0], x[1])):
k = k1[2]
norm = False
j = k.find('_ft')
j1 = k.find('_normalized')
if j1 > 0:
norm = True
j = j1
km = k[:j].replace('_', ' ')
if km.startswith('milepost'):
ext = km[9:]
if ext != '': ext = ' (' + ext + ')'
km = 'milepost nn' + ext
km = km.replace(' depth', '; depth')
km = km.replace(' iteration', '; iteration')
kf = k[j + 1:]
if norm:
kf = kf[11:] + '\\newline' + '(normalized)'
kf = kf.replace('_', ' .. ')
a4 = "%.2f" % d['GCC 4.9.2'][k]
a7 = "%.2f" % d['GCC 7.1.0'][k]
# Full table (for interactive report)
line = [km, kf, a4, a7]
table.append(line)
# Shorter version for paper - ugly but didn't have time to make it nicer ;)
if 'depth 3' not in km and \
'depth 5' not in km and \
'depth 6' not in km and \
'depth 7' not in km and \
'depth 9' not in km and \
'depth 10' not in km and \
'depth 11' not in km and \
'depth 12' not in km and \
'depth 13' not in km and \
'depth 14' not in km and \
'depth 15' not in km and \
'depth 17' not in km and \
'depth 18' not in km and \
'depth 19' not in km and \
'depth 21' not in km and \
'depth 22' not in km and \
'depth 23' not in km and \
'depth 24' not in km and \
'depth 26' not in km and \
'depth 27' not in km and \
'depth 28' not in km and \
'iteration 5' not in km and \
'iteration 6' not in km and \
'iteration 7' not in km and \
'iteration 8' not in km and \
'iteration 9' not in km:
table_short.append(line)
# Only short MILEPOST
if km == 'milepost nn' and kf == 'ft1 .. ft56':
table_milepost.append(line)
dd = {
"table_style":
"border=\"1\"",
"table_header": [{
"name": "Model",
"html_before": "<b>",
"html_after": "</b>",
"tex": "l",
"tex_before": "\\textbf{",
"tex_after": "}"
}, {
"name": "Features",
"html_change_space": "yes",
"tex": "p{1.2in}"
}, {
"name": "Accuracy (GCC 4.9.2)",
"html_change_space": "yes",
"tex": "p{0.9in}"
}, {
"name": "Accuracy (GCC 7.1.0)",
"html_change_space": "yes",
"tex": "p{0.9in}"
}]
}
# Save full table file (for all models)
dd['table'] = table
r = ck.save_json_to_file({
'json_file': 'save_all_model_data_tmp_table_full.json',
'dict': dd,
'sort_keys': 'yes'
})
if r['return'] > 0: return r
# Save common table file (for all models)
dd['table'] = table_short
r = ck.save_json_to_file({
'json_file': 'save_all_model_data_tmp_table_short.json',
'dict': dd,
'sort_keys': 'yes'
})
if r['return'] > 0: return r
# Save common table file (for all models)
dd['table'] = table_milepost
r = ck.save_json_to_file({
'json_file': 'save_all_model_data_tmp_table_milepost.json',
'dict': dd,
'sort_keys': 'yes'
})
if r['return'] > 0: return r
return {'return': 0}
def access(i):
"""
Input: {
(filename) [str] - load JSON from this file
or
(json) [str] - parse JSON string from command line (use ' instead of ")
or
(dict) [dict] - dictionary to send to the cK API
}
Output: {
return [int] - return code = 0 if success or >0 if error
(error) [str] - error string if return>0
}
"""
import json
filename = i.get('filename', '')
json_string = i.get('json', '')
display = i.get('display', '')
data = i.get('dict', {})
if filename == '' and json_string == '' and len(data) == 0:
return {
'return':
1,
'error':
'either "filename" or "json" or "dict" should define data to be pushed to cK API'
}
if filename != '':
r = ck.load_json_file({'json_file': filename})
if r['return'] > 0: return r
data2 = r['dict']
data.update(data2)
if json_string != '':
json_string = json_string.replace("'", '"')
data2 = json.loads(json_string)
data.update(data2)
if display == '':
display = False
# Get current configuration
r = config.load({})
if r['return'] > 0: return r
cfg = r['dict']
# Prepare request
ii = {'config': cfg}
ii.update(data)
# Sending request to download
r = send(ii)
if r['return'] > 0: return r
if display is True:
ck.out('Output:')
ck.out('')
ck.out(json.dumps(r, indent=2))
return r
def main(i):
# Load common table file (for all models)
ddd = {}
r = ck.load_json_file({'json_file': 'save_all_model_data_tmp.json'})
if r['return'] == 0:
ddd = r['dict']
# Searching for features
for g in gcc:
ck.out(
'********************************************************************************'
)
ck.out('Modeling optimizations for ' + g)
ck.out('')
if g not in ddd: ddd[g] = {}
gx = g.replace(' ', '_')
r = ck.load_json_file(
{'json_file': 'prepare_train_data_tmp.' + gx + '.json'})
if r['return'] > 0: ck.err(r)
d = r['dict']
ftable = d['ftable']
ctable = d['ctable']
# Normalize (all features 0..1)
ftable_range = {}
for f in ftable:
for k in range(0, 121):
v = f[k]
if k not in ftable_range:
ftable_range[k] = {'min': None, 'max': None}
if ftable_range[k]['min'] == None or v < ftable_range[k]['min']:
ftable_range[k]['min'] = v
if ftable_range[k]['max'] == None or v > ftable_range[k]['max']:
ftable_range[k]['max'] = v
ftable_normalized = []
for f in ftable:
x = []
for k in range(0, 121):
v = 0
if ftable_range[k]['max'] != 0:
v = f[k] / ftable_range[k]['max']
x.append(v)
ftable_normalized.append(x)
features_mask = []
for f in range(0, 121):
features_mask.append(1)
r = model({
'ftable': ftable_normalized,
'features_mask': features_mask,
'ctable': ctable
})
if r['return'] > 0: return r
r1 = ck.save_json_to_file({
'json_file':
'process_model_using_nearest_neighbour_reduce_features/prepare_reactions_model_train_ref_result.'
+ gx + '.json',
'dict':
r
})
if r1['return'] > 0: ck.err(r1)
ref_acc = r['accuracy'] # Reference accuracy
x = 'Reference accuracy: ' + str(ref_acc)
s = x + '\n\n'
ck.out('---------------------')
ck.out(x)
ck.out('')
# Calculating Euclidean distance as in our MILEPOST GCC paper: https://hal.inria.fr/hal-00685276
# MILPOST features: https://github.com/ctuning/ck-autotuning/blob/master/module/program.static.features/.cm/meta.json
# 0..55 - original MILEPOST features
# 56..64 - added by Jeremy Singer
# 65..121 - 0..55/ft24 (normalized by total number of instructions)
for k in range(0, 121):
features_mask[k] = 0
r = model({
'ftable': ftable_normalized,
'features_mask': features_mask,
'ctable': ctable,
'skip_out': 'yes'
})
if r['return'] > 0: return r
acc = r['accuracy']
keep = False
sx = ''
if acc < ref_acc:
keep = True
sx = 'kept'
elif acc == ref_acc:
sx = 'removed'
elif acc > ref_acc:
ref_acc = acc
sx = 'removed (accuracy even improved)'
if keep:
features_mask[k] = 1
x = 'ft' + str(k + 1) + ') ' + str(
acc) + ' ' + sx + ' (ref acc=' + str(ref_acc) + ')'
ck.out(x)
s += x + '\n'
# Final accuracy
r = model({
'ftable': ftable_normalized,
'features_mask': features_mask,
'ctable': ctable,
'skip_out': 'yes'
})
if r['return'] > 0: return r
acc = r['accuracy']
r1 = ck.save_json_to_file({
'json_file':
'process_model_using_nearest_neighbour_reduce_features/prepare_reactions_model_train_reduced_result.'
+ gx + '.json',
'dict':
r
})
if r1['return'] > 0: ck.err(r1)
# Final result
ck.out('')
ck.out('Final features mask:')
ck.out('')
s += '\nFinal features mask:\n\n'
for f in range(0, len(features_mask)):
x = ' ft' + str(f + 1) + ') ' + str(features_mask[f])
ck.out(x)
s += x + '\n'
s += '\nFinal features mask:\n\n'
s1 = ''
for f in range(0, len(features_mask)):
x = ' ft' + str(f + 1) + ') ' + str(features_mask[f])
ck.out(x)
s += x + '\n'
if features_mask[f] == 1:
if s1 != '': s1 += ','
s1 += 'ft' + str(f + 1)
s += '\nFinal accuracy: ' + str(acc) + '\n'
r = ck.save_text_file({
'text_file':
'process_model_using_nearest_neighbour_reduce_features/log.' + gx +
'.txt',
'string':
s
})
if r['return'] > 0: return r
r = ck.save_text_file({
'text_file':
'process_model_using_nearest_neighbour_reduce_features/influential_features.'
+ gx + '.txt',
'string':
s1
})
if r['return'] > 0: return r
ddd[g]['milepost_reduce_complexity2_normalized_ft1_ft121'] = acc
# Save common data
r = ck.save_json_to_file({
'json_file': 'save_all_model_data_tmp.json',
'dict': ddd
})
if r['return'] > 0: return r
return {'return': 0}
def main(i):
cur_dir = os.getcwd()
fas = os.path.join(cur_dir, aggregated_stats)
# Get some info about current platform
ii = {'action': 'detect', 'module_uoa': 'platform', 'out': 'con'}
r = ck.access(ii)
if r['return'] > 0: return r
hos = r['host_os_uid']
hosx = r['host_os_uoa']
hosd = r['host_os_dict']
tos = r['os_uid']
tosx = r['os_uoa']
tosd = r['os_dict']
cpu_name = r['features']['cpu']['name']
plat_name = r['features']['platform']['name']
#############################################################
ck.out(line)
ck.out('CPU name: ' + cpu_name)
ck.out('Plat name: ' + plat_name)
#############################################################
ck.out(line)
ck.out('Loading aggregated stats ...')
aa = []
if os.path.isfile(fas):
r = ck.load_json_file({'json_file': fas})
if r['return'] > 0: return r
ax = r['dict']
if 'all' not in ax: ax['all'] = []
aa = ax['all']
#############################################################
ck.out(line)
ck.out('Finding entry related to this platform ...')
found = False
for a in aa:
if a.get('cpu_name', '') == cpu_name and a.get('plat_name',
'') == plat_name:
found = True
if not found:
a = {'cpu_name': cpu_name, 'plat_name': plat_name}
aa.append(a)
if 'data' not in a: a['data'] = {}
data = a.get('data', {})
# Init pipeline
r = ck.access({
'action': 'pipeline',
'module_uoa': 'program',
'data_uoa': 'shared-matmul-c2',
'cpu_freq': 'max',
'gpu_freq': 'max',
'speed': 'yes',
'compiler_vars': {
'USE_BLOCKED_MATMUL': 'YES'
},
'no_state_check': 'yes',
'prepare': 'yes',
'out': 'con'
})
if r['return'] > 0: return r
ready = r['ready']
if ready != 'yes':
return {'return': 1, 'error': 'can\'t init pipeline'}
pipeline = r
# Compile program ones
tpipeline = copy.deepcopy(pipeline)
r = ck.access({
'action': 'autotune',
'module_uoa': 'pipeline',
'pipeline': pipeline,
'pipeline_update': {
'env': {
'CT_MATRIX_DIMENSION': 32,
'CT_BLOCK_SIZE': 32
}
},
'iterations': 1,
'repetitions': 1,
'out': 'con'
})
if r['return'] > 0: return r
lsa = r.get('last_stat_analysis', {}).get('dict_flat', {})
time_min = lsa.get('##characteristics#run#execution_time#min', None)
if time_min == None or time_min == 0.0:
return {'return': 1, 'error': 'failed to run default pipeline'}
# data is per N size
for nn in range(-1, 2):
for n in range(3, 32):
N = (2**n) + nn
SN = str(N)
if SN not in data: data[SN] = {}
xdata = data.get(SN, {})
tmin = xdata.get('tmin', None)
tmax = xdata.get('tmax', None)
gmin = xdata.get('gmin', None)
gmax = xdata.get('gmax', None)
best_tile = xdata.get('best_tile', None)
for opts in range(0, 16):
# Choose if random BS or power of two or power of two -+1
if opts == 0:
BS = 1
elif opts == 1:
BS = N
else:
BS = 2**(opts - 1)
if BS > N: continue
ck.out('Matrix size: ' + str(N))
ck.out('Tile size: ' + str(BS))
# Run pipeline
tpipeline = copy.deepcopy(pipeline)
r = ck.access({
'action': 'autotune',
'module_uoa': 'pipeline',
'pipeline': pipeline,
'pipeline_update': {
'no_compile': 'yes',
'env': {
'CT_MATRIX_DIMENSION': N,
'CT_BLOCK_SIZE': BS
}
},
'iterations': 1,
'repetitions': 3,
'out': 'con'
})
if r['return'] > 0: return r
lsa = r.get('last_stat_analysis', {}).get('dict_flat', {})
tmin = lsa.get('##characteristics#run#execution_time#min',
None)
changed = False
if tmin != None:
ops = 2 * (N * N * N)
gflops = 1.0e-9 * ops / tmin
SBS = str(BS)
xdata[SBS] = {'tmin': tmin, 'gflops': gflops}
ck.out(line)
ck.out('Saving aggregated stats ...')
r = ck.save_json_to_file({
'json_file': fas,
'dict': {
'all': aa
},
'sort_keys': 'yes'
})
if r['return'] > 0: return r
#############################################################
ck.out(line)
ck.out('Saving aggregated stats ...')
r = ck.save_json_to_file({
'json_file': fas,
'dict': {
'all': aa
},
'sort_keys': 'yes'
})
if r['return'] > 0: return r
return {'return': 0}
def process_web_request(i):
"""
Input: {
http - Python http object
}
Output: { None }
"""
from . import solution
# http object
http = i['http']
# Parse GET variables and path
xget = {}
xpath = {
'host': '',
'port': '',
'first': '',
'rest': '',
'query': ''
} # May be used in the future
xt = 'json'
xpath['host'] = i.get('host', '')
xpath['port'] = i.get('port', '')
# Check GET variables
if http.path != '':
http.send_response(200)
a = urlparse.urlparse(http.path)
xp = a.path
xr = ''
if xp.startswith('/'): xp = xp[1:]
u = xp.find('/')
if u >= 0:
xr = xp[u + 1:]
xp = xp[:u]
xt = xp
xpath['first'] = xp
xpath['rest'] = xr
xpath['query'] = a.query
b = urlparse.parse_qs(
a.query,
keep_blank_values=True,
)
xget = {}
for k in b:
xget[k] = urlunquote(b[k][0])
if sys.version_info[0] < 3:
xget[k] = xget[k].decode('utf8')
# Check POST
xpost = {}
xpost1 = {}
try:
headers = http.headers
content_type = headers.get('content-type')
ctype = ''
if content_type != None:
ctype, pdict = cgi.parse_header(content_type)
# Python3 cgi.parse_multipart expects boundary to be bytes, not str.
if sys.version_info[0] < 3 and 'boundary' in pdict:
pdict['boundary'] = pdict['boundary'].encode()
if ctype == 'multipart/form-data':
if sys.version_info[0] < 3:
xpost1 = cgi.parse_multipart(http.rfile, pdict)
else:
xxpost1 = cgi.FieldStorage(fp=http.rfile,
headers=headers,
environ={'REQUEST_METHOD': 'POST'})
for k in xxpost1.keys():
xpost1[k] = [xxpost1[k].value]
elif ctype == 'application/x-www-form-urlencoded':
length = int(http.headers.get('content-length'))
s = http.rfile.read(length)
if sys.version_info[0] > 2: s = s.decode('utf8')
xpost1 = cgi.parse_qs(s, keep_blank_values=1)
except Exception as e:
web_err({'http': http, 'type': xt, 'bin': bin})
ck.out(ck.cfg['error'] + bin.decode('utf8'))
return
# Post processing
for k in xpost1:
v = xpost1[k]
if k.endswith('[]'):
k1 = k[:-2]
xpost[k1] = []
for l in v:
xpost[k1].append(urlunquote(l))
else:
if k != 'file_content':
xpost[k] = urlunquote(v[0])
else:
xpost[k] = v[0]
if k == 'file_content':
fcrt = xpost1.get('file_content_record_to_tmp', '')
if (type(fcrt) == list and len(fcrt) > 0
and fcrt[0] == 'yes') or fcrt == 'yes':
fd, fn = tempfile.mkstemp(
suffix='.tmp', prefix='ck-'
) # suffix is important - CK will delete such file!
os.close(fd)
f = open(fn, 'wb')
f.write(xpost[k])
f.close()
xpost[k + '_uploaded'] = fn
del (xpost[k])
k += '_uploaded'
else:
import base64
xpost[k + '_base64'] = base64.urlsafe_b64encode(
xpost[k]).decode('utf8')
del (xpost[k])
k += '_base64'
if sys.version_info[0] < 3:
xpost[k] = xpost[k].decode('utf8')
# Prepare input and check if CK json present
ii = xget
ii.update(xpost)
act = ii.get('action', '')
# Generate tmp file (to output images for example)
fd, fn = tempfile.mkstemp(
suffix='.tmp',
prefix='ck-') # suffix is important - CK will delete such file!
os.close(fd)
if os.path.isfile(fn): os.remove(fn)
# Get tmp dir
p = tempfile.gettempdir()
# Execute command *********************************************************
ck.out('***************************************************************')
ck.out('Received action request: ' + act)
if act == 'get_host_platform_info':
r = ck.access({'action': 'detect', 'module_uoa': 'platform'})
if r['return'] > 0:
# Process error properly
web_err({
'http': http,
'type': xt,
'bin': r['error'].encode('utf8')
})
return
s = json.dumps(r, indent=2, sort_keys=True)
web_out({'http': http, 'type': 'json', 'bin': s.encode('utf8')})
return
#############################################################################################################3
elif act == 'run_program':
data_id = ii.get('data_id', '')
r = solution.run({'uid': data_id})
# start program
# r=ck.access({'action':'run',
# 'module_uoa':'program',
# 'data_uoa':ii.get('program_name',''),
# 'cmd_key': 'use_continuous',
# 'deps.python': 'a699c0c7de43a121',
# 'quiet': 'yes'})
if r['return'] > 0:
# Process error properly
web_err({
'http': http,
'type': xt,
'bin': r['error'].encode('utf8')
})
return
solution = {'status': True}
s = json.dumps(solution, indent=4, sort_keys=True)
web_out({'http': http, 'type': 'json', 'bin': s.encode('utf8')})
return
elif act == 'benchmark_program':
data_id = ii.get('data_id', '')
r = solution.benchmark({'uid': data_id})
if r['return'] > 0:
# Process error properly
web_err({
'http': http,
'type': xt,
'bin': r['error'].encode('utf8')
})
return
solution = {'status': True}
s = json.dumps(solution, indent=4, sort_keys=True)
web_out({'http': http, 'type': 'json', 'bin': s.encode('utf8')})
return
elif act == 'publish_result':
data_id = ii.get('data_id', '')
r = solution.publish_result({'uid': data_id})
if r['return'] > 0:
# Process error properly
web_err({
'http': http,
'type': xt,
'bin': r['error'].encode('utf8')
})
return
solution = {'status': True}
s = json.dumps(solution, indent=4, sort_keys=True)
web_out({'http': http, 'type': 'json', 'bin': s.encode('utf8')})
return
#############################################################################################################3
elif act == 'get_program_result_image':
data_id = ii['data_id']
program_name = ii['program_name']
jpeg = ii.get('jpeg', '')
ck_entry = program_name.split(':')
# Find solution
r = ck.access({
'action': 'load',
'module_uoa': 'cr-solution',
'data_uoa': data_id
})
if r['return'] > 0:
# Process error properly
web_err({
'http': http,
'type': xt,
'bin': r['error'].encode('utf8')
})
return
p = r['path']
meta = r['dict']
workflow_output_dir = meta.get('workflow_output_dir', '')
workflow_repo = meta.get('workflow_repo_url', '')
j = workflow_repo.rfind('/')
if j > 0:
workflow_repo = workflow_repo[j + 1:]
cur_dir = os.path.join(p, 'CK', workflow_repo, ck_entry[0],
ck_entry[1])
if workflow_output_dir != '':
cur_dir = os.path.join(cur_dir, workflow_output_dir)
# r=ck.access({'action':'find',
# 'module_uoa':'program',
# 'data_uoa':ii.get('program_name','')})
#
# if r['return']>0:
# # Process error properly
# web_err({'http':http, 'type':xt, 'bin':r['error'].encode('utf8')})
# return
# cur_dir = 'D:\\Work1\\CK\\ck-repos\\local\\cr-solution\\demo-obj-detection-kitti-min-tf-cpu-win\\CK\\ck-tensorflow\\program\\squeezedet\\tmp\\out' #os.path.join(r['path'],"tmp/out")
# cur_dir='/home/cindex/CK/local/cr-solution/demo-obj-detection-self-driving-win/CK/ck-tensorflow/program/squeezedet/tmp/out'
# cur_dir='/home/cindex/CK/local/cr-solution/demo-obj-detection-kitti-min-tf-cpu-win/CK/ck-tensorflow/program/squeezedet/tmp/out'
# find the penultimate image provided
try:
st = False
filepath = ''
filepath_buf = ''
found_files = []
ck.out('')
ck.out('Checking for output files in directory:')
ck.out(' ' + cur_dir)
ck.out('')
sorted_list = sorted(os.listdir(cur_dir))
for file in sorted_list:
if file.endswith(".png") and file.startswith("boxed_"):
found_files.append(file)
if len(found_files) == 3:
break
except:
err = 'no files available'
web_err({'http': http, 'type': xt, 'bin': err.encode('utf8')})
return
if len(found_files) == 0:
err = 'no files available'
web_err({'http': http, 'type': xt, 'bin': err.encode('utf8')})
return
if len(found_files) == 1:
filepath = ''
filepath_buf = found_files[0]
elif len(found_files) == 2:
filepath = ''
filepath_buf = found_files[1]
elif len(found_files) == 3:
filepath = found_files[0]
filepath_buf = found_files[1]
# Check if convert to jpeg
file_type = 'png'
pinp = os.path.join(cur_dir, filepath_buf)
if jpeg == 'yes':
quality = ii.get('jpeg_quality', '')
if quality == None or quality == '': quality = '70'
pout = os.path.join(cur_dir, filepath_buf + '.jpg')
s = 'convert -quality ' + quality + ' ' + pinp + ' ' + pout
ck.out('')
ck.out(' Converting to jpeg: ' + s)
os.system(s)
pinp = pout
filepath_buf += '.jpg'
file_type = 'jpg'
# First file will be deleted (only if 2 afterwards), second served
ck.out(' Loading file ' + filepath_buf)
r = ck.load_text_file({'text_file': pinp, 'keep_as_bin': 'yes'})
if jpeg == 'yes':
if os.path.isfile(pinp):
os.remove(pinp)
# Remove first
if filepath != '':
ck.out(' Trying to delete file ' + filepath)
x = os.path.join(cur_dir, filepath)
if os.path.isfile(x):
os.remove(x)
# Then finish checking previous one
if r['return'] > 0:
bout = r['error'].encode('utf-8')
else:
bout = r['bin']
web_out({'http': http, 'type': file_type, 'bin': bout})
return
#############################################################################################################3
elif act == 'process_webcam':
data_id = ii['data_id']
program_name = ii['program_name']
ck_entry = program_name.split(':')
# Find solution
r = ck.access({
'action': 'load',
'module_uoa': 'cr-solution',
'data_uoa': data_id
})
if r['return'] > 0:
# Process error properly
web_err({
'http': http,
'type': xt,
'bin': r['error'].encode('utf8')
})
return
pp = r['path'] # Path to solution!
meta = r['dict']
# Find workflow output path
workflow_input_dir = meta.get('workflow_input_dir', '')
workflow_output_dir = meta.get('workflow_output_dir', '')
workflow_repo = meta.get('workflow_repo_url', '')
j = workflow_repo.rfind('/')
if j > 0:
workflow_repo = workflow_repo[j + 1:]
workflow_dir = os.path.join(pp, 'CK', workflow_repo, ck_entry[0],
ck_entry[1])
if workflow_input_dir != '':
p = os.path.join(workflow_dir, workflow_input_dir)
else:
p = os.path.join(workflow_dir, "tmp", "input")
if not os.path.isdir(p): os.makedirs(p)
if workflow_output_dir != '':
pout = os.path.join(workflow_dir, workflow_output_dir)
else:
pout = os.path.join(workflow_dir, "tmp")
if not os.path.isdir(pout): os.makedirs(pout)
# Record image
image_uri = xpost.get('image_uri', '')
x = 'data:image/jpeg;base64,'
if image_uri.startswith(x):
image64 = image_uri[len(x):]
# Finding last file and incrementing
ff = 'cr-stream-'
l = os.listdir(p)
inum = 0
ffound = ''
for f in os.listdir(p):
if f.startswith(ff) and f.endswith('.jpg'):
j = f.find('.')
num = f[len(ff):j]
if int(num) > inum:
inum = int(num)
ffound = f
# New logic: if file already exists, just skip next request from web (otherwise many parallel requests)
# When program starts, it should clean input/output to let this code continue processing image
if (inum > 0):
time.sleep(1)
ss = 'request skipped because there is already file in queue'
ck.out(' Warning: ' + ss + ' (' + os.path.join(p, ffound) +
') ...')
s = '{"return":16, "error":"' + ss + '"}'
web_out({'http': http, 'type': 'json', 'bin': s.encode('utf8')})
return
# Otherwise continue processing ...
if inum == 0:
inum += 1
sinum = str(inum)
filename = ff + ('0' * (8 - len(sinum))) + sinum
filename2 = filename + '.jpg'
pf = os.path.join(p, filename2)
r = ck.convert_upload_string_to_file({
'file_content_base64': image64,
'filename': pf
})
if r['return'] > 0: return r
ck.out(
' !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!'
)
ck.out(' Recorded external image to ' + pf)
# Need extra converting
pp1 = os.path.join(pp, 'support-script-convert.sh')
if os.path.isfile(pp1):
ck.out('')
ck.out('Extra image processing ...')
ck.out('')
extra_cmd = 'cd "' + p + '"\n'
extra_cmd += '. "' + pp1 + '" ' + filename2 + '\n'
r = solution.run({'uid': data_id, 'cmd': extra_cmd})
if r['return'] > 0:
# Process error properly
web_err({
'http': http,
'type': xt,
'bin': r['error'].encode('utf8')
})
return
else:
sinum = str(inum)
filename = ff + ('0' * (8 - len(sinum))) + sinum
filename2 = filename + '.jpg'
pf = os.path.join(p, filename2)
# Need extra pushing
pp1 = os.path.join(pp, 'support-script-push.sh')
if os.path.isfile(pp1):
ck.out('')
ck.out('Extra image pushing to device ...')
ck.out('')
extra_cmd = 'cd "' + p + '"\n'
extra_cmd += '. "' + pp1 + '" ' + filename + '\n'
r = solution.run({'uid': data_id, 'cmd': extra_cmd})
if r['return'] > 0:
# Process error properly
web_err({
'http': http,
'type': xt,
'bin': r['error'].encode('utf8')
})
return
# If Android-like device wait for the file ...
ppull = os.path.join(pp, 'support-script-pull.sh')
# Waiting for output file
poutf = os.path.join(pout, filename + '.json')
if not os.path.isfile(poutf):
ck.out('Waiting for output file: ' + poutf)
while not os.path.isfile(poutf):
# Check if need to pull
if os.path.isfile(ppull):
ck.out('Trying to pull from device ...')
extra_cmd = 'cd "' + pout + '"\n'
extra_cmd += 'export CODEREEF_SOLUTION_PATH="' + pp + '"\n'
extra_cmd += '. "' + ppull + '" ' + filename + '\n'
r = solution.run({'uid': data_id, 'cmd': extra_cmd})
if r['return'] > 0:
# Process error properly
web_err({
'http': http,
'type': xt,
'bin': r['error'].encode('utf8')
})
return
time.sleep(0.1)
ck.out('')
ck.out('Found solution!')
ck.out('')
with open(poutf) as json_file:
solution = json.load(json_file)
ck.out(json.dumps(solution, indent=2))
if os.path.isfile(poutf):
os.remove(poutf)
if inum == 1 and os.path.isfile(pf):
ck.out(' REMOVING ' + pf)
os.remove(pf)
ck.out('')
s = json.dumps(solution, indent=4, sort_keys=True)
web_out({'http': http, 'type': 'json', 'bin': s.encode('utf8')})
return
#############################################################################################################3
elif act == 'get_image':
num = ii.get('num', '')
inum = int(num)
sinum = str(inum)
# Finding last file and incrementing
ff = 'cr-stream-'
pf = os.path.join(p, ff + ('0' * (8 - len(sinum))) + sinum + '.jpg')
ck.out(' Loaded file ' + pf)
r = ck.load_text_file({'text_file': pf, 'keep_as_bin': 'yes'})
if r['return'] > 0:
bout = r['error'].encode('utf-8')
else:
bout = r['bin']
web_out({'http': http, 'type': 'jpeg', 'bin': bout})
return
#############################################################################################################3
elif act == 'get_result':
data_id = ii['data_id']
# Find solution
r = ck.access({
'action': 'load',
'module_uoa': 'cr-solution',
'data_uoa': data_id
})
if r['return'] > 0:
# Process error properly
web_err({
'http': http,
'type': xt,
'bin': r['error'].encode('utf8')
})
return
pp = r['path'] # Path to solution!
meta = r['dict']
program_name = meta.get('workflow', '')
ck_entry = program_name.split(':')
# Find workflow output path
result_file = meta.get('result_file', '')
workflow_repo = meta.get('workflow_repo_url', '')
j = workflow_repo.rfind('/')
if j > 0:
workflow_repo = workflow_repo[j + 1:]
workflow_dir = os.path.join(pp, 'CK', workflow_repo, ck_entry[0],
ck_entry[1])
if result_file != '':
pout = os.path.join(workflow_dir, result_file)
else:
pout = os.path.join(workflow_dir, "tmp", "tmp-ck-timer.json")
# if not os.path.isdir(pout): os.makedirs(pout)
# If Android-like device wait for the file ...
ppull = os.path.join(pp, 'support-script-pull.sh')
# Waiting for output file
if not os.path.isfile(pout):
ck.out('Waiting for output file: ' + pout)
while not os.path.isfile(pout):
# Check if need to pull
if os.path.isfile(ppull):
ck.out('Trying to pull from device ...')
extra_cmd = 'cd "' + pout + '"\n'
extra_cmd += 'export CODEREEF_SOLUTION_PATH="' + pp + '"\n'
extra_cmd += '. "' + ppull + '" ' + filename + '\n'
r = solution.run({'uid': data_id, 'cmd': extra_cmd})
if r['return'] > 0:
# Process error properly
web_err({
'http': http,
'type': xt,
'bin': r['error'].encode('utf8')
})
return
time.sleep(0.1)
ck.out('')
ck.out('Found solution!')
ck.out('')
rx = ck.load_json_file({'json_file': pout})
if rx['return'] > 0: return rx
rx = ck.flatten_dict(rx)
if rx['return'] > 0: return rx
rdf = rx['dict']
crdf = {}
# Remove first ## (do not need here)
for k in rdf:
v = rdf[k]
if k.startswith('##'): k = k[2:]
crdf[k] = v
ck.out(json.dumps(crdf, indent=2))
# if os.path.isfile(pout):
# os.remove(pout)
# if inum==1 and os.path.isfile(pf):
# ck.out(' REMOVING '+pf)
# os.remove(pf)
ck.out('')
s = json.dumps(crdf, indent=4, sort_keys=True)
web_out({'http': http, 'type': 'json', 'bin': s.encode('utf8')})
return
elif act == 'get_status':
data_id = ii['data_id']
# Find solution
r = ck.access({
'action': 'load',
'module_uoa': 'cr-solution',
'data_uoa': data_id
})
if r['return'] > 0:
# Process error properly
web_err({
'http': http,
'type': xt,
'bin': r['error'].encode('utf8')
})
return
pp = r['path'] # Path to solution!
tmp_solStatus = os.path.join(pp, "tmp", "status.json")
rx = ck.load_json_file({'json_file': tmp_solStatus})
if rx['return'] > 0: return rx
ck.out(json.dumps(rx, indent=2))
rdf = rx['dict']
ck.out('')
s = json.dumps(rdf, indent=4, sort_keys=True)
web_out({'http': http, 'type': 'json', 'bin': s.encode('utf8')})
return
#############################################################################################################3
elif act == 'heartbit':
locdir = os.path.dirname(os.path.realpath(__file__))
ck.out(' Local directory: ' + locdir)
# Finding last file and incrementing
pf = os.path.join(locdir, 'static/favicon.ico')
ck.out(' Loaded file ' + pf)
r = ck.load_text_file({'text_file': pf, 'keep_as_bin': 'yes'})
if r['return'] > 0:
bout = r['error'].encode('utf-8')
else:
bout = r['bin']
web_out({'http': http, 'type': 'jpeg', 'bin': bout})
return
r = {'return': 0}
xt = 'web'
bout = b'TEST WORKS'
web_out({'http': http, 'type': xt, 'bin': bout})
return
# Process output
if r['return'] > 0:
if os.path.isfile(fn): os.remove(fn)
bout = r['error']
try:
bout = bout.encode('utf-8')
except Exception as e:
pass
web_err({'http': http, 'type': xt, 'bin': bout})
return
# If json or web
# Try to load output file
if not os.path.isfile(fn):
web_err({
'http':
http,
'type':
xt,
'bin':
b'Output file was not created, see output (' +
r['std'].encode('utf8') + b')!'
})
return
r = ck.load_text_file({'text_file': fn, 'keep_as_bin': 'yes'})
if r['return'] > 0:
bout = r['error']
try:
bout = bout.encode('utf-8')
except Exception as e:
pass
web_err({'http': http, 'type': xt, 'bin': bout})
return
bin = r['bin']
# Process JSON output from file
fx = ''
if sys.version_info[0] > 2: bin = bin.decode('utf-8')
ru = ck.convert_json_str_to_dict({
'str': bin,
'skip_quote_replacement': 'yes'
})
if ru['return'] > 0:
bout = ru['error']
try:
bout = bout.encode('utf-8')
except Exception as e:
pass
web_err({'http': http, 'type': xt, 'bin': bout})
return
rr = ru['dict']
if rr['return'] > 0:
bout = rr['error']
try:
bout = bout.encode('utf-8')
except Exception as e:
pass
web_err({'http': http, 'type': xt, 'bin': bout})
return
# Check if file was returned
fr = False
if 'file_content_base64' in rr and rr.get('filename', '') != '':
fr = True
# Check if download
if (xt == 'web' and fr) or (act == 'pull' and xt != 'json'):
import base64
x = rr.get('file_content_base64', '')
fx = rr.get('filename', '')
if fx == '': fx = ck.cfg['default_archive_name']
# Fixing Python bug
if sys.version_info[0] == 3 and sys.version_info[1] < 3:
x = x.encode('utf-8')
else:
x = str(x)
bin = base64.urlsafe_b64decode(
x) # convert from unicode to str since base64 works on strings
# should be safe in Python 2.x and 3.x
# Process extension
fn1, fne = os.path.splitext(fx)
if fne.startswith('.'): fne = fne[1:]
if fne != '': xt = fne
else: xt = 'unknown'
else:
# Check and output html
if rr.get('html', '') != '':
bin = rr['html'].encode('utf-8')
else:
if sys.version_info[0] > 2: # Unknown output
bin = bin.encode('utf-8')
web_out({'http': http, 'type': xt, 'bin': bin, 'filename': fx})
return {'return': 0}
dd['##characteristics#gpu_copy_is_much_better_cpu#min']=False
if (fcpu/fgpu_only)>1.07:
dd['##characteristics#gpu_only_is_much_better_cpu#min']=True
else:
dd['##characteristics#gpu_only_is_much_better_cpu#min']=False
return {'return':0, 'changed':changed, 'dict':dd}
########################################################
ff=getattr(sys.modules[__name__], 'filter_data')
ii={'action':'filter',
'module_uoa':'experiment',
'out':'con',
'filter_func_addr':ff}
r=ck.load_json_file({'json_file':'filter-add-characteristic-gpu-copy-is-much-better-cpu.py'})
if r['return']>0:
ck.out('Error:'+r['error'])
exit(1)
ii.update(r['dict'])
r=ck.access(ii)
if r['return']>0:
ck.out('Error:'+r['error'])
exit(1)
exit(0)
def push(i):
"""
Input: {
uid [str] - graph identifyer
(version) [str] - graph version
(filename) [str] - JSON file with results
(json) [str] - JSON string from command line (use ' instead of ")
(point) [str] - specific point name to add/update
}
Output: {
return [int] - return code = 0 if success or >0 if error
(error) [str] - error string if return>0
dict [dict] - configuration dictionary
path [str] - path to CK cfg entry
}
"""
# CID ###########################################################
uid=i['uid']
if uid=='':
return {'return':1, 'error':'graph UID is not defined!'}
version=i.get('version')
if version==None: version=''
filename=i.get('filename','')
json_string=i.get('json','')
if filename=='' and json_string=='':
return {'return':1, 'error':'either "filename" or "json" should define results to be pushed'}
point=i.get('point','')
# Prepare data
data=[]
if filename!='':
r=ck.load_json_file({'json_file':filename})
if r['return']>0: return r
data2=r['dict']
if type(data2)==dict:
data2=[data2]
data+=data2
if json_string!='':
import json
json_string=json_string.replace("'", '"')
data2=json.loads(json_string)
if type(data2)==dict:
data2=[data2]
data+=data2
# Send request
r=config.load({})
if r['return']>0: return r
cfg=r['dict']
# Check if username and API_Key are empty and then use default crowd-user ...
username=cfg.get('username','')
if username=='' or username==None:
cfg['username']=config.CR_DEFAULT_SERVER_USER
cfg['api_key']=config.CR_DEFAULT_SERVER_API_KEY
# Sending request to download
r=comm.send({'config':cfg,
'action':'push_result',
'dict':{
'data_uoa':uid,
'version':version,
'point':point,
'data':data
}
})
if r['return']>0: return r
url=r.get('url','')
ck.out(' Successfully pushed your point to a graph!')
if url!='':
ck.out(' URL: '+url)
return r
def do(i):
top1 = {}
top5 = {}
# List accuracy entries
r = ck.access({
'action': 'search',
'module_uoa': 'experiment',
'data_uoa': 'ck-request-asplos18-caffe-intel-performance-*',
'repo_uoa': 'local',
'add_meta': 'yes'
})
if r['return'] > 0: return r
lst = r['lst']
for q in lst:
duid = q['data_uid']
duoa = q['data_uoa']
path = q['path']
if 'inception-v3' in duoa:
model = 'inception-v3'
model_species = '1b339ddb13408f8f'
elif 'resnet50' in duoa:
model = 'resnet50'
model_species = 'd777f6335496db61'
if model == '':
return {'return': 1, 'error': 'model is not recognized'}
prec = ''
if '-fp32' in duoa:
prec = 'fp32'
elif '-int8' in duoa:
prec = 'int8'
if prec == '':
return {'return': 1, 'error': 'model precision is not recognized'}
ck.out('* ' + duoa + ' / ' + model + ' / ' + prec)
# Search matching accuracy entry (if intel-request)
x = 'ck-request-asplos18-caffe-intel-accuracy.*.' + model + '-' + prec
r = ck.access({
'action': 'search',
'module_uoa': 'experiment',
'data_uoa': x,
'repo_uoa': 'local'
})
if r['return'] > 0: return r
alst = r['lst']
if len(alst) != 1:
return {'return': 1, 'error': 'ambiguity when search for accuracy'}
a = alst[0]
apath = a['path']
# There is only one point normally (no model tuning)
dacc = {}
xacc = os.listdir(apath)
for f in xacc:
if f.endswith('.flat.json'):
r = ck.load_json_file({'json_file': os.path.join(apath, f)})
if r['return'] > 0: return r
dx = r['dict']
# Get only accuracy keys (convert to common format)
for k in dx:
if k.startswith('##characteristics#run#acc/top-'):
k1 = '##characteristics#run#accuracy_top' + k[30:]
dacc[k1] = dx[k]
elif k.startswith('##characteristics#run#accuracy/top-'):
k1 = '##characteristics#run#accuracy_top' + k[35:]
dacc[k1] = dx[k]
break
if len(dacc) == 0:
return {
'return': 1,
'error': 'strange - no match for accuracy entries'
}
# Iterating over points to aggregate
dperf = os.listdir(path)
for f in dperf:
if f.endswith('.flat.json'):
ck.out(' * ' + f)
# Load performance file
p1 = os.path.join(path, f)
r = ck.load_json_file({'json_file': p1})
if r['return'] > 0: return r
d = r['dict']
# Merge accuracy
for k in dacc:
d[k] = dacc[k]
# Save updated dict
r = ck.save_json_to_file({
'json_file': p1,
'dict': d,
'sort_keys': 'yes'
})
if r['return'] > 0: return r
return {'return': 0}
def init(i):
"""
Input: {
uid [str] - graph identifyer
(version) [str] - graph version
(desc_file) [str] - file with graph description
(tags) [str] - tags separated by comma
}
Output: {
return [int] - return code = 0 if success or >0 if error
(error) [str] - error string if return>0
dict [dict] - configuration dictionary
path [str] - path to CK cfg entry
}
"""
# Get main configuration
r=config.load({})
if r['return']>0: return r
cfg=r.get('dict',{})
pcfg=r.get('path','')
# CID ###########################################################
uid=i['uid']
if uid==None: uid=''
version=i.get('version')
if version==None: version=''
desc_file=i.get('desc_file','')
if desc_file==None: desc_file=''
# If UID!='', check if already exists ...
found=False
meta=meta_template
path=''
data_name=''
tags=[]
meta_info=''
source=''
extra_info={}
if uid!='':
r=ck.access({'action':'load',
'module_uoa':'result',
'data_uoa':uid})
if r['return']>0:
if r['return']!=16: return r
else:
found=True
meta=r['dict']
path=r['path']
data_name=r['data_name']
tags=meta.get('tags',[])
source=meta.get('source','')
meta_info=meta.get('meta',{}).get('info','')
extra_info=r['info'].get('control',{})
# Check if init from scratch and no title
if i.get('name')!=None and i.get('name','')!='':
data_name=i['name'].strip()
elif not found or data_name=='':
r=ck.inp({'text':'Select a title for your graph: '})
if r['return']>0: return r
data_name=r['string'].strip()
meta['meta']['title']=data_name
# Check if init from scratch and no title
if not found or meta_info=='':
r=ck.inp({'text':'Enter general info about your graph: '})
if r['return']>0: return r
x=r['string'].strip()
if x=='': x=' '
meta['meta']['info']=x
# Adding tags
if i.get('tags')!=None and i.get('tags','')!='':
xtags=i['tags'].strip().split(',')
for t in xtags:
t1=t.strip()
if t1!='' and t1 not in tags:
tags.append(t1)
meta['tags']=tags
elif not found or (len(tags)==1 and 'result' in tags):
r=ck.inp({'text':'Enter tags for your graph separated by commas: '})
if r['return']>0: return r
xtags=r['string'].strip().split(',')
for t in xtags:
t1=t.strip()
if t1!='' and t1 not in tags:
tags.append(t1)
meta['tags']=tags
# Checking source
if not found or source=='':
r=ck.inp({'text':'Enter source of results for your graph (can be URL): '})
if r['return']>0: return r
source=r['string'].strip()
meta['source']=source
# Checking authors
for x in extra_info_desc:
k=x['key']
n=x['name']
if not found or extra_info.get(k,'')=='':
r=ck.inp({'text':'Enter '+n+': '})
if r['return']>0: return r
s=r['string'].strip()
extra_info[k]=s
# Creating/updating graph
a='add'
if found: a='update'
ii={'action':a,
'module_uoa':'result',
'data_uoa':uid,
'dict':meta,
'sort_keys':'yes',
'data_name':data_name,
'substitute':'yes',
'extra_info':extra_info}
r=ck.access(ii)
if r['return']>0: return r
data_uoa=r['data_uoa']
data_uid=r['data_uid']
path=r['path']
x='initialized'
if found: x='updated'
ck.out('Graph was successfully '+x+':')
ck.out('')
ck.out(' CK UID: '+data_uid)
ck.out(' CK name: '+data_uoa)
ck.out(' CK path: '+path)
# Add desc
p1=os.path.join(path, 'desc.json')
dt=copy.deepcopy(desc_template)
if desc_file!='':
rx=ck.load_json_file({'json_file':desc_file})
if rx['return']>0: return rx
dx=rx['dict']
dt['data_config'].update(dx)
if desc_file!='' or not os.path.isfile(p1):
rx=ck.save_json_to_file({'json_file':p1, 'dict':dt, 'sort_keys':'yes'})
if rx['return']>0: return rx
p2=os.path.join(path, '.cm', 'meta.json')
ck.out('')
ck.out('You can continue updating graph using following files: ')
ck.out('')
ck.out(' Graph general meta info: '+p1)
ck.out(' See example at '+config.CR_DEFAULT_SERVER+'/result/sota-mlperf-inference-results-v0.5-open-available/?action=download&filename=.cm/meta.json')
ck.out('')
ck.out(' Graph axes info: '+p2)
ck.out(' See example at '+config.CR_DEFAULT_SERVER+'/result/sota-mlperf-inference-results-v0.5-open-available/?action=download&filename=desc.json')
# Need to publish
ck.out('')
rx=ck.inp({'text':'Publish graph on the portal (Y/n)?'})
if rx['return']>0: return rx
s=rx['string'].strip().lower()
if s=='' or s=='y':
ck.out('')
r=obj.publish({'cid':'result:'+data_uoa,
'version':version,
'force':True})
else:
ck.out('')
ck.out('You can publish your graph on the portal using the following commands when ready: ')
ck.out('')
ck.out(' cb publish result:'+data_uoa+' --version=1.0.0 --force (--private)')
return r
def main(i):
# Load common table file (for all models)
ddd = {}
r = ck.load_json_file({'json_file': 'save_all_model_data_tmp.json'})
if r['return'] == 0:
ddd = r['dict']
# Searching for features
for g in gcc:
ck.out(
'********************************************************************************'
)
ck.out('Reducing model complexity for ' + g)
ck.out('')
if g not in ddd: ddd[g] = {}
gx = g.replace(' ', '_')
r = ck.load_json_file(
{'json_file': 'prepare_train_data_tmp.' + gx + '.json'})
if r['return'] > 0: ck.err(r)
d = r['dict']
ftable = d['ftable']
ctable = d['ctable']
# Normalize (all features 0..1)
ftable_range = {}
for f in ftable:
for k in range(0, 121):
v = f[k]
if k not in ftable_range:
ftable_range[k] = {'min': None, 'max': None}
if ftable_range[k]['min'] == None or v < ftable_range[k]['min']:
ftable_range[k]['min'] = v
if ftable_range[k]['max'] == None or v > ftable_range[k]['max']:
ftable_range[k]['max'] = v
ftable_normalized = []
for f in ftable:
x = []
for k in range(0, 121):
v = 0
if ftable_range[k]['max'] != 0:
v = f[k] / ftable_range[k]['max']
x.append(v)
ftable_normalized.append(x)
features_mask = []
for f in range(0, 121):
features_mask.append(0)
r = model({
'ftable': ftable_normalized,
'features_mask': features_mask,
'ctable': ctable
})
if r['return'] > 0: return r
r1 = ck.save_json_to_file({
'json_file':
'process_model_using_nearest_neighbour_assemble_features/prepare_reactions_model_train_ref_result.'
+ gx + '.json',
'dict':
r
})
if r1['return'] > 0: ck.err(r1)
ref_acc = r['accuracy'] # Reference accuracy
x = 'Reference accuracy: ' + str(ref_acc)
s = x + '\n\n'
ck.out('---------------------')
ck.out(x)
ck.out('')
# Assembling features (adding features one by one) 0:121 - normalized features!
for k in range(0, 121):
features_mask[k] = 1
r = model({
'ftable': ftable_normalized,
'features_mask': features_mask,
'ctable': ctable,
'skip_out': 'yes'
})
if r['return'] > 0: return r
acc = r['accuracy']
keep = False
sx = ''
if acc < ref_acc:
keep = True
sx = 'removed'
elif acc == ref_acc:
sx = 'kept'
elif acc > ref_acc:
ref_acc = acc
sx = 'kept (accuracy even improved)'
if keep:
features_mask[k] = 0
x = 'ft' + str(k + 1) + ') ' + str(
acc) + ' ' + sx + ' (ref acc=' + str(ref_acc) + ')'
ck.out(x)
s += x + '\n'
# Final accuracy
r = model({
'ftable': ftable_normalized,
'features_mask': features_mask,
'ctable': ctable,
'skip_out': 'yes'
})
if r['return'] > 0: return r
acc = r['accuracy']
r1 = ck.save_json_to_file({
'json_file':
'process_model_using_nearest_neighbour_assemble_features/prepare_reactions_model_train_reduced_result.'
+ gx + '.json',
'dict':
r
})
if r1['return'] > 0: ck.err(r1)
# Final result
ck.out('')
ck.out('Final features mask:')
ck.out('')
s += '\nFinal features mask:\n\n'
s1 = ''
for f in range(0, len(features_mask)):
x = ' ft' + str(f + 1) + ') ' + str(features_mask[f])
ck.out(x)
s += x + '\n'
if features_mask[f] == 1:
if s1 != '': s1 += ','
s1 += 'ft' + str(f + 1)
s += '\nFinal accuracy: ' + str(acc) + '\n'
r = ck.save_text_file({
'text_file':
'process_model_using_nearest_neighbour_assemble_features/log.' +
gx + '.txt',
'string':
s
})
if r['return'] > 0: return r
r = ck.save_text_file({
'text_file':
'process_model_using_nearest_neighbour_assemble_features/influential_features.'
+ gx + '.txt',
'string':
s1
})
if r['return'] > 0: return r
ddd[g]['milepost_reduce_complexity1_normalized_ft1_ft121'] = acc
# Save common data
r = ck.save_json_to_file({
'json_file': 'save_all_model_data_tmp.json',
'dict': ddd
})
if r['return'] > 0: return r
return {'return': 0}
def main(i):
# Load common table file (for all models)
ddd = {}
r = ck.load_json_file({'json_file': 'save_all_model_data_tmp.json'})
if r['return'] == 0:
ddd = r['dict']
# Searching for features
for g in gcc:
ck.out(
'********************************************************************************'
)
ck.out('Modeling optimizations for ' + g)
ck.out('')
if g not in ddd: ddd[g] = {}
gx = g.replace(' ', '_')
r = ck.load_json_file(
{'json_file': 'prepare_train_data_tmp.' + gx + '.json'})
if r['return'] > 0: ck.err(r)
d = r['dict']
ftable = d['ftable']
ctable = d['ctable']
# Normalize (all features 0..1)
ftable_range = {}
for f in ftable:
for k in range(0, 121):
v = f[k]
if k not in ftable_range:
ftable_range[k] = {'min': None, 'max': None}
if ftable_range[k]['min'] == None or v < ftable_range[k]['min']:
ftable_range[k]['min'] = v
if ftable_range[k]['max'] == None or v > ftable_range[k]['max']:
ftable_range[k]['max'] = v
ftable_normalized = []
for f in ftable:
x = []
for k in range(0, 121):
v = 0
if ftable_range[k]['max'] != 0:
v = f[k] / ftable_range[k]['max']
x.append(v)
ftable_normalized.append(x)
# ft1..ft56
for ft in [[0, 56], [0, 65], [0, 121], [56, 65], [56, 121], [65, 121]]:
ft_start = ft[0]
ft_stop = ft[1]
ext = 'ft' + str(ft_start + 1) + '_' + 'ft' + str(ft_stop)
ck.out('')
ck.out('Using non-normalized features ' + ext + ' ...')
r = model({
'ftable': ftable,
'ctable': ctable,
'ft_start': ft_start,
'ft_stop': ft_stop
})
if r['return'] > 0: return r
ddd[g]['milepost_' + ext] = r['accuracy']
r1 = ck.save_json_to_file({
'json_file':
'process_model_using_nearest_neighbour/process_model_using_nearest_neighbour_'
+ ext + '_tmp.' + gx + '.json',
'dict':
r
})
if r1['return'] > 0: ck.err(r1)
# Normalized ft1..ft56
ck.out('')
ck.out('Using normalized features ' + ext + ' ...')
r = model({
'ftable': ftable_normalized,
'ctable': ctable,
'ft_start': ft_start,
'ft_stop': ft_stop
})
if r['return'] > 0: return r
r1 = ck.save_json_to_file({
'json_file':
'process_model_using_nearest_neighbour/process_model_using_nearest_neighbour_'
+ ext + '_tmp.' + gx + '.normalized.json',
'dict':
r
})
if r1['return'] > 0: ck.err(r1)
ddd[g]['milepost_normalized_' + ext] = r['accuracy']
# Save common data
r = ck.save_json_to_file({
'json_file': 'save_all_model_data_tmp.json',
'dict': ddd
})
if r['return'] > 0: return r
return {'return': 0}
def do(i):
# List performance entries
r = ck.access({
'action': 'search',
'module_uoa': 'experiment',
'repo_uoa': 'local',
# 'repo_uoa':'ck-request-asplos18-results'})
'data_uoa': '*ck-request-asplos18-caffe-intel-performance-*'
})
if r['return'] > 0: return r
lst = r['lst']
for q in lst:
duid = q['data_uid']
duoa = q['data_uoa']
ruid = q['repo_uid']
path = q['path']
ck.out(duoa)
# Load entry
r = ck.access({
'action': 'load',
'module_uoa': 'experiment',
'data_uoa': duid,
'repo_uoa': ruid
})
if r['return'] > 0: return r
dd = r['dict']
ruid = r['repo_uid']
apath = r['path']
# Check extra info
if 'inception-v3' in duoa:
model = 'inception-v3'
model_species = '1b339ddb13408f8f'
model_size = 95533753
elif 'resnet50' in duoa:
model = 'resnet50'
model_species = 'd777f6335496db61'
model_size = 102462397
if model == '':
return {'return': 1, 'error': 'model is not recognized'}
prec = ''
if '-fp32' in duoa:
prec = 'fp32'
elif '-int8' in duoa:
prec = 'int8'
model_size = model_size / 4 # Guess
if prec == '':
return {'return': 1, 'error': 'model precision is not recognized'}
# Updating meta if needed
dd['meta'][
'scenario_module_uoa'] = 'a555738be4b65860' # module:request.asplos18
dd['meta']['model_species'] = model_species # model.species:mobilenets
dd['meta'][
'dataset_species'] = 'ImageNet' # dataset species (free format)
dd['meta']['dataset_size'] = 50000 # number of images ...
dd['meta'][
'platform_species'] = 'server' # embedded vs server (maybe other classifications such as edge)
# Unified full name for some deps
ds = dd['meta']['deps_summary']
x = ds['caffemodel']
r = ck.access({
'action': 'make_deps_full_name',
'module_uoa': 'request.asplos18',
'deps': x
})
if r['return'] > 0: return r
dd['meta']['model_design_name'] = r['full_name']
x = ds['lib-caffe']
r = ck.access({
'action': 'make_deps_full_name',
'module_uoa': 'request.asplos18',
'deps': x
})
if r['return'] > 0: return r
dd['meta']['library_name'] = r['full_name']
x = x['deps']['compiler']
r = ck.access({
'action': 'make_deps_full_name',
'module_uoa': 'request.asplos18',
'deps': x
})
if r['return'] > 0: return r
dd['meta']['compiler_name'] = r['full_name']
used_gpgpu = False
if ds.get('lib-caffe', {}).get('deps', {}).get(
'compiler-cuda', {}).get('data_name', '') != '':
used_gpgpu = True
if used_gpgpu:
# GPU used
dd['meta']['cpu_name'] = ''
dd['meta']['cpu_vendor'] = ''
dd['meta']['platform_peak_power'] = 180 #Watts
dd['meta']['platform_price'] = 700 # $
dd['meta']['platform_price_date'] = '20180101' # date
else:
dd['meta']['gpgpu_name'] = ''
dd['meta']['gpgpu_vendor'] = ''
dd['meta']['platform_peak_power'] = 105 #Watts
dd['meta']['platform_price'] = 1166 # $
dd['meta']['platform_price_date'] = '20141212' # date
dd['meta']['artifact'] = 'e7cc77d72f13441e' # artifact description
dd['meta']['model_precision'] = prec
dd['meta']['processed'] = 'yes'
# Updating entry
r = ck.access({
'action': 'update',
'module_uoa': 'experiment',
'data_uoa': duid,
'repo_uoa': ruid,
'dict': dd,
'substitute': 'yes',
'ignore_update': 'yes',
'sort_keys': 'yes'
})
if r['return'] > 0: return r
# Checking points to aggregate
os.chdir(path)
dperf = os.listdir(path)
for f in dperf:
if f.endswith('.cache.json'):
os.system('git rm -f ' + f)
elif f.endswith('.flat.json'):
ck.out(' * ' + f)
# Load performance file
p1 = os.path.join(path, f)
r = ck.load_json_file({'json_file': p1})
if r['return'] > 0: return r
d = r['dict']
d['##features#processed#min'] = 'yes'
# Clean up keys
d1 = {}
for k in d:
v = d[k]
if not k.startswith('##characteristics#run#inference_latency') and \
not k.startswith('##characteristics#run#prediction_time_avg_s') and \
not k.startswith('##characteristics#run#inference_throughput') and \
not k.startswith('##characteristics#run#usage_cost'):
d1[k] = v
d = d1
# Unify execution time + batch size
x = d.get(
'##characteristics#run#REAL_ENV_CK_CAFFE_BATCH_SIZE#min',
'')
if x != None and x != '':
batch = int(x)
d['##features#batch_size#min'] = batch
tall = d.get('##characteristics#run#time_fw_s#all', [])
if batch == 1:
# inference latency
d['##features#measuring_latency#min'] = 'yes'
r = ck.access({
'action': 'stat_analysis',
'module_uoa': 'experiment',
'dict': d,
'dict1': {
'##characteristics#run#inference_latency': tall
}
})
if r['return'] > 0: return r
tnew = []
cnew = []
for t in tall:
t1 = t / batch
tnew.append(t1)
c1 = t1 * cost / (60 * 60)
if c1 != 0:
cnew.append(c1)
r = ck.access({
'action': 'stat_analysis',
'module_uoa': 'experiment',
'dict': d,
'dict1': {
'##characteristics#run#prediction_time_avg_s': tnew
}
})
if r['return'] > 0: return r
if len(cnew) > 0:
r = ck.access({
'action': 'stat_analysis',
'module_uoa': 'experiment',
'dict': d,
'dict1': {
'##characteristics#run#usage_cost': cnew
}
})
if r['return'] > 0: return r
d['##characteristics#run#usage_cost_per_hour#min'] = cost
d['##characteristics#run#usage_cost_date'] = '20180403'
# Throughput for all batches
if len(tnew) > 0:
tall = tnew # from previous calculation
tnew = []
for t in tall:
t1 = 1 / t
tnew.append(t1)
r = ck.access({
'action': 'stat_analysis',
'module_uoa': 'experiment',
'dict': d,
'dict1': {
'##characteristics#run#inference_throughput':
tnew
}
})
if r['return'] > 0: return r
d['##features#model_size#min'] = model_size
if not used_gpgpu:
d['##features#cpu_freq#min'] = 2000
d['##features#freq#min'] = d['##features#cpu_freq#min']
else:
d['##features#gpu_freq#min'] = 1600
d['##features#freq#min'] = d['##features#gpu_freq#min']
# Save updated dict
r = ck.save_json_to_file({
'json_file': p1,
'dict': d,
'sort_keys': 'yes'
})
if r['return'] > 0: return r
return {'return': 0}
# See CK LICENSE.txt for licensing details # See CK COPYRIGHT.txt for copyright details # # Developer: Grigori Fursin, [email protected], http://fursin.net, 2017 # import ck.kernel as ck import os import json extract_missing_features = False gcc = ['GCC 4.9.2', 'GCC 7.1.0'] # Load info file r = ck.load_json_file({'json_file': 'init_reactions_tmp_info.json'}) if r['return'] > 0: ck.err(r) info = r['dict'] all_classes = info['classes'] # Load features name r = ck.access({ 'action': 'load', 'module_uoa': 'module', 'data_uoa': 'program.static.features' }) if r['return'] > 0: ck.err(r) features = r['dict']['milepost_features_description'] milepost_normalization_feature = r['dict']['milepost_normalization_feature']
def do(i):
# List performance entries
r=ck.access({'action':'search',
'module_uoa':'experiment',
'data_uoa':'ck-request-asplos18-mobilenets-tvm-arm-performance-*',
'repo_uoa':'local'})
if r['return']>0: return r
lst=r['lst']
for q in lst:
duid=q['data_uid']
duoa=q['data_uoa']
path=q['path']
ck.out(duoa)
# Search matching accuracy entry
aduoa=duoa.replace('-performance-','-accuracy-')
r=ck.access({'action':'find',
'module_uoa':'experiment',
'data_uoa':aduoa,
'repo_uoa':'local'})
if r['return']>0: return r
apath=r['path']
# Checking points to aggregate
dperf=os.listdir(path)
for f in dperf:
if f.endswith('.flat.json'):
ck.out(' * '+f)
# Load performance file
p1=os.path.join(path, f)
r=ck.load_json_file({'json_file':p1})
if r['return']>0: return r
d=r['dict']
p2=os.path.join(path, f[:-10]+'.features_flat.json') # Features
r=ck.load_json_file({'json_file':p2})
if r['return']>0: return r
df=r['dict']
# Remove batch
del(df['##choices#env#CK_BATCH_COUNT'])
# Find matching features file to merge
dacc=os.listdir(apath)
matched=False
for af in dacc:
if af.endswith('.features_flat.json'):
r=ck.load_json_file({'json_file':os.path.join(apath,af)})
if r['return']>0: return r
adf=r['dict']
# Remove batch
del(adf['##choices#env#CK_BATCH_COUNT'])
# Compare dicts
r=ck.compare_dicts({'dict1':df, 'dict2':adf})
if r['return']>0: return r
if r['equal']=='yes':
matched=True
# Load accuracy data to merge
px=os.path.join(apath,af[:-19]+'.flat.json')
r=ck.load_json_file({'json_file':px})
if r['return']>0: return r
dd=r['dict']
# Merge keys
for k in dd:
if k.startswith('##characteristics#run#accuracy_top'):
d[k]=dd[k]
break
if not matched:
return {'return':1, 'error':'no match - strange'}
# Save updated dict
r=ck.save_json_to_file({'json_file':p1, 'dict':d, 'sort_keys':'yes'})
if r['return']>0: return r
return {'return':0}
def do(i):
# List performance entries
r=ck.access({'action':'search',
'module_uoa':'experiment',
'data_uoa':'mobilenets-performance-*'
# 'repo_uoa':'ck-request-asplos18-results'
})
if r['return']>0: return r
lst=r['lst']
for q in lst:
duid=q['data_uid']
duoa=q['data_uoa']
ruid=q['repo_uid']
path=q['path']
ck.out(duoa)
# Search matching accuracy entry
r=ck.access({'action':'load',
'module_uoa':'experiment',
'data_uoa':duid,
'repo_uoa':ruid})
if r['return']>0: return r
dd=r['dict']
ruid=r['repo_uid']
apath=r['path']
# Updating meta if needed
dd['meta']['scenario_module_uoa']='a555738be4b65860' # module:request.asplos18
dd['meta']['model_species']='07d4e7aa3750ddc6' # model.species:mobilenets
dd['meta']['dataset_species']='ImageNet' # dataset species (free format)
dd['meta']['dataset_size']=500 # number of images ...
dd['meta']['platform_species']='embedded' # embedded vs server (maybe other classifications such as edge)
dd['meta']['platform_peak_power']=4.5 #Watts
dd['meta']['platform_price']=239 # $
dd['meta']['platform_price_date']='20170425' # date
dd['meta']['artifact']='08da9685582866a0' # artifact description
dd['meta']['model_precision']='fp32'
dd['meta']['processed']='yes'
# Unified full name for some deps
ds=dd['meta']['deps_summary']
x=ds['weights']
r=ck.access({'action':'make_deps_full_name','module_uoa':'request.asplos18','deps':x})
if r['return']>0: return r
dd['meta']['model_design_name']=r['full_name']
x=ds['compiler']
r=ck.access({'action':'make_deps_full_name','module_uoa':'request.asplos18','deps':x})
if r['return']>0: return r
dd['meta']['compiler_name']=r['full_name']
x=ds['library']
r=ck.access({'action':'make_deps_full_name','module_uoa':'request.asplos18','deps':x})
if r['return']>0: return r
dd['meta']['library_name']=r['full_name']
# Updating entry
r=ck.access({'action':'update',
'module_uoa':'experiment',
'data_uoa':duid,
'repo_uoa':ruid,
'dict':dd,
'substitute':'yes',
'ignore_update':'yes',
'sort_keys':'yes'
})
if r['return']>0: return r
# Checking points to aggregate
os.chdir(path)
dperf=os.listdir(path)
for f in dperf:
if f.endswith('.cache.json'):
os.system('git rm -f '+f)
elif f.endswith('.flat.json'):
ck.out(' * '+f)
# Load performance file
p1=os.path.join(path, f)
r=ck.load_json_file({'json_file':p1})
if r['return']>0: return r
d=r['dict']
mult=d.get('##choices#env#CK_ENV_MOBILENET_WIDTH_MULTIPLIER#min','')
if mult==0.25: size=1990786
elif mult==0.5: size=5459810
elif mult==0.75: size=10498594
elif mult==1.0: size=17106694
else:
return {'return':1, 'error':'unknown width multiplier "'+str(mult)+'"'}
d['##features#model_size#min']=size
d['##features#gpu_freq#min']=807
d['##features#cpu_freq#min']=''
d['##features#freq#min']=d['##features#gpu_freq#min']
d['##features#processed#min']='yes'
# Add throughput (images/second)
tall=d.get('##characteristics#run#prediction_time_avg_s#all',[])
if len(tall)>0:
tnew=[]
for t in tall:
t1=1/t
tnew.append(t1)
r=ck.access({'action':'stat_analysis',
'module_uoa':'experiment',
'dict':d,
'dict1':{'##characteristics#run#inference_throughput':tnew}
})
if r['return']>0: return r
# Unify batch size
x=d.get('##choices#env#CK_BATCH_SIZE#min','')
if x!=None and x!='':
batch=int(x)
d['##features#batch_size#min']=batch
if batch==1:
# inference latency
d['##features#measuring_latency#min']='yes'
r=ck.access({'action':'stat_analysis',
'module_uoa':'experiment',
'dict':d,
'dict1':{'##characteristics#run#inference_latency':tall}
})
if r['return']>0: return r
# Save updated dict
r=ck.save_json_to_file({'json_file':p1, 'dict':d, 'sort_keys':'yes'})
if r['return']>0: return r
return {'return':0}
import ck.kernel as ck
r = ck.access('list #milepost* out=none')
if r['return'] > 0:
print 'Error: ' + r['error']
exit(1)
lst = r['lst']
r = ck.load_json_file({'json_file': 'convert2.json'})
if r['return'] > 0:
print 'Error: ' + r['error']
exit(1)
d1 = r['dict']
dtags = {}
for q in lst:
m = q['module_uoa']
d = q['data_uoa']
print('***********************************')
print(d)
r = ck.access({'action': 'load', 'module_uoa': m, 'data_uoa': d})
if r['return'] > 0:
print 'Error: ' + r['error']
exit(1)
dd = r['dict']
dm = r['info']
def update(i):
"""
Input: {
(force) [bool] - if True, force update
}
Output: {
return [int] - return code = 0 if success or >0 if error
(error) [str] - error string if return>0
}
"""
import os
global bootstrapping
bootstrapping = True
force = i.get('force')
cfg = i.get('cfg', {})
from . import obj
title = 'Bootstrapping'
if cfg.get('bootstrapped', '') == 'yes': title = 'Updating'
ck.out(title + ' cBench to support portable actions and workflows:')
ck.out('')
# Check release notes
server_url = cfg.get('server_url', '')
if server_url == '': server_url = 'https://cKnowledge.io/api/v1/?'
from . import comm_min
r = comm_min.send({
'url': server_url,
'action': 'event',
'dict': {
'type': 'get-cbench-bootstrap-notes'
}
})
notes = r.get('notes', '')
if notes != '':
ck.out('***********************************************')
ck.out(notes)
ck.out('***********************************************')
lst_all = []
sbf = os.environ.get('CB_SAVE_BOOTSTRAP_FILES', '')
if sbf == '':
fboot = 'cb-bootstrap-20200529'
files = [fboot + '.json']
if os.name == 'nt':
files.append(fboot + '-win.json')
for fn in files:
r = ck.gen_tmp_file({'prefix': 'cb-bootstrap-', 'suffix': '.json'})
if r['return'] > 0: return r
ftmp = r['file_name']
burl = CR_DEFAULT_SERVER + '/static/bootstrap/' + fn
ck.out('Downloading ' + burl)
from . import comm
rx = comm.download_file({'url': burl, 'file': ftmp})
if rx['return'] > 0: return rx
rx = ck.load_json_file({'json_file': ftmp})
if rx['return'] > 0: return rx
lst_all += rx['dict']
os.remove(ftmp)
r = obj.download({'components': lst_all, 'force': force})
if r['return'] > 0 and r['return'] != 8: return r
else:
for x in CR_SOLUTION_CK_COMPONENTS:
r = obj.download({
'cid': x['cid'],
'version': x.get('version', ''),
'force': force
})
if r['return'] > 0:
if r['return'] != 8: return r
else: ck.out(' Skipped - already exists!')
else:
lst_all += r['components']
rx = ck.save_json_to_file({
'json_file': sbf,
'dict': lst_all,
'sort_keys': 'yes'
})
if rx['return'] > 0: return rx
ck.out('')
# Update cfg
cfg['bootstrapped'] = 'yes'
ii = {
'action': 'update',
'repo_uoa': CK_CFG_REPO_UOA,
'module_uoa': CK_CFG_MODULE_UID,
'data_uoa': CK_CFG_DATA_UOA,
'dict': cfg,
'sort_keys': 'yes'
}
r = ck.access(ii)
ck.out(title + ' finished!')
ck.out('')
return r
