def do(i): # List performance entries r=ck.access({'action':'search', 'module_uoa':'experiment', 'data_uoa':'mobilenets-performance-*' # 'repo_uoa':'ck-request-asplos18-results' }) if r['return']>0: return r lst=r['lst'] for q in lst: duid=q['data_uid'] duoa=q['data_uoa'] ruid=q['repo_uid'] path=q['path'] ck.out(duoa) # Search matching accuracy entry r=ck.access({'action':'load', 'module_uoa':'experiment', 'data_uoa':duid, 'repo_uoa':ruid}) if r['return']>0: return r dd=r['dict'] ruid=r['repo_uid'] apath=r['path'] # Updating meta if needed dd['meta']['scenario_module_uoa']='a555738be4b65860' # module:request.asplos18 dd['meta']['model_species']='07d4e7aa3750ddc6' # model.species:mobilenets dd['meta']['dataset_species']='ImageNet' # dataset species (free format) dd['meta']['dataset_size']=500 # number of images ... dd['meta']['platform_species']='embedded' # embedded vs server (maybe other classifications such as edge) dd['meta']['platform_peak_power']=4.5 #Watts dd['meta']['platform_price']=239 # $ dd['meta']['platform_price_date']='20170425' # date dd['meta']['artifact']='08da9685582866a0' # artifact description dd['meta']['model_precision']='fp32' dd['meta']['processed']='yes' # Unified full name for some deps ds=dd['meta']['deps_summary'] x=ds['weights'] r=ck.access({'action':'make_deps_full_name','module_uoa':'request.asplos18','deps':x}) if r['return']>0: return r dd['meta']['model_design_name']=r['full_name'] x=ds['compiler'] r=ck.access({'action':'make_deps_full_name','module_uoa':'request.asplos18','deps':x}) if r['return']>0: return r dd['meta']['compiler_name']=r['full_name'] x=ds['library'] r=ck.access({'action':'make_deps_full_name','module_uoa':'request.asplos18','deps':x}) if r['return']>0: return r dd['meta']['library_name']=r['full_name'] # Updating entry r=ck.access({'action':'update', 'module_uoa':'experiment', 'data_uoa':duid, 'repo_uoa':ruid, 'dict':dd, 'substitute':'yes', 'ignore_update':'yes', 'sort_keys':'yes' }) if r['return']>0: return r # Checking points to aggregate os.chdir(path) dperf=os.listdir(path) for f in dperf: if f.endswith('.cache.json'): os.system('git rm -f '+f) elif f.endswith('.flat.json'): ck.out(' * '+f) # Load performance file p1=os.path.join(path, f) r=ck.load_json_file({'json_file':p1}) if r['return']>0: return r d=r['dict'] mult=d.get('##choices#env#CK_ENV_MOBILENET_WIDTH_MULTIPLIER#min','') if mult==0.25: size=1990786 elif mult==0.5: size=5459810 elif mult==0.75: size=10498594 elif mult==1.0: size=17106694 else: return {'return':1, 'error':'unknown width multiplier "'+str(mult)+'"'} d['##features#model_size#min']=size d['##features#gpu_freq#min']=807 d['##features#cpu_freq#min']='' d['##features#freq#min']=d['##features#gpu_freq#min'] d['##features#processed#min']='yes' # Add throughput (images/second) tall=d.get('##characteristics#run#prediction_time_avg_s#all',[]) if len(tall)>0: tnew=[] for t in tall: t1=1/t tnew.append(t1) r=ck.access({'action':'stat_analysis', 'module_uoa':'experiment', 'dict':d, 'dict1':{'##characteristics#run#inference_throughput':tnew} }) if r['return']>0: return r # Unify batch size x=d.get('##choices#env#CK_BATCH_SIZE#min','') if x!=None and x!='': batch=int(x) d['##features#batch_size#min']=batch if batch==1: # inference latency d['##features#measuring_latency#min']='yes' r=ck.access({'action':'stat_analysis', 'module_uoa':'experiment', 'dict':d, 'dict1':{'##characteristics#run#inference_latency':tall} }) if r['return']>0: return r # Save updated dict r=ck.save_json_to_file({'json_file':p1, 'dict':d, 'sort_keys':'yes'}) if r['return']>0: return r return {'return':0}
def main(arg): # CUSTOMIZABLE VARIABLES!!!! module_uoa = 'experiment' repo_uoa = 'explore-matrix-size-xgemm-fp32-firefly-rk3399' tags='explore-clblast-matrix-size' output_filename = 'tmp-ck-clblast-tune.json' weights_filename='NULL' WEIGHTS = 0 if arg.fp is not None: weights_filename = arg.fp if (os.path.isfile(weights_filename)): print("{RANKING ERROR %s not found. USE WEIGHTS=0}" %(weights_filename)) WEIGHTS = 1 else: print("[RANKING ERROR] %s not found. USE WEIGHTS=0" %(weights_filename)) ### END CUST dlist=[] dblist=[] r=ck.access({'action':'search', 'repo_uoa':repo_uoa, 'module_uoa':module_uoa, 'tags':tags}) if r['return']>0: print ("Error: %s" % r['error']) experiments=r['lst'] if len(experiments) == 0: print("No experiments found in repo %s with tags %s" %(repo_uoa, tags)) exit(1) for exp in experiments: # print exp data_uoa = exp['data_uoa'] r = ck.access({'action':'list_points', 'repo_uoa':repo_uoa, 'module_uoa':module_uoa, 'data_uoa':data_uoa}) if r['return']>0: print ("Error: %s" % r['error']) exit(1) tags = r['dict']['tags'] # print tags npoint = len(r['points']) print ("[RANKING] Number of matrices: %s" %(npoint)) # print npoint for point in r['points']: point_file = os.path.join(r['path'], 'ckp-%s.0001.json' % point) d = get_data(point_file) dlist.append(d) # LOAD WEIGHTS w = [] # WEIGHTS = os.path.isfile(weights_filename) and WEIGHTS if (WEIGHTS == 0): for i in range(0,npoint): w.append(1) else: print("Loading weights %s" %(weights_filename)) wdict = json.loads(open(weights_filename).read()) for i in wdict: print i.get('Execution time (%)') w.append(float(i.get('Execution time (%)'))) output = get_data_header(point_file) # FIND THE BEST #configurations = dlist[0] #best = sys.float_info.max #configuration_best_index = len(configurations) #configuration_count = 0 #final_score = [] #for configuration in configurations: # res = find_it(configuration['parameters'],dlist) # tmp_time = 0 # for ctime in res["time"]: # tmp_time +=ctime # if tmp_time < best: # best= tmp_time # configuration_best_index = configuration_count # res["total_time"] = tmp_time # final_score.append(res) # configuration_count +=1 #mbest = final_score[configuration_best_index] mbest= find_best(dlist, w) ### PREPARE OUTPUT del mbest['time'] del mbest['total_time'] mbest['GFLOPS'] = 0.0 mbest['kernel'] = output.get('kernel') output['data'] = 'na' output['db'] = 'na' output['statistics'] = {'default_family':{}, 'default_configuration':{}, 'best_configuration': mbest } #print (json.dumps(output, indent=2)) rr=ck.save_json_to_file({'json_file':output_filename, 'dict':output})
def main(i): # Load common table file (for all models) ddd = {} r = ck.load_json_file({'json_file': 'save_all_model_data_tmp.json'}) if r['return'] == 0: ddd = r['dict'] # Searching for features for g in gcc: ck.out( '********************************************************************************' ) ck.out('Modeling optimizations for ' + g) ck.out('') if g not in ddd: ddd[g] = {} gx = g.replace(' ', '_') r = ck.load_json_file( {'json_file': 'prepare_train_data_tmp.' + gx + '.json'}) if r['return'] > 0: ck.err(r) d = r['dict'] ftable = d['ftable'] ctable = d['ctable'] # Normalize (all features 0..1) ftable_range = {} for f in ftable: for k in range(0, 121): v = f[k] if k not in ftable_range: ftable_range[k] = {'min': None, 'max': None} if ftable_range[k]['min'] == None or v < ftable_range[k]['min']: ftable_range[k]['min'] = v if ftable_range[k]['max'] == None or v > ftable_range[k]['max']: ftable_range[k]['max'] = v ftable_normalized = [] for f in ftable: x = [] for k in range(0, 121): v = 0 if ftable_range[k]['max'] != 0: v = f[k] / ftable_range[k]['max'] x.append(v) ftable_normalized.append(x) # ft1..ft56 for ft in [[0, 56], [0, 65], [0, 121], [56, 65], [56, 121], [65, 121]]: ft_start = ft[0] ft_stop = ft[1] ext = 'ft' + str(ft_start + 1) + '_' + 'ft' + str(ft_stop) ck.out('') ck.out('Using non-normalized features ' + ext + ' ...') r = model({ 'ftable': ftable, 'ctable': ctable, 'ft_start': ft_start, 'ft_stop': ft_stop }) if r['return'] > 0: return r ddd[g]['milepost_' + ext] = r['accuracy'] r1 = ck.save_json_to_file({ 'json_file': 'process_model_using_nearest_neighbour/process_model_using_nearest_neighbour_' + ext + '_tmp.' + gx + '.json', 'dict': r }) if r1['return'] > 0: ck.err(r1) # Normalized ft1..ft56 ck.out('') ck.out('Using normalized features ' + ext + ' ...') r = model({ 'ftable': ftable_normalized, 'ctable': ctable, 'ft_start': ft_start, 'ft_stop': ft_stop }) if r['return'] > 0: return r r1 = ck.save_json_to_file({ 'json_file': 'process_model_using_nearest_neighbour/process_model_using_nearest_neighbour_' + ext + '_tmp.' + gx + '.normalized.json', 'dict': r }) if r1['return'] > 0: ck.err(r1) ddd[g]['milepost_normalized_' + ext] = r['accuracy'] # Save common data r = ck.save_json_to_file({ 'json_file': 'save_all_model_data_tmp.json', 'dict': ddd }) if r['return'] > 0: return r return {'return': 0}
label = label + 1 # Prepare features fkeys = [] ffkd = {} for q in range(1, 65 + 1): ffkd[q] = {'name': features[str(q)]['desc']} # Add real name fkeys.append('ft' + str(q)) for q in range(1, 56 + 1): q1 = 65 + q fkeys.append('ft' + str(q1)) ffkd[q1] = { 'name': 'Normalized ' + features[str(q)]['desc'] + ' (by ft24)' } # Add real name # Recording input for model such as TensorFlow DNN r = ck.save_json_to_file({ 'json_file': 'prepare_train_data_tmp.' + gx + '.json', 'dict': { 'ftable': ftable, 'fkeys': fkeys, 'features_flat_keys_desc': ffkd, 'ctable': ctable, "ckeys": ["Optimization class"], 'optimization_classes': labels } }) if r['return'] > 0: ck.err(r)
def init(i): """ Input: { uid [str] - graph identifyer (version) [str] - graph version (desc_file) [str] - file with graph description (tags) [str] - tags separated by comma } Output: { return [int] - return code = 0 if success or >0 if error (error) [str] - error string if return>0 dict [dict] - configuration dictionary path [str] - path to CK cfg entry } """ # Get main configuration r=config.load({}) if r['return']>0: return r cfg=r.get('dict',{}) pcfg=r.get('path','') # CID ########################################################### uid=i['uid'] if uid==None: uid='' version=i.get('version') if version==None: version='' desc_file=i.get('desc_file','') if desc_file==None: desc_file='' # If UID!='', check if already exists ... found=False meta=meta_template path='' data_name='' tags=[] meta_info='' source='' extra_info={} if uid!='': r=ck.access({'action':'load', 'module_uoa':'result', 'data_uoa':uid}) if r['return']>0: if r['return']!=16: return r else: found=True meta=r['dict'] path=r['path'] data_name=r['data_name'] tags=meta.get('tags',[]) source=meta.get('source','') meta_info=meta.get('meta',{}).get('info','') extra_info=r['info'].get('control',{}) # Check if init from scratch and no title if i.get('name')!=None and i.get('name','')!='': data_name=i['name'].strip() elif not found or data_name=='': r=ck.inp({'text':'Select a title for your graph: '}) if r['return']>0: return r data_name=r['string'].strip() meta['meta']['title']=data_name # Check if init from scratch and no title if not found or meta_info=='': r=ck.inp({'text':'Enter general info about your graph: '}) if r['return']>0: return r x=r['string'].strip() if x=='': x=' ' meta['meta']['info']=x # Adding tags if i.get('tags')!=None and i.get('tags','')!='': xtags=i['tags'].strip().split(',') for t in xtags: t1=t.strip() if t1!='' and t1 not in tags: tags.append(t1) meta['tags']=tags elif not found or (len(tags)==1 and 'result' in tags): r=ck.inp({'text':'Enter tags for your graph separated by commas: '}) if r['return']>0: return r xtags=r['string'].strip().split(',') for t in xtags: t1=t.strip() if t1!='' and t1 not in tags: tags.append(t1) meta['tags']=tags # Checking source if not found or source=='': r=ck.inp({'text':'Enter source of results for your graph (can be URL): '}) if r['return']>0: return r source=r['string'].strip() meta['source']=source # Checking authors for x in extra_info_desc: k=x['key'] n=x['name'] if not found or extra_info.get(k,'')=='': r=ck.inp({'text':'Enter '+n+': '}) if r['return']>0: return r s=r['string'].strip() extra_info[k]=s # Creating/updating graph a='add' if found: a='update' ii={'action':a, 'module_uoa':'result', 'data_uoa':uid, 'dict':meta, 'sort_keys':'yes', 'data_name':data_name, 'substitute':'yes', 'extra_info':extra_info} r=ck.access(ii) if r['return']>0: return r data_uoa=r['data_uoa'] data_uid=r['data_uid'] path=r['path'] x='initialized' if found: x='updated' ck.out('Graph was successfully '+x+':') ck.out('') ck.out(' CK UID: '+data_uid) ck.out(' CK name: '+data_uoa) ck.out(' CK path: '+path) # Add desc p1=os.path.join(path, 'desc.json') dt=copy.deepcopy(desc_template) if desc_file!='': rx=ck.load_json_file({'json_file':desc_file}) if rx['return']>0: return rx dx=rx['dict'] dt['data_config'].update(dx) if desc_file!='' or not os.path.isfile(p1): rx=ck.save_json_to_file({'json_file':p1, 'dict':dt, 'sort_keys':'yes'}) if rx['return']>0: return rx p2=os.path.join(path, '.cm', 'meta.json') ck.out('') ck.out('You can continue updating graph using following files: ') ck.out('') ck.out(' Graph general meta info: '+p1) ck.out(' See example at '+config.CR_DEFAULT_SERVER+'/result/sota-mlperf-inference-results-v0.5-open-available/?action=download&filename=.cm/meta.json') ck.out('') ck.out(' Graph axes info: '+p2) ck.out(' See example at '+config.CR_DEFAULT_SERVER+'/result/sota-mlperf-inference-results-v0.5-open-available/?action=download&filename=desc.json') # Need to publish ck.out('') rx=ck.inp({'text':'Publish graph on the portal (Y/n)?'}) if rx['return']>0: return rx s=rx['string'].strip().lower() if s=='' or s=='y': ck.out('') r=obj.publish({'cid':'result:'+data_uoa, 'version':version, 'force':True}) else: ck.out('') ck.out('You can publish your graph on the portal using the following commands when ready: ') ck.out('') ck.out(' cb publish result:'+data_uoa+' --version=1.0.0 --force (--private)') return r
sorted_heatmap2.append([ibench,iopt,reaction]) ibench+=1 # Move all without improvements to the end for bx in bench_index: sbx=str(bx) if sbx not in remapping: remapping[sbx]=ibench ibench+=1 # remap for h in heatmap: bench=h[0] iopt=h[1] reaction=h[2] x=[remapping[str(bench)],iopt,reaction] sorted_heatmap.append(x) # save dd['table']['0']=sorted_heatmap r=ck.save_json_to_file({'json_file':fn+'_clustered.json','dict':dd,'sort_keys':'yes'}) if r['return']>0: ck.err(r) # save dd['table']['0']=sorted_heatmap2 r=ck.save_json_to_file({'json_file':fn+'_clustered2.json','dict':dd,'sort_keys':'yes'}) if r['return']>0: ck.err(r)
def main(i): cur_dir = os.getcwd() fas = os.path.join(cur_dir, aggregated_stats) # Get some info about current platform ii = {'action': 'detect', 'module_uoa': 'platform', 'out': 'con'} r = ck.access(ii) if r['return'] > 0: return r hos = r['host_os_uid'] hosx = r['host_os_uoa'] hosd = r['host_os_dict'] tos = r['os_uid'] tosx = r['os_uoa'] tosd = r['os_dict'] cpu_name = r['features']['cpu']['name'] plat_name = r['features']['platform']['name'] ############################################################# ck.out(line) ck.out('CPU name: ' + cpu_name) ck.out('Plat name: ' + plat_name) ############################################################# ck.out(line) ck.out('Loading aggregated stats ...') aa = [] if os.path.isfile(fas): r = ck.load_json_file({'json_file': fas}) if r['return'] > 0: return r ax = r['dict'] if 'all' not in ax: ax['all'] = [] aa = ax['all'] ############################################################# ck.out(line) ck.out('Finding entry related to this platform ...') found = False for a in aa: if a.get('cpu_name', '') == cpu_name and a.get('plat_name', '') == plat_name: found = True if not found: a = {'cpu_name': cpu_name, 'plat_name': plat_name} aa.append(a) if 'data' not in a: a['data'] = {} data = a.get('data', {}) # Init pipeline r = ck.access({ 'action': 'pipeline', 'module_uoa': 'program', 'data_uoa': 'shared-matmul-c2', 'cpu_freq': 'max', 'gpu_freq': 'max', 'speed': 'yes', 'compiler_vars': { 'USE_BLOCKED_MATMUL': 'YES' }, 'no_state_check': 'yes', 'prepare': 'yes', 'out': 'con' }) if r['return'] > 0: return r ready = r['ready'] if ready != 'yes': return {'return': 1, 'error': 'can\'t init pipeline'} pipeline = r # Compile program ones tpipeline = copy.deepcopy(pipeline) r = ck.access({ 'action': 'autotune', 'module_uoa': 'pipeline', 'pipeline': pipeline, 'pipeline_update': { 'env': { 'CT_MATRIX_DIMENSION': 16, 'CT_BLOCK_SIZE': 16 } }, 'iterations': 1, 'repetitions': 1, 'out': 'con' }) if r['return'] > 0: return r lsa = r.get('last_stat_analysis', {}).get('dict_flat', {}) time_min = lsa.get('##characteristics#run#execution_time#min', None) if time_min == None or time_min == 0.0: return {'return': 1, 'error': 'failed to run default pipeline'} # data is per N size while True: # continue infinite loop until stopping ck.out(line) n = random.randint(0, 3) # Matrix size generator if n == 0: N = random.randint(4, 1024) # Matrix size else: NX = random.randint(2, 10) N = 2**NX if n == 2: N = N - 1 if n == 3: N = N + 1 SN = str(N) if SN not in data: data[SN] = {} xdata = data.get(SN, {}) tmin = xdata.get('tmin', None) tmax = xdata.get('tmax', None) gmin = xdata.get('gmin', None) gmax = xdata.get('gmax', None) best_tile = xdata.get('best_tile', None) for opts in range(0, 16): # Choose if random BS or power of two or power of two -+1 if opts == 0: BS = 1 elif opts == 1: BS = N else: b = random.randint(0, 3) if b == 0: BS = random.randint(1, N) else: B1 = math.frexp(N)[1] - 1 B2 = random.randint(0, B1) BS = 2**B2 if b == 2 and BS > 1: BS = BS - 1 elif b == 3 and BS < N - 1: BS = BS + 1 ck.out('Matrix size: ' + str(N)) ck.out('Tile size: ' + str(BS)) # Run pipeline tpipeline = copy.deepcopy(pipeline) r = ck.access({ 'action': 'autotune', 'module_uoa': 'pipeline', 'pipeline': pipeline, 'pipeline_update': { 'no_compile': 'yes', 'env': { 'CT_MATRIX_DIMENSION': N, 'CT_BLOCK_SIZE': BS } }, 'iterations': 1, 'repetitions': 3, 'out': 'con' }) if r['return'] > 0: return r lsa = r.get('last_stat_analysis', {}).get('dict_flat', {}) time_min = lsa.get('##characteristics#run#execution_time#min', None) changed = False if time_min != None: ops = 2 * (N * N * N) if tmin == None or time_min < tmin: tmin = time_min best_tile = BS gmax = 1.0e-9 * ops / tmin changed = True if tmax == None or time_min > tmax: tmax = time_min gmin = 1.0e-9 * ops / tmax changed = True if changed: xdata['tmin'] = tmin xdata['tmax'] = tmax xdata['gmin'] = gmin xdata['gmax'] = gmax xdata['best_tile'] = best_tile if opts == 0: xdata['tbs1'] = time_min xdata['gbs1'] = 1.0e-9 * ops / time_min changed = True elif opts == 1: xdata['tbsn'] = time_min xdata['gbsn'] = 1.0e-9 * ops / time_min xdata['bsn'] = N changed = True if changed: ck.out(line) ck.out('Saving aggregated stats ...') r = ck.save_json_to_file({ 'json_file': fas, 'dict': { 'all': aa }, 'sort_keys': 'yes' }) if r['return'] > 0: return r ############################################################# ck.out(line) ck.out('Saving aggregated stats ...') r = ck.save_json_to_file({ 'json_file': fas, 'dict': { 'all': aa }, 'sort_keys': 'yes' }) if r['return'] > 0: return r return {'return': 0}
def main(i): # Load common table file (for all models) ddd = {} r = ck.load_json_file({'json_file': 'save_all_model_data_tmp.json'}) if r['return'] == 0: ddd = r['dict'] # Searching for features for g in gcc: ck.out( '********************************************************************************' ) ck.out('Modeling optimizations for ' + g) ck.out('') if g not in ddd: ddd[g] = {} gx = g.replace(' ', '_') r = ck.load_json_file( {'json_file': 'prepare_train_data_tmp.' + gx + '.json'}) if r['return'] > 0: ck.err(r) d = r['dict'] ftable = d['ftable'] ctable = d['ctable'] s = '' for depth in range(1, 30): # Full cross-validation acc = 0 obs = 0 wrong = 0 acc_min = None acc_max = None s = '==============================================================\n' s += 'Depth: ' + str(depth) + '\n\n' ck.out(s) for n in range( 0, 3 ): # Trying to build model N times (random - sometimes slightly different result) # Building decision tree on all data ii = { 'action': 'build', 'module_uoa': 'model', 'ftable': ftable, 'ctable': ctable, 'keep_temp_files': 'yes', "model_module_uoa": "model.sklearn", "model_name": "dtc", "model_file": "process_model_using_decision_trees/model-sklearn-dtc-" + gx + '-depth' + str(depth), "model_params": { "max_depth": depth }, "out": "" } # Training cii = copy.deepcopy(ii) r = ck.access(ii) if r['return'] > 0: ck.err(r) # Validating ii = copy.deepcopy(ii) ii['action'] = 'validate' r = ck.access(ii) if r['return'] > 0: ck.err(r) obs += r['observations'] wrong += r['mispredictions'] acc = float(obs - wrong) / float(obs) x = ' Accuracy on all data (' + str( n + 1) + ' out of 3): ' + str(acc) s += x ck.out(x) acc = float(obs - wrong) / float(obs) if acc_min == None or acc < acc_min: acc_min = acc if acc_max == None or acc > acc_max: acc_max = acc x = '\nDepth: ' + str( depth ) + ' ; accuracy (min/max): ' + '%.2f' % acc_min + ' .. ' + '%.2f' % acc_max + '\n' s += x ck.out(x) # Cross-validating (for simplicity 1 run) cross_obs = 0 cross_wrong = 0 x = ' *************************************************\n' x += ' Cross-validating model (leave one out)\n\n' s += x ck.out(x) for bench in range(0, len(ftable)): train_ftable = [] train_ctable = [] test_ftable = [] test_ctable = [] for k in range(0, len(ftable)): if k != bench: train_ftable.append(ftable[k]) train_ctable.append(ctable[k]) else: test_ftable.append(ftable[k]) test_ctable.append(ctable[k]) # Selecting model ii = { 'action': 'build', 'module_uoa': 'model', 'ftable': train_ftable, 'ctable': train_ctable, 'keep_temp_files': 'no', "model_module_uoa": "model.sklearn", "model_name": "dtc", "model_file": "tmp-model-sklearn-dtc", "model_params": { "max_depth": depth }, "out": "" } # Training cii = copy.deepcopy(ii) r = ck.access(ii) if r['return'] > 0: ck.err(r) # Validating ii = copy.deepcopy(ii) ii['action'] = 'validate' ii['ftable'] = test_ftable ii['ctable'] = test_ctable r = ck.access(ii) if r['return'] > 0: ck.err(r) cross_obs += r['observations'] cross_wrong += r['mispredictions'] cross_acc = float(cross_obs - cross_wrong) / float(cross_obs) x = ' ' + str(bench) + ' out of ' + str( len(ftable) ) + ' ) current cross-validation accuracy: ' + '%.2f' % cross_acc s += x ck.out(x) x = '\nDepth: ' + str( depth ) + ' ; accuracy (with cross-validation): ' + '%.2f' % cross_acc + '\n' s += x ck.out(x) ddd[g]['decision_trees_with_cross_validation_depth_' + str(depth) + '_ft1_ft65'] = cross_acc ddd[g]['decision_trees_without_cross_validation_depth_' + str(depth) + '_ft1_ft65'] = acc_max r = ck.save_text_file({ 'text_file': 'process_model_using_decision_trees/log.' + gx + '.txt', 'string': s }) if r['return'] > 0: return r # Save common data (continuously, not to loose data) r = ck.save_json_to_file({ 'json_file': 'save_all_model_data_tmp.json', 'dict': ddd }) if r['return'] > 0: return r return {'return': 0}
def do(i): # List performance entries r = ck.access({ 'action': 'search', 'module_uoa': 'experiment', 'data_uoa': 'ck-request-asplos18-tvm-fpga-performance-*' # 'repo_uoa':'ck-request-asplos18-results' }) if r['return'] > 0: return r lst = r['lst'] for q in lst: duid = q['data_uid'] duoa = q['data_uoa'] ruid = q['repo_uid'] path = q['path'] ck.out(duoa) # Search matching accuracy entry r = ck.access({ 'action': 'load', 'module_uoa': 'experiment', 'data_uoa': duid, 'repo_uoa': ruid }) if r['return'] > 0: return r dd = r['dict'] ruid = r['repo_uid'] apath = r['path'] # Updating meta if needed dd['meta'][ 'scenario_module_uoa'] = 'a555738be4b65860' # module:request.asplos18 dd['meta'][ 'model_species'] = 'd41bbf1e489ab5e0' # model.species:resnet18 dd['meta'][ 'dataset_species'] = 'ImageNet' # dataset species (free format) dd['meta']['dataset_size'] = 2000 # number of images ... dd['meta'][ 'platform_species'] = 'fpga' # embedded vs server vs fpga (maybe other classifications such as edge) dd['meta']['platform_peak_power'] = 2.5 #Watts dd['meta']['platform_price'] = 229 # $ dd['meta']['platform_price_date'] = '20180404' # date dd['meta']['artifact'] = '9375838469ad4029' # artifact description dd['meta']['model_precision'] = 'int8' dd['meta']['processed'] = 'yes' # Unified full name for some deps ds = dd['meta']['deps_summary'] x = ds['model'] r = ck.access({ 'action': 'make_deps_full_name', 'module_uoa': 'request.asplos18', 'deps': x }) if r['return'] > 0: return r dd['meta']['model_design_name'] = r['full_name'] dd['meta']['plat_name'] = 'Xilinx PYNQ-Z1 FPGA (ZYNQ XC7Z020-1CLG400C)' dd['meta']['os_name'] = 'Ubuntu 15.10' dd['meta'][ 'cpu_name'] = 'Programmable logic equivalent to Artix-7 FPGA' # Updating entry r = ck.access({ 'action': 'update', 'module_uoa': 'experiment', 'data_uoa': duid, 'repo_uoa': ruid, 'dict': dd, 'substitute': 'yes', 'ignore_update': 'yes', 'sort_keys': 'yes' }) if r['return'] > 0: return r # Checking points to aggregate os.chdir(path) dperf = os.listdir(path) for f in dperf: if f.endswith('.cache.json'): os.system('git rm -f ' + f) elif f.endswith('.flat.json'): ck.out(' * ' + f) # Load performance file p1 = os.path.join(path, f) r = ck.load_json_file({'json_file': p1}) if r['return'] > 0: return r d1 = r['dict'] # Prune some old value d = {} for k in d1: if not k.startswith('##characteristics#run#accuracy_top1') and \ not k.startswith('##characteristics#run#accuracy_top5') and \ not k.startswith('##characteristics#run#inference_throughput') and \ not k.startswith('##characteristics#run#inference_latency'): d[k] = d1[k] d['##features#model_size#min'] = 129770000 # Bytes d['##features#gpu_freq#min'] = 100 d['##features#cpu_freq#min'] = '' d['##features#freq#min'] = d['##features#gpu_freq#min'] d['##features#processed#min'] = 'yes' # Add throughput (images/second) tall = d.get( '##characteristics#run#execution_time_classify_internal#all', []) # It's internal VTA measurements if len(tall) > 0: tnew = [] for t in tall: t1 = 1 / t tnew.append(t1) r = ck.access({ 'action': 'stat_analysis', 'module_uoa': 'experiment', 'dict': d, 'dict1': { '##characteristics#run#inference_throughput': tnew } }) if r['return'] > 0: return r # Unify batch size batch = 1 # for now only 1 is supported in this artifact d['##features#batch_size#min'] = batch # inference latency d['##features#measuring_latency#min'] = 'yes' r = ck.access({ 'action': 'stat_analysis', 'module_uoa': 'experiment', 'dict': d, 'dict1': { '##characteristics#run#inference_latency': tall } }) if r['return'] > 0: return r r = ck.access({ 'action': 'stat_analysis', 'module_uoa': 'experiment', 'dict': d, 'dict1': { '##characteristics#run#prediction_time_avg_s': tall } }) if r['return'] > 0: return r # Add accuracy (was calculated through separate experiment) r = ck.access({ 'action': 'stat_analysis', 'module_uoa': 'experiment', 'dict': d, 'dict1': { '##characteristics#run#accuracy_top1': [accuracy_top1] } }) if r['return'] > 0: return r # Add accuracy (was calculated through separate experiment) r = ck.access({ 'action': 'stat_analysis', 'module_uoa': 'experiment', 'dict': d, 'dict1': { '##characteristics#run#accuracy_top5': [accuracy_top5] } }) if r['return'] > 0: return r # Save updated dict r = ck.save_json_to_file({ 'json_file': p1, 'dict': d, 'sort_keys': 'yes' }) if r['return'] > 0: return r return {'return': 0}
heatmap=heatmaps[hm] dd={'table':{"0":heatmap}} xx=[] iq=0 for k in uopts[comp]: xx.append(iq) iq+=1 dd['axis_y_labels']=xx name='init_reactions_tmp_heatmap_'+comp.replace(' ','_')+'_'+hm1 r=ck.save_json_to_file({'json_file':name+'.json','dict':dd,'sort_keys':'yes'}) if r['return']>0: ck.err(r) # Plot ii.update(dd) ii['out_to_file']=name+'.pdf' # Check number of labels x=[] for iq in range(0,len(datasets[hm])+1): x.append(iq) ii['axis_x_labels']=x r=ck.access(ii)
def main(i): # Load common table file (for all models) r = ck.load_json_file({'json_file': 'save_all_model_data_tmp.json'}) if r['return'] > 0: return r d = r['dict'] # Get all models/features kk = list(d['GCC 4.9.2']) table = [] table_milepost = [] table_short = [] ikk = [] for k in kk: sort = 0 k1 = k j = k.find('_depth_') if j > 0: j1 = k.find('_', j + 7) if j1 > 0: sort = int(k[j + 7:j1]) k1 = k[:j] j = k.find('_iteration_') if sort == '' and j > 0: j1 = k.find('_', j + 11) if j1 > 0: sort = int(k[j + 11:j1]) k1 = k[:j] ikk.append([k1, sort, k]) for k1 in sorted(ikk, key=lambda x: (x[0], x[1])): k = k1[2] norm = False j = k.find('_ft') j1 = k.find('_normalized') if j1 > 0: norm = True j = j1 km = k[:j].replace('_', ' ') if km.startswith('milepost'): ext = km[9:] if ext != '': ext = ' (' + ext + ')' km = 'milepost nn' + ext km = km.replace(' depth', '; depth') km = km.replace(' iteration', '; iteration') kf = k[j + 1:] if norm: kf = kf[11:] + '\\newline' + '(normalized)' kf = kf.replace('_', ' .. ') a4 = "%.2f" % d['GCC 4.9.2'][k] a7 = "%.2f" % d['GCC 7.1.0'][k] # Full table (for interactive report) line = [km, kf, a4, a7] table.append(line) # Shorter version for paper - ugly but didn't have time to make it nicer ;) if 'depth 3' not in km and \ 'depth 5' not in km and \ 'depth 6' not in km and \ 'depth 7' not in km and \ 'depth 9' not in km and \ 'depth 10' not in km and \ 'depth 11' not in km and \ 'depth 12' not in km and \ 'depth 13' not in km and \ 'depth 14' not in km and \ 'depth 15' not in km and \ 'depth 17' not in km and \ 'depth 18' not in km and \ 'depth 19' not in km and \ 'depth 21' not in km and \ 'depth 22' not in km and \ 'depth 23' not in km and \ 'depth 24' not in km and \ 'depth 26' not in km and \ 'depth 27' not in km and \ 'depth 28' not in km and \ 'iteration 5' not in km and \ 'iteration 6' not in km and \ 'iteration 7' not in km and \ 'iteration 8' not in km and \ 'iteration 9' not in km: table_short.append(line) # Only short MILEPOST if km == 'milepost nn' and kf == 'ft1 .. ft56': table_milepost.append(line) dd = { "table_style": "border=\"1\"", "table_header": [{ "name": "Model", "html_before": "<b>", "html_after": "</b>", "tex": "l", "tex_before": "\\textbf{", "tex_after": "}" }, { "name": "Features", "html_change_space": "yes", "tex": "p{1.2in}" }, { "name": "Accuracy (GCC 4.9.2)", "html_change_space": "yes", "tex": "p{0.9in}" }, { "name": "Accuracy (GCC 7.1.0)", "html_change_space": "yes", "tex": "p{0.9in}" }] } # Save full table file (for all models) dd['table'] = table r = ck.save_json_to_file({ 'json_file': 'save_all_model_data_tmp_table_full.json', 'dict': dd, 'sort_keys': 'yes' }) if r['return'] > 0: return r # Save common table file (for all models) dd['table'] = table_short r = ck.save_json_to_file({ 'json_file': 'save_all_model_data_tmp_table_short.json', 'dict': dd, 'sort_keys': 'yes' }) if r['return'] > 0: return r # Save common table file (for all models) dd['table'] = table_milepost r = ck.save_json_to_file({ 'json_file': 'save_all_model_data_tmp_table_milepost.json', 'dict': dd, 'sort_keys': 'yes' }) if r['return'] > 0: return r return {'return': 0}
def main(i): # Load common table file (for all models) ddd = {} r = ck.load_json_file({'json_file': 'save_all_model_data_tmp.json'}) if r['return'] == 0: ddd = r['dict'] # Searching for features for g in gcc: ck.out( '********************************************************************************' ) ck.out('Modeling optimizations for ' + g) ck.out('') if g not in ddd: ddd[g] = {} gx = g.replace(' ', '_') r = ck.load_json_file( {'json_file': 'prepare_train_data_tmp.' + gx + '.json'}) if r['return'] > 0: ck.err(r) d = r['dict'] ftable = d['ftable'] ctable = d['ctable'] # Normalize (all features 0..1) ftable_range = {} for f in ftable: for k in range(0, 121): v = f[k] if k not in ftable_range: ftable_range[k] = {'min': None, 'max': None} if ftable_range[k]['min'] == None or v < ftable_range[k]['min']: ftable_range[k]['min'] = v if ftable_range[k]['max'] == None or v > ftable_range[k]['max']: ftable_range[k]['max'] = v ftable_normalized = [] for f in ftable: x = [] for k in range(0, 121): v = 0 if ftable_range[k]['max'] != 0: v = f[k] / ftable_range[k]['max'] x.append(v) ftable_normalized.append(x) features_mask = [] for f in range(0, 121): features_mask.append(1) r = model({ 'ftable': ftable_normalized, 'features_mask': features_mask, 'ctable': ctable }) if r['return'] > 0: return r r1 = ck.save_json_to_file({ 'json_file': 'process_model_using_nearest_neighbour_reduce_features/prepare_reactions_model_train_ref_result.' + gx + '.json', 'dict': r }) if r1['return'] > 0: ck.err(r1) ref_acc = r['accuracy'] # Reference accuracy x = 'Reference accuracy: ' + str(ref_acc) s = x + '\n\n' ck.out('---------------------') ck.out(x) ck.out('') # Calculating Euclidean distance as in our MILEPOST GCC paper: https://hal.inria.fr/hal-00685276 # MILPOST features: https://github.com/ctuning/ck-autotuning/blob/master/module/program.static.features/.cm/meta.json # 0..55 - original MILEPOST features # 56..64 - added by Jeremy Singer # 65..121 - 0..55/ft24 (normalized by total number of instructions) for k in range(0, 121): features_mask[k] = 0 r = model({ 'ftable': ftable_normalized, 'features_mask': features_mask, 'ctable': ctable, 'skip_out': 'yes' }) if r['return'] > 0: return r acc = r['accuracy'] keep = False sx = '' if acc < ref_acc: keep = True sx = 'kept' elif acc == ref_acc: sx = 'removed' elif acc > ref_acc: ref_acc = acc sx = 'removed (accuracy even improved)' if keep: features_mask[k] = 1 x = 'ft' + str(k + 1) + ') ' + str( acc) + ' ' + sx + ' (ref acc=' + str(ref_acc) + ')' ck.out(x) s += x + '\n' # Final accuracy r = model({ 'ftable': ftable_normalized, 'features_mask': features_mask, 'ctable': ctable, 'skip_out': 'yes' }) if r['return'] > 0: return r acc = r['accuracy'] r1 = ck.save_json_to_file({ 'json_file': 'process_model_using_nearest_neighbour_reduce_features/prepare_reactions_model_train_reduced_result.' + gx + '.json', 'dict': r }) if r1['return'] > 0: ck.err(r1) # Final result ck.out('') ck.out('Final features mask:') ck.out('') s += '\nFinal features mask:\n\n' for f in range(0, len(features_mask)): x = ' ft' + str(f + 1) + ') ' + str(features_mask[f]) ck.out(x) s += x + '\n' s += '\nFinal features mask:\n\n' s1 = '' for f in range(0, len(features_mask)): x = ' ft' + str(f + 1) + ') ' + str(features_mask[f]) ck.out(x) s += x + '\n' if features_mask[f] == 1: if s1 != '': s1 += ',' s1 += 'ft' + str(f + 1) s += '\nFinal accuracy: ' + str(acc) + '\n' r = ck.save_text_file({ 'text_file': 'process_model_using_nearest_neighbour_reduce_features/log.' + gx + '.txt', 'string': s }) if r['return'] > 0: return r r = ck.save_text_file({ 'text_file': 'process_model_using_nearest_neighbour_reduce_features/influential_features.' + gx + '.txt', 'string': s1 }) if r['return'] > 0: return r ddd[g]['milepost_reduce_complexity2_normalized_ft1_ft121'] = acc # Save common data r = ck.save_json_to_file({ 'json_file': 'save_all_model_data_tmp.json', 'dict': ddd }) if r['return'] > 0: return r return {'return': 0}
# Record heat map dd = {'table': {"0": heatmap}} xx = [] iq = 0 for k in uopts[comp]: xx.append(iq) iq += 1 dd['axis_y_labels'] = xx r = ck.save_json_to_file({ 'json_file': 'init_reactions_tmp_heatmap_' + comp.replace(' ', '_') + '.json', 'dict': dd, 'sort_keys': 'yes' }) if r['return'] > 0: ck.err(r) # Sort classes for o in classes[comp]: b = classes[comp][o] b1 = sorted(b, key=lambda k: k.get('improvement', 0.0), reverse=True) classes[comp][o] = b1 # Prepare individual flags to predict (YES/NO) for opt in classes[comp]:
def main(i): # Load common table file (for all models) ddd = {} r = ck.load_json_file({'json_file': 'save_all_model_data_tmp.json'}) if r['return'] == 0: ddd = r['dict'] # Searching for features for g in gcc: ck.out( '********************************************************************************' ) ck.out('Preparing accuracy graph for ' + g) ck.out('') gx = g.replace(' ', '_') d = ddd[g] table = {"0": [], "1": []} for depth in range(1, 30): key1 = 'decision_trees_with_cross_validation_depth_' + str( depth) + '_ft1_ft65' key2 = 'decision_trees_without_cross_validation_depth_' + str( depth) + '_ft1_ft65' acc1 = d[key1] acc2 = d[key2] table["0"].append([depth, acc1]) table["1"].append([depth, acc2]) # Graph input ii = { "action": "plot", "module_uoa": "graph", "table": table, "add_x_loop": "no", "ignore_point_if_none": "yes", "plot_type": "mpl_2d_scatter", "display_y_error_bar": "no", "title": "Powered by Collective Knowledge", "axis_x_desc": "Decision tree depth", "axis_y_desc": "Model accuracy for " + g + " (%)", "plot_grid": "yes", "mpl_image_size_x": "12", "mpl_image_size_y": "6", "mpl_image_dpi": "100", "font_size": 22, "out_to_file": 'process_model_using_decision_trees_accuracy_graph_output.' + gx + '.pdf', "point_style": { "0": { "marker": "o" }, "1": { "marker": "x" } } } # Save common data (continuously, not to loose data) r = ck.save_json_to_file({ 'json_file': 'process_model_using_decision_trees_accuracy_graph_input.' + gx + '.json', 'dict': ii }) if r['return'] > 0: return r # Plot graph (save to pdf) r = ck.access(ii) if r['return'] > 0: return r return {'return': 0}
def update(i): """ Input: { (force) [bool] - if True, force update } Output: { return [int] - return code = 0 if success or >0 if error (error) [str] - error string if return>0 } """ import os global bootstrapping bootstrapping = True force = i.get('force') cfg = i.get('cfg', {}) from . import obj title = 'Bootstrapping' if cfg.get('bootstrapped', '') == 'yes': title = 'Updating' ck.out(title + ' cBench to support portable actions and workflows:') ck.out('') # Check release notes server_url = cfg.get('server_url', '') if server_url == '': server_url = 'https://cKnowledge.io/api/v1/?' from . import comm_min r = comm_min.send({ 'url': server_url, 'action': 'event', 'dict': { 'type': 'get-cbench-bootstrap-notes' } }) notes = r.get('notes', '') if notes != '': ck.out('***********************************************') ck.out(notes) ck.out('***********************************************') lst_all = [] sbf = os.environ.get('CB_SAVE_BOOTSTRAP_FILES', '') if sbf == '': fboot = 'cb-bootstrap-20200529' files = [fboot + '.json'] if os.name == 'nt': files.append(fboot + '-win.json') for fn in files: r = ck.gen_tmp_file({'prefix': 'cb-bootstrap-', 'suffix': '.json'}) if r['return'] > 0: return r ftmp = r['file_name'] burl = CR_DEFAULT_SERVER + '/static/bootstrap/' + fn ck.out('Downloading ' + burl) from . import comm rx = comm.download_file({'url': burl, 'file': ftmp}) if rx['return'] > 0: return rx rx = ck.load_json_file({'json_file': ftmp}) if rx['return'] > 0: return rx lst_all += rx['dict'] os.remove(ftmp) r = obj.download({'components': lst_all, 'force': force}) if r['return'] > 0 and r['return'] != 8: return r else: for x in CR_SOLUTION_CK_COMPONENTS: r = obj.download({ 'cid': x['cid'], 'version': x.get('version', ''), 'force': force }) if r['return'] > 0: if r['return'] != 8: return r else: ck.out(' Skipped - already exists!') else: lst_all += r['components'] rx = ck.save_json_to_file({ 'json_file': sbf, 'dict': lst_all, 'sort_keys': 'yes' }) if rx['return'] > 0: return rx ck.out('') # Update cfg cfg['bootstrapped'] = 'yes' ii = { 'action': 'update', 'repo_uoa': CK_CFG_REPO_UOA, 'module_uoa': CK_CFG_MODULE_UID, 'data_uoa': CK_CFG_DATA_UOA, 'dict': cfg, 'sort_keys': 'yes' } r = ck.access(ii) ck.out(title + ' finished!') ck.out('') return r