def do(i, arg): random_name = arg.random_name # Detect basic platform info. ii = {'action': 'detect', 'module_uoa': 'platform', 'out': 'out'} r = ck.access(ii) if r['return'] > 0: return r # Keep to prepare ReQuEST meta. platform_dict = copy.deepcopy(r) # Host and target OS params. hos = r['host_os_uoa'] hosd = r['host_os_dict'] tos = r['os_uoa'] tosd = r['os_dict'] tdid = r['device_id'] # Program and command. program = 'caffe' cmd_key = 'time_cpu' # Load Caffe program meta and desc to check deps. ii = {'action': 'load', 'module_uoa': 'program', 'data_uoa': program} rx = ck.access(ii) if rx['return'] > 0: return rx mm = rx['dict'] # Get compile-time and run-time deps. cdeps = mm.get('compile_deps', {}) rdeps = mm.get('run_deps', {}) # Merge rdeps with cdeps for setting up the pipeline (which uses # common deps), but tag them as "for_run_time". for k in rdeps: cdeps[k] = rdeps[k] cdeps[k]['for_run_time'] = 'yes' # Caffe libs. depl = copy.deepcopy(cdeps['lib-caffe']) if (arg.tos is not None) and (arg.did is not None): tos = arg.tos tdid = arg.did ii = { 'action': 'resolve', 'module_uoa': 'env', 'host_os': hos, 'target_os': tos, 'device_id': tdid, 'out': 'con', 'quiet': 'yes', 'deps': { 'lib-caffe': copy.deepcopy(depl) } } r = ck.access(ii) if r['return'] > 0: return r udepl = r['deps']['lib-caffe'].get('choices', []) # All UOAs of env for Caffe libs. if len(udepl) == 0: return {'return': 1, 'error': 'no installed Caffe libs'} # Caffe models. depm = copy.deepcopy(cdeps['caffemodel']) ii = { 'action': 'resolve', 'module_uoa': 'env', 'host_os': hos, 'target_os': tos, 'device_id': tdid, 'out': 'con', 'quiet': 'yes', 'deps': { 'caffemodel': copy.deepcopy(depm) } } r = ck.access(ii) if r['return'] > 0: return r udepm = r['deps']['caffemodel'].get( 'choices', []) # All UOAs of env for Caffe models. if len(udepm) == 0: return {'return': 1, 'error': 'no installed Caffe models'} # ImageNet datasets. depd = copy.deepcopy(cdeps['dataset-imagenet-lmdb']) ii = { 'action': 'resolve', 'module_uoa': 'env', 'host_os': hos, 'target_os': tos, 'device_id': tdid, 'out': 'con', 'quiet': 'yes', 'deps': { 'dataset-imagenet-lmdb': copy.deepcopy(depd) } } r = ck.access(ii) if r['return'] > 0: return r udepd = r['deps']['dataset-imagenet-lmdb'].get( 'choices', []) # All UOAs of env for ImageNet datasets. if len(udepd) == 0: return {'return': 1, 'error': 'no installed ImageNet datasets'} # Prepare pipeline. cdeps['lib-caffe']['uoa'] = udepl[0] cdeps['caffemodel']['uoa'] = udepm[0] cdeps['dataset-imagenet-lmdb']['uoa'] = udepd[0] ii = { 'action': 'pipeline', 'prepare': 'yes', 'dependencies': cdeps, 'module_uoa': 'program', 'data_uoa': program, 'cmd_key': cmd_key, 'target_os': tos, 'device_id': tdid, 'no_state_check': 'yes', 'no_compiler_description': 'yes', 'skip_calibration': 'yes', 'cpu_freq': 'max', 'gpu_freq': 'max', 'flags': '-O3', 'speed': 'no', 'energy': 'no', 'skip_print_timers': 'yes', 'out': 'con' } r = ck.access(ii) if r['return'] > 0: return r fail = r.get('fail', '') if fail == 'yes': return { 'return': 10, 'error': 'pipeline failed (' + r.get('fail_reason', '') + ')' } ready = r.get('ready', '') if ready != 'yes': return {'return': 11, 'error': 'pipeline not ready'} state = r['state'] tmp_dir = state['tmp_dir'] # Remember resolved deps for this benchmarking session. xcdeps = r.get('dependencies', {}) # Clean pipeline. if 'ready' in r: del (r['ready']) if 'fail' in r: del (r['fail']) if 'return' in r: del (r['return']) pipeline = copy.deepcopy(r) # For each Caffe lib.******************************************************* for lib_uoa in udepl: # Load Caffe lib. ii = {'action': 'load', 'module_uoa': 'env', 'data_uoa': lib_uoa} r = ck.access(ii) if r['return'] > 0: return r real_tags = r['dict']['tags'] # if 'vrequest' in real_tags: continue # Get the tags from e.g. 'BVLC Caffe framework (intel, request)' lib_name = r['data_name'] lib_tags = re.match('BVLC Caffe framework \((?P<tags>.*)\)', lib_name) lib_tags = lib_tags.group('tags').replace(' ', '').replace(',', '-') # Skip some libs with "in [..]" or "not in [..]". # Check if Intel artifact and select OpenMP tuning tuning_dims = { 'choices_order': [['##choices#env#CK_CAFFE_BATCH_SIZE']], 'choices_selection': [{ 'type': 'loop', 'default': bs['default'], 'choice': bs['choice'] }], 'features_keys_to_process': ['##choices#env#CK_CAFFE_BATCH_SIZE'] } intel_artifact = False if 'intel' in real_tags and 'vrequest' in real_tags: intel_artifact = True # Add extra tuning dimension (OpenMP) tuning_dims['choices_order'].append( ['##choices#env#OMP_NUM_THREADS']) tuning_dims['choices_selection'].append({ 'type': 'loop', 'default': nt['default'], 'choice': nt['choice'] }) tuning_dims['features_keys_to_process'].append( '##choices#env#OMP_NUM_THREADS') # ReQuEST CUDA/CUDNN cuda = False rtags = r['dict'].get('tags', []) if 'vcuda' in rtags: # Detect gpgpu r = ck.access({ 'action': 'detect', 'module_uoa': 'platform.gpgpu', 'cuda': 'yes', 'select': 'yes' }) if r['return'] > 0: return r platform_dict['features'].update(r['features']) cuda = True # Check proper command line for CPU or GPU cmd_key = 'time_cpu' if cuda: cmd_key = 'time_gpu' # Remark next one if you want to check other libs # if lib_tags in [ 'intel-request' ]: continue skip_compile = 'no' # For each Caffe model.************************************************* for model_uoa in udepm: # Load Caffe model. ii = {'action': 'load', 'module_uoa': 'env', 'data_uoa': model_uoa} r = ck.access(ii) if r['return'] > 0: return r model_real_tags = r['dict']['tags'] if 'vint8' in model_real_tags and not intel_artifact: continue # Get the tags from e.g. 'Caffe model (net and weights) (inception-v3, fp32)' model_name = r['data_name'] model_tags = re.match( 'Caffe model \(net and weights\) \((?P<tags>.*)\)', model_name) model_tags = model_tags.group('tags').replace(' ', '').replace( ',', '-') # Skip some models with "in [..]" or "not in [..]". if model_tags not in [ 'resnet50-fp32', 'resnet50-int8', 'inception-v3-fp32', 'inception-v3-int8' ]: continue record_repo = 'local' record_uoa = 'ck-request-asplos18-caffe-intel-performance-' + platform_tags + '.' + lib_tags + '.' + model_tags # Prepare pipeline. ck.out( '---------------------------------------------------------------------------------------' ) ck.out('%s - %s' % (lib_name, lib_uoa)) ck.out('%s - %s' % (model_name, model_uoa)) ck.out('Experiment - %s:%s' % (record_repo, record_uoa)) # Prepare autotuning input. cpipeline = copy.deepcopy(pipeline) # Reset deps and change UOA. new_deps = { 'lib-caffe': copy.deepcopy(depl), 'caffemodel': copy.deepcopy(depm) } new_deps['lib-caffe']['uoa'] = lib_uoa new_deps['caffemodel']['uoa'] = model_uoa jj = { 'action': 'resolve', 'module_uoa': 'env', 'host_os': hos, 'target_os': tos, 'device_id': tdid, 'deps': new_deps } r = ck.access(jj) if r['return'] > 0: return r cpipeline['dependencies'].update(new_deps) cpipeline['no_clean'] = skip_compile cpipeline['no_compile'] = skip_compile cpipeline['cmd_key'] = cmd_key cpipeline['extra_run_cmd'] = '-phase TEST' # Prepare common meta for ReQuEST tournament features = copy.deepcopy(cpipeline['features']) platform_dict['features'].update(features) r = ck.access({ 'action': 'prepare_common_meta', 'module_uoa': 'request.asplos18', 'platform_dict': platform_dict, 'deps': cpipeline['dependencies'], 'request_dict': request_dict }) if r['return'] > 0: return r record_dict = r['record_dict'] meta = r['meta'] if random_name: rx = ck.gen_uid({}) if rx['return'] > 0: return rx record_uoa = rx['data_uid'] tags = r['tags'] tags.append('explore-batch-size-openmp-threads') tags.append(program) tags.append(model_tags) tags.append(lib_tags) tags.append(platform_tags) ii = { 'action': 'autotune', 'module_uoa': 'pipeline', 'data_uoa': 'program', 'iterations': -1, 'repetitions': num_repetitions, 'record': 'yes', 'record_failed': 'yes', 'record_params': { 'search_point_by_features': 'yes' }, 'tags': tags, 'meta': meta, 'record_dict': record_dict, 'record_repo': record_repo, 'record_uoa': record_uoa, 'pipeline': cpipeline, 'out': 'con' } ii.update(copy.deepcopy(tuning_dims)) r = ck.access(ii) if r['return'] > 0: return r fail = r.get('fail', '') if fail == 'yes': return { 'return': 10, 'error': 'pipeline failed (' + r.get('fail_reason', '') + ')' } skip_compile = 'yes' return {'return': 0}
def do(i, arg): # Process arguments. if (arg.accuracy): experiment_type = 'accuracy' num_repetitions = 1 else: experiment_type = 'performance' num_repetitions = arg.repetitions random_name = arg.random_name share_platform = arg.share_platform # Detect basic platform info. ii={'action':'detect', 'module_uoa':'platform', 'out':'con'} if share_platform: ii['exchange']='yes' r=ck.access(ii) if r['return']>0: return r # Keep to prepare ReQuEST meta. platform_dict=copy.deepcopy(r) # Host and target OS params. hos=r['host_os_uoa'] hosd=r['host_os_dict'] tos=r['os_uoa'] tosd=r['os_dict'] tdid=r['device_id'] # program='mobilenets-armcl-opencl' program='image-classification-tf-py' ii={'action':'show', 'module_uoa':'env', 'tags':'dataset,imagenet,raw,val'} rx=ck.access(ii) if len(rx['lst']) == 0: return rx # FIXME: It's probably better to use CK_ENV_DATASET_IMAGE_DIR. img_dir_val = rx['lst'][0]['meta']['env']['CK_CAFFE_IMAGENET_VAL'] if (arg.accuracy): batch_count = len([f for f in os.listdir(img_dir_val) if f.endswith('.JPEG') and os.path.isfile(os.path.join(img_dir_val, f))]) else: batch_count = 1 ii={'action':'show', 'module_uoa':'env', 'tags':'dataset,imagenet,aux'} rx=ck.access(ii) if len(rx['lst']) == 0: return rx img_dir_aux = rx['lst'][0]['meta']['env']['CK_ENV_DATASET_IMAGENET_AUX'] ii={'action':'load', 'module_uoa':'program', 'data_uoa':program} rx=ck.access(ii) if rx['return']>0: return rx mm=rx['dict'] # Get compile-time and run-time deps. cdeps=mm.get('compile_deps',{}) rdeps=mm.get('run_deps',{}) # Merge rdeps with cdeps for setting up the pipeline (which uses # common deps), but tag them as "for_run_time". for k in rdeps: cdeps[k]=rdeps[k] cdeps[k]['for_run_time']='yes' print cdeps depl=copy.deepcopy(cdeps['lib-tensorflow']) if (arg.tos is not None) and (arg.did is not None): tos=arg.tos tdid=arg.did ii={'action':'resolve', 'module_uoa':'env', 'host_os':hos, 'target_os':tos, 'device_id':tdid, 'out':'con', 'deps':{'library':copy.deepcopy(depl)}, 'quiet':'yes' } r=ck.access(ii) if r['return']>0: return r udepl=r['deps']['library'].get('choices',[]) # All UOAs of env for TF lib if len(udepl)==0: return {'return':1, 'error':'no installed TensorFlow'} cdeps['lib-tensorflow']['uoa']=udepl[0] depm=copy.deepcopy(cdeps['model-and-weights']) ii={'action':'resolve', 'module_uoa':'env', 'host_os':hos, 'target_os':tos, 'device_id':tdid, 'out':'con', 'deps':{'weights':copy.deepcopy(depm)}, 'quiet':'yes' } r=ck.access(ii) if r['return']>0: return r udepm=r['deps']['weights'].get('choices',[]) if len(udepm)==0: return {'return':1, 'error':'no installed Weights'} cdeps['lib-tensorflow']['uoa']=udepl[0] cdeps['model-and-weights']['uoa']=udepm[0] ii={'action':'pipeline', 'prepare':'yes', 'dependencies':cdeps, 'module_uoa':'program', 'data_uoa':program, 'target_os':tos, 'device_id':tdid, 'no_state_check':'yes', 'no_compiler_description':'yes', 'skip_calibration':'yes', 'env':{ 'CK_ENV_DATASET_IMAGENET_VAL':img_dir_val, 'CK_BATCH_COUNT':batch_count, 'CK_BATCHES_DIR':'../batches', 'CK_BATCH_LIST':'../batches', 'CK_IMAGE_LIST':'../images', 'CK_RESULTS_DIR':'predictions', 'CK_SKIP_IMAGES':0 }, 'cpu_freq':'max', 'gpu_freq':'max', 'flags':'-O3', 'speed':'no', 'energy':'no', 'skip_print_timers':'yes', 'out':'con' } r=ck.access(ii) if r['return']>0: return r fail=r.get('fail','') if fail=='yes': return {'return':10, 'error':'pipeline failed ('+r.get('fail_reason','')+')'} ready=r.get('ready','') if ready!='yes': return {'return':11, 'error':'pipeline not ready'} state=r['state'] tmp_dir=state['tmp_dir'] # Remember resolved deps for this benchmarking session. xcdeps=r.get('dependencies',{}) # Clean pipeline. if 'ready' in r: del(r['ready']) if 'fail' in r: del(r['fail']) if 'return' in r: del(r['return']) pipeline=copy.deepcopy(r) for lib_uoa in udepl: # Load ArmCL lib. ii={'action':'load', 'module_uoa':'env', 'data_uoa':lib_uoa} r=ck.access(ii) if r['return']>0: return r lib_name=r['data_name'] lib_tags=r['dict']['customize']['version'] # Skip some libs with "in [..]" or "not in [..]". if arg.accuracy and lib_tags in [ ]: continue skip_compile='no' # For each MobileNets model.************************************************* for model_uoa in udepm: # Load model. ii={'action':'load', 'module_uoa':'env', 'data_uoa':model_uoa} r=ck.access(ii) if r['return']>0: return r model_name=r['data_name'] if 'mobilenet' not in r['dict']['tags']: continue alpha = float(r['dict']['env']['CK_ENV_TENSORFLOW_MODEL_MOBILENET_MULTIPLIER']) rho = int(r['dict']['env']['CK_ENV_TENSORFLOW_MODEL_MOBILENET_RESOLUTION']) record_repo='local' record_uoa='mobilenets-'+experiment_type+'-'+str(rho)+'-'+str(alpha)+'-tensorflow-'+lib_tags # Prepare pipeline. ck.out('---------------------------------------------------------------------------------------') ck.out('%s - %s' % (lib_name, lib_uoa)) ck.out('%s - %s' % (model_name, model_uoa)) ck.out('Experiment - %s:%s' % (record_repo, record_uoa)) # Prepare autotuning input. cpipeline=copy.deepcopy(pipeline) # Reset deps and change UOA. new_deps={'library':copy.deepcopy(depl), 'weights':copy.deepcopy(depm)} new_deps['library']['uoa']=lib_uoa new_deps['weights']['uoa']=model_uoa jj={'action':'resolve', 'module_uoa':'env', 'host_os':hos, 'target_os':tos, 'device_id':tdid, 'deps':new_deps} r=ck.access(jj) if r['return']>0: return r cpipeline['dependencies'].update(new_deps) cpipeline['no_clean']=skip_compile cpipeline['no_compile']=skip_compile # Prepare common meta for ReQuEST tournament features=copy.deepcopy(cpipeline['features']) platform_dict['features'].update(features) r=ck.access({'action':'prepare_common_meta', 'module_uoa':'request.asplos18', 'platform_dict':platform_dict, 'deps':cpipeline['dependencies'], 'request_dict':request_dict}) if r['return']>0: return r record_dict=r['record_dict'] meta=r['meta'] if random_name: rx=ck.gen_uid({}) if rx['return']>0: return rx record_uoa=rx['data_uid'] tags=r['tags'] tags.append(experiment_type) tags.append('explore-mobilenets-'+experiment_type) tags.append(lib_tags) tags.append(platform_tags) tags.append(str(rho)) tags.append(str(alpha)) ii={'action':'autotune', 'module_uoa':'pipeline', 'data_uoa':'program', 'choices_order':[ [ '##choices#env#CK_BATCH_SIZE' ], [ '##choices#env#CK_CONVOLUTION_METHOD_HINT' ], [ '##choices#env#CK_ENV_MOBILENET_RESOLUTION' ], [ '##choices#env#CK_ENV_MOBILENET_WIDTH_MULTIPLIER' ] ], 'choices_selection':[ {'type':'loop', 'start':bs['start'], 'stop':bs['stop'], 'step':bs['step'], 'default':bs['default']}, {'type':'loop', 'start':ch['start'], 'stop':ch['stop'], 'step':ch['step'], 'default':ch['default']}, {'type':'loop', 'choice': [rho], 'default': 224}, {'type':'loop', 'choice': [alpha], 'default': 1.0}, ], 'features_keys_to_process':['##choices#*'], 'iterations':-1, 'repetitions': num_repetitions, 'record':'yes', 'record_failed':'yes', 'record_params':{ 'search_point_by_features':'yes' }, 'tags':tags, 'meta':meta, 'record_dict':record_dict, 'record_repo':record_repo, 'record_uoa':record_uoa, 'pipeline':cpipeline, 'out':'con' } r=ck.access(ii) if r['return']>0: return r fail=r.get('fail','') if fail=='yes': return {'return':10, 'error':'pipeline failed ('+r.get('fail_reason','')+')'} ### end pipeline return {'return':0}
def do(i, arg): # Process arguments. if (arg.accuracy): experiment_type = 'accuracy' num_repetitions = 1 cmd_key = 'test' else: experiment_type = 'performance' num_repetitions = arg.repetitions cmd_key = 'classify' random_name = arg.random_name share_platform = arg.share_platform # Detect basic platform info. ii = {'action': 'detect', 'module_uoa': 'platform', 'out': 'con'} if share_platform: ii['exchange'] = 'yes' r = ck.access(ii) if r['return'] > 0: return r # Keep to prepare ReQuEST meta. platform_dict = copy.deepcopy(r) # Host and target OS params. hos = r['host_os_uoa'] hosd = r['host_os_dict'] tos = r['os_uoa'] tosd = r['os_dict'] tdid = r['device_id'] ii = { 'action': 'show', 'module_uoa': 'env', 'tags': 'dataset,imagenet,raw,val' } rx = ck.access(ii) if len(rx['lst']) == 0: return rx # FIXME: It's probably better to use CK_ENV_DATASET_IMAGE_DIR. img_dir_val = rx['lst'][0]['meta']['env']['CK_CAFFE_IMAGENET_VAL'] if arg.accuracy: batch_count = len([ f for f in os.listdir(img_dir_val) if f.endswith('.JPEG') and os.path.isfile(os.path.join(img_dir_val, f)) ]) else: batch_count = 1 ii = { 'action': 'show', 'module_uoa': 'env', 'tags': 'dataset,imagenet,aux' } rx = ck.access(ii) if len(rx['lst']) == 0: return rx img_dir_aux = rx['lst'][0]['meta']['env']['CK_ENV_DATASET_IMAGENET_AUX'] ii = {'action': 'load', 'module_uoa': 'program', 'data_uoa': program} rx = ck.access(ii) if rx['return'] > 0: return rx mm = rx['dict'] # Get compile-time and run-time deps. cdeps = mm.get('compile_deps', {}) rdeps = mm.get('run_deps', {}) rdeps.update(mm.get('run_cmds', {}).get(cmd_key, {}).get('run_deps', {})) # Merge rdeps with cdeps for setting up the pipeline (which uses # common deps), but tag them as "for_run_time". for k in rdeps: cdeps[k] = rdeps[k] cdeps[k]['for_run_time'] = 'yes' depl = copy.deepcopy(cdeps['lib-nnvm-tvm']) if (arg.tos is not None) and (arg.did is not None): tos = arg.tos tdid = arg.did ii = { 'action': 'resolve', 'module_uoa': 'env', 'host_os': hos, 'target_os': tos, 'device_id': tdid, 'out': 'con', 'deps': { 'lib-nnvm-tvm': copy.deepcopy(depl) }, 'quiet': 'yes' } r = ck.access(ii) if r['return'] > 0: return r udepl = r['deps']['lib-nnvm-tvm'].get('choices', []) if len(udepl) == 0: return {'return': 1, 'error': 'no installed TVM'} cdeps['lib-nnvm-tvm']['uoa'] = udepl[0] depm = copy.deepcopy(cdeps['mxnet-model']) ii = { 'action': 'resolve', 'module_uoa': 'env', 'host_os': hos, 'target_os': tos, 'device_id': tdid, 'out': 'con', 'deps': { 'mxnet-model': copy.deepcopy(depm) }, 'quiet': 'yes' } r = ck.access(ii) if r['return'] > 0: return r udepm = r['deps']['mxnet-model'].get('choices', []) if len(udepm) == 0: return {'return': 1, 'error': 'no installed mxnet-model'} cdeps['mxnet-model']['uoa'] = udepm[0] ii = { 'action': 'pipeline', 'prepare': 'yes', 'dependencies': cdeps, 'module_uoa': 'program', 'data_uoa': program, 'target_os': tos, 'device_id': tdid, 'cmd_key': cmd_key, 'dataset_uoa': dataset, 'no_state_check': 'yes', 'no_compiler_description': 'yes', 'skip_calibration': 'yes', 'cpu_freq': 'max', 'gpu_freq': 'max', 'env': { 'STAT_REPEAT': 5 }, 'flags': '-O3', 'speed': 'no', 'energy': 'no', 'skip_print_timers': 'yes', 'out': 'con' } r = ck.access(ii) if r['return'] > 0: return r fail = r.get('fail', '') if fail == 'yes': return { 'return': 10, 'error': 'pipeline failed (' + r.get('fail_reason', '') + ')' } ready = r.get('ready', '') if ready != 'yes': return {'return': 11, 'error': 'pipeline not ready'} state = r['state'] tmp_dir = state['tmp_dir'] # Remember resolved deps for this benchmarking session. xcdeps = r.get('dependencies', {}) # Clean pipeline. if 'ready' in r: del (r['ready']) if 'fail' in r: del (r['fail']) if 'return' in r: del (r['return']) pipeline = copy.deepcopy(r) for precision in ['float32', 'float16']: ck.out('') ck.out('Precision: ' + precision) ck.out('') xprecision = 'fp32' if precision == 'float16': xprecision = 'fp16' for lib_uoa in udepl: # Load lib. ii = {'action': 'load', 'module_uoa': 'env', 'data_uoa': lib_uoa} r = ck.access(ii) if r['return'] > 0: return r lib_name = r['data_name'] lib_tags = r['dict']['customize']['version'] # Skip some libs with "in [..]" or "not in [..]". if arg.accuracy and lib_tags not in use_lib_tags: continue skip_compile = 'no' # For each model************************************************* for model_uoa in udepm: # Load model. ii = { 'action': 'load', 'module_uoa': 'env', 'data_uoa': model_uoa } r = ck.access(ii) if r['return'] > 0: return r model_real_tags = r['dict']['tags'] # Get the tags from e.g. 'Caffe model (net and weights) (inception-v3, fp32)' model_name = r['data_name'] model_tags = re.match( 'MXNet model \(net and weights\) \((?P<tags>.*)\)', model_name) model_tags = model_tags.group('tags').replace(' ', '').replace( ',', '-') record_repo = 'local' record_uoa = 'ck-request-asplos18-nnvm-tvm-arm-' + experiment_type + '.' + model_tags + '.' + xprecision # Prepare pipeline. ck.out( '---------------------------------------------------------------------------------------' ) ck.out('%s - %s' % (lib_name, lib_uoa)) ck.out('%s - %s' % (model_name, model_uoa)) ck.out('Experiment - %s:%s' % (record_repo, record_uoa)) # Prepare autotuning input. cpipeline = copy.deepcopy(pipeline) cpipeline['choices']['env']['CK_TVM_DTYPE'] = precision # Reset deps and change UOA. new_deps = { 'lib-nnvm-tvm': copy.deepcopy(depl), 'mxnet-model': copy.deepcopy(depm) } new_deps['lib-nnvm-tvm']['uoa'] = lib_uoa new_deps['mxnet-model']['uoa'] = model_uoa jj = { 'action': 'resolve', 'module_uoa': 'env', 'host_os': hos, 'target_os': tos, 'device_id': tdid, 'deps': new_deps } r = ck.access(jj) if r['return'] > 0: return r cpipeline['dependencies'].update(new_deps) cpipeline['no_clean'] = skip_compile cpipeline['no_compile'] = skip_compile # Prepare common meta for ReQuEST tournament features = copy.deepcopy(cpipeline['features']) platform_dict['features'].update(features) r = ck.access({ 'action': 'prepare_common_meta', 'module_uoa': 'request.asplos18', 'platform_dict': platform_dict, 'deps': cpipeline['dependencies'], 'request_dict': request_dict }) if r['return'] > 0: return r record_dict = r['record_dict'] meta = r['meta'] meta['model_precision'] = xprecision if random_name: rx = ck.gen_uid({}) if rx['return'] > 0: return rx record_uoa = rx['data_uid'] tags = r['tags'] tags.append(experiment_type) tags.append('explore-mxnet-' + experiment_type) tags.append(lib_tags) tags.append(platform_tags) ii = { 'action': 'autotune', 'module_uoa': 'pipeline', 'data_uoa': 'program', 'choices_order': [['##choices#env#CK_BATCH_SIZE']], 'choices_selection': [{ 'type': 'loop', 'start': bs['start'], 'stop': bs['stop'], 'step': bs['step'], 'default': bs['default'] }], 'features_keys_to_process': ['##choices#*'], 'iterations': -1, 'repetitions': num_repetitions, 'record': 'yes', 'record_failed': 'yes', 'record_params': { 'search_point_by_features': 'yes' }, 'tags': tags, 'meta': meta, 'record_dict': record_dict, 'record_repo': record_repo, 'record_uoa': record_uoa, 'pipeline': cpipeline, 'out': 'con' } r = ck.access(ii) if r['return'] > 0: return r fail = r.get('fail', '') if fail == 'yes': return { 'return': 10, 'error': 'pipeline failed (' + r.get('fail_reason', '') + ')' } ### end pipeline return {'return': 0}
def do(i, arg): random_name = arg.random_name # Detect basic platform info. ii={'action':'detect', 'module_uoa':'platform', 'target':target, 'out':'out'} r=ck.access(ii) if r['return']>0: return r # Keep to prepare ReQuEST meta. platform_dict=copy.deepcopy(r) # Host and target OS params. hos=r['host_os_uoa'] hosd=r['host_os_dict'] tos=r['os_uoa'] tosd=r['os_dict'] tdid=r['device_id'] # Program and command. program='request-tvm-vta-pynq' cmd_key='classify' dset='e496fdf046e6ac13' # image-jpeg-dnn-cat # Load program meta and desc to check deps. ii={'action':'load', 'module_uoa':'program', 'data_uoa':program} rx=ck.access(ii) if rx['return']>0: return rx mm=rx['dict'] # Get deps deps=mm.get('run_cmds',{}).get('classify',{}).get('run_deps',{}) # Models depm=copy.deepcopy(deps['model']) ii={'action':'resolve', 'module_uoa':'env', 'host_os':hos, 'target_os':tos, 'device_id':tdid, 'out':'con', 'quiet':'yes', 'deps':{'model':copy.deepcopy(depm)} } r=ck.access(ii) if r['return']>0: return r udepm=r['deps']['model'].get('choices',[]) # All UOAs of env for Caffe models. if len(udepm)==0: return {'return':1, 'error':'no installed VTA models'} deps['model']['uoa']=udepm[0] ii={'action':'pipeline', 'prepare':'yes', 'dependencies':deps, 'module_uoa':'program', 'data_uoa':program, 'cmd_key':cmd_key, 'dataset_uoa':dset, 'target':target, 'target_os':tos, 'device_id':tdid, 'no_state_check':'yes', 'no_compiler_description':'yes', 'skip_calibration':'yes', # not needed since it's not running on host ... # 'cpu_freq':'max', # 'gpu_freq':'max', 'flags':'-O3', 'speed':'no', 'energy':'no', 'skip_print_timers':'yes', 'out':'con' } r=ck.access(ii) if r['return']>0: return r fail=r.get('fail','') if fail=='yes': return {'return':10, 'error':'pipeline failed ('+r.get('fail_reason','')+')'} ready=r.get('ready','') if ready!='yes': return {'return':11, 'error':'pipeline not ready'} state=r['state'] tmp_dir=state['tmp_dir'] # Remember resolved deps for this benchmarking session. xdeps=r.get('dependencies',{}) # Clean pipeline. if 'ready' in r: del(r['ready']) if 'fail' in r: del(r['fail']) if 'return' in r: del(r['return']) pipeline=copy.deepcopy(r) for dummy in ['dummy']: # For each model.************************************************* for model_uoa in udepm: # Load model. ii={'action':'load', 'module_uoa':'env', 'data_uoa':model_uoa} r=ck.access(ii) if r['return']>0: return r model_real_tags=r['dict']['tags'] # Get the tags from e.g. 'Caffe model (net and weights) (inception-v3, fp32)' model_name=r['data_name'] model_tags = re.match('VTA model \(net and weights\) \((?P<tags>.*)\)', model_name) model_tags = model_tags.group('tags').replace(' ', '').replace(',', '-') record_repo='local' record_uoa='ck-request-asplos18-tvm-fpga-performance-'+platform_tags+'.'+model_tags # Prepare pipeline. ck.out('---------------------------------------------------------------------------------------') ck.out('%s - %s' % (model_name, model_uoa)) ck.out('Experiment - %s:%s' % (record_repo, record_uoa)) # Prepare autotuning input. cpipeline=copy.deepcopy(pipeline) # Reset deps and change UOA. new_deps={'model':copy.deepcopy(depm)} new_deps['model']['uoa']=model_uoa jj={'action':'resolve', 'module_uoa':'env', 'host_os':hos, 'target_os':tos, 'device_id':tdid, 'deps':new_deps} r=ck.access(jj) if r['return']>0: return r cpipeline['dependencies'].update(new_deps) cpipeline['cmd_key']=cmd_key # Prepare common meta for ReQuEST tournament features=copy.deepcopy(cpipeline['features']) platform_dict['features'].update(features) r=ck.access({'action':'prepare_common_meta', 'module_uoa':'request.asplos18', 'platform_dict':platform_dict, 'deps':cpipeline['dependencies'], 'request_dict':request_dict}) if r['return']>0: return r record_dict=r['record_dict'] meta=r['meta'] if random_name: rx=ck.gen_uid({}) if rx['return']>0: return rx record_uoa=rx['data_uid'] tags=r['tags'] tags.append(program) tags.append(model_tags) tags.append(platform_tags) ii={'action':'autotune', 'target':target, 'module_uoa':'pipeline', 'data_uoa':'program', 'iterations':1, 'repetitions':num_repetitions, 'record':'yes', 'record_failed':'yes', 'record_params':{ 'search_point_by_features':'yes' }, 'tags':tags, 'meta':meta, 'record_dict':record_dict, 'record_repo':record_repo, 'record_uoa':record_uoa, 'pipeline':cpipeline, 'out':'con'} r=ck.access(ii) if r['return']>0: return r fail=r.get('fail','') if fail=='yes': return {'return':10, 'error':'pipeline failed ('+r.get('fail_reason','')+')'} skip_compile='yes' return {'return':0}
def do(i, arg): # Process arguments. if (arg.accuracy): experiment_type = 'accuracy' num_repetitions = 1 kernel_tuners = [default_kernel_tuner] else: experiment_type = 'performance' num_repetitions = arg.repetitions random_name = arg.random_name share_platform = arg.share_platform # Detect basic platform info. ii = {'action': 'detect', 'module_uoa': 'platform', 'out': 'con'} if share_platform: ii['exchange'] = 'yes' r = ck.access(ii) if r['return'] > 0: return r # Keep to prepare ReQuEST meta. platform_dict = copy.deepcopy(r) # Host and target OS params. hos = r['host_os_uoa'] hosd = r['host_os_dict'] tos = r['os_uoa'] tosd = r['os_dict'] tdid = r['device_id'] # Determine platform tags: if one of the known platforms, use its id; otherwise, 'unknown-platform'. # FIXME: only works when the target platform is the same as the host platform. platform_tags = platform_config.get(r['features']['platform']['model'], {'id': 'unknown-platform'})['id'] # The only supported program. program = 'mobilenets-armcl-opencl' # Select ImageNet dataset. r = select_ImageNet() if r['return'] > 0: return r imagenet_val = r['dataset'] img_dir_val = get_ImageNet_path(imagenet_val) ck.out('ImageNet path: ' + img_dir_val) if arg.accuracy: # Use as many batches (of size 1), as there are JPEG images in the directory. batch_count = len([ f for f in os.listdir(img_dir_val) if f.endswith('.JPEG') and os.path.isfile(os.path.join(img_dir_val, f)) ]) else: # FIXME: Use 2 batches, using the first for warm up (to be excluded from the average)? batch_count = 1 # Restrict accuracy testing to the ReQuEST fork of ArmCL. TODO: restrict to direct convolution for large datasets. if arg.accuracy and batch_count > 500: use_lib_tags = ['request-d8f69c13', '18.05-0acd60ed-request'] else: use_lib_tags = [ 'request-d8f69c13', '18.11-b9abeae08', '18.08-52ba29e9', '18.05-0acd60ed-request', '18.05-b3a371bc', '18.03-e40997bb', '18.01-f45d5a9b', '17.12-48bc34ea' ] # On Firefly-RK3399, the version hash has only 7 characters, not 8. if platform_tags == 'firefly-rk3399': use_lib_tags = [tag[:-1] for tag in use_lib_tags] ii = { 'action': 'show', 'module_uoa': 'env', 'tags': 'dataset,imagenet,aux' } rx = ck.access(ii) if len(rx['lst']) == 0: return rx img_dir_aux = rx['lst'][0]['meta']['env']['CK_ENV_DATASET_IMAGENET_AUX'] ii = {'action': 'load', 'module_uoa': 'program', 'data_uoa': program} rx = ck.access(ii) if rx['return'] > 0: return rx mm = rx['dict'] # Get compile-time and run-time deps. cdeps = mm.get('compile_deps', {}) rdeps = mm.get('run_deps', {}) # Merge rdeps with cdeps for setting up the pipeline (which uses # common deps), but tag them as "for_run_time". for k in rdeps: cdeps[k] = rdeps[k] cdeps[k]['for_run_time'] = 'yes' depl = copy.deepcopy(cdeps['library']) if (arg.tos is not None) and (arg.did is not None): tos = arg.tos tdid = arg.did ii = { 'action': 'resolve', 'module_uoa': 'env', 'host_os': hos, 'target_os': tos, 'device_id': tdid, 'out': 'con', 'deps': { 'library': copy.deepcopy(depl) }, 'quiet': 'yes' } r = ck.access(ii) if r['return'] > 0: return r udepl = r['deps']['library'].get( 'choices', []) # All UOAs of env for Arm Compute Libraries. if len(udepl) == 0: return {'return': 1, 'error': 'no installed Arm Compute Libraries'} cdeps['library']['uoa'] = udepl[0] depm = copy.deepcopy(cdeps['weights']) ii = { 'action': 'resolve', 'module_uoa': 'env', 'host_os': hos, 'target_os': tos, 'device_id': tdid, 'out': 'con', 'deps': { 'weights': copy.deepcopy(depm) }, 'quiet': 'yes' } r = ck.access(ii) if r['return'] > 0: return r udepm = r['deps']['weights'].get('choices', []) if len(udepm) == 0: return {'return': 1, 'error': 'no installed Weights'} cdeps['library']['uoa'] = udepl[0] cdeps['weights']['uoa'] = udepm[0] ii = { 'action': 'pipeline', 'prepare': 'yes', 'dependencies': cdeps, 'module_uoa': 'program', 'data_uoa': program, 'target_os': tos, 'device_id': tdid, 'no_state_check': 'yes', 'no_compiler_description': 'yes', 'skip_calibration': 'yes', 'env': { 'CK_ENV_DATASET_IMAGENET_VAL': img_dir_val, 'CK_BATCH_COUNT': batch_count, 'CK_SKIP_IMAGES': 0 }, 'cpu_freq': 'max', 'gpu_freq': 'max', 'flags': '-O3', 'speed': 'no', 'energy': 'no', 'skip_print_timers': 'yes', 'out': 'con' } r = ck.access(ii) if r['return'] > 0: return r fail = r.get('fail', '') if fail == 'yes': return { 'return': 10, 'error': 'pipeline failed (' + r.get('fail_reason', '') + ')' } ready = r.get('ready', '') if ready != 'yes': return {'return': 11, 'error': 'pipeline not ready'} state = r['state'] tmp_dir = state['tmp_dir'] # Remember resolved deps for this benchmarking session. xcdeps = r.get('dependencies', {}) # Clean pipeline. if 'ready' in r: del (r['ready']) if 'fail' in r: del (r['fail']) if 'return' in r: del (r['return']) experiment_count = 0 pipeline = copy.deepcopy(r) for lib_uoa in udepl: # Load ArmCL lib. ii = {'action': 'load', 'module_uoa': 'env', 'data_uoa': lib_uoa} r = ck.access(ii) if r['return'] > 0: return r lib_name = r['data_name'] lib_tags = r['dict']['customize']['version'] # Skip other libraries if one is explicitly specified. if arg.library_uoa and lib_uoa != arg.library_uoa: continue # Skip some libs with "in [..]" or "not in [..]". if arg.accuracy and lib_tags not in use_lib_tags: continue skip_compile = 'no' # For each MobileNets model.************************************************* for model_uoa in udepm: # Load model. ii = {'action': 'load', 'module_uoa': 'env', 'data_uoa': model_uoa} r = ck.access(ii) if r['return'] > 0: return r model_name = r['data_name'] # Skip other models if one is explicitly specified. if arg.model_uoa and model_uoa != arg.model_uoa: continue # Skip aggregate MobileNets packages. if 'mobilenet-all' in r['dict']['tags']: continue batch_size = 1 version = 1 multiplier = float(r['dict']['env']['CK_ENV_MOBILENET_MULTIPLIER']) resolution = int(r['dict']['env']['CK_ENV_MOBILENET_RESOLUTION']) record_repo = 'local' record_uoa = '{}-{}-{}-mobilenet-v{}-{:.2f}-{}'.format( experiment_type, platform_tags, lib_tags, version, multiplier, resolution) # Skip the experiment if it already exists. if arg.resume: r = ck.access({ 'action': 'search', 'module_uoa': 'experiment', 'repo_uoa': record_repo, 'data_uoa': record_uoa }) if r['return'] > 0: return r if len(r['lst']) > 0: ck.out('Experiment "%s" already exists, skipping...' % record_uoa) continue # Prepare pipeline. ck.out( '---------------------------------------------------------------------------------------' ) ck.out('%s - %s' % (lib_name, lib_uoa)) ck.out('%s - %s' % (model_name, model_uoa)) ck.out('Experiment - %s:%s' % (record_repo, record_uoa)) experiment_count += 1 # Prepare autotuning input. cpipeline = copy.deepcopy(pipeline) # Reset deps and change UOA. new_deps = { 'library': copy.deepcopy(depl), 'weights': copy.deepcopy(depm) } new_deps['library']['uoa'] = lib_uoa new_deps['weights']['uoa'] = model_uoa jj = { 'action': 'resolve', 'module_uoa': 'env', 'host_os': hos, 'target_os': tos, 'device_id': tdid, 'deps': new_deps } r = ck.access(jj) if r['return'] > 0: return r cpipeline['dependencies'].update(new_deps) cpipeline['no_clean'] = skip_compile cpipeline['no_compile'] = skip_compile # Prepare common meta for ReQuEST tournament features = copy.deepcopy(cpipeline['features']) platform_dict['features'].update(features) r = ck.access({ 'action': 'prepare_common_meta', 'module_uoa': 'request.asplos18', 'platform_dict': platform_dict, 'deps': cpipeline['dependencies'], 'request_dict': request_dict }) if r['return'] > 0: return r record_dict = r['record_dict'] meta = r['meta'] if random_name: rx = ck.gen_uid({}) if rx['return'] > 0: return rx record_uoa = rx['data_uid'] tags = r['tags'] tags.append(experiment_type) tags.append('explore-mobilenets-' + experiment_type) tags.append(lib_tags) tags.append(platform_tags) tags.append(str(resolution)) tags.append(str(multiplier)) tags.append('mobilenet-v{}'.format(version)) tags.append('mobilenet-v{}-{:.2f}-{}'.format( version, multiplier, resolution)) ii = { 'action': 'autotune', 'module_uoa': 'pipeline', 'data_uoa': 'program', 'choices_order': [ ['##choices#env#CK_BATCH_SIZE'], ['##choices#env#CK_ENV_MOBILENET_VERSION'], ['##choices#env#CK_ENV_MOBILENET_MULTIPLIER'], ['##choices#env#CK_ENV_MOBILENET_RESOLUTION'], ['##choices#env#CK_CONVOLUTION_METHOD'], ['##choices#env#CK_LWS_TUNER_TYPE'], ['##choices#env#CK_DATA_LAYOUT'], ], 'choices_selection': [ { 'type': 'loop', 'choice': [batch_size], 'default': 1 }, # Only batch_size=1 is supported. { 'type': 'loop', 'choice': [version], 'default': 1 }, # Only version=1 is supported. { 'type': 'loop', 'choice': [multiplier], 'default': 1.0 }, { 'type': 'loop', 'choice': [resolution], 'default': 224 }, { 'type': 'loop', 'choice': convolution_methods, 'default': default_convolution_method }, { 'type': 'loop', 'choice': kernel_tuners, 'default': default_kernel_tuner }, { 'type': 'loop', 'choice': data_layouts, 'default': default_data_layout }, ], 'features_keys_to_process': ['##choices#*'], 'iterations': -1, 'repetitions': num_repetitions, 'record': 'yes', 'record_failed': 'yes', 'record_params': { 'search_point_by_features': 'yes' }, 'tags': tags, 'meta': meta, 'record_dict': record_dict, 'record_repo': record_repo, 'record_uoa': record_uoa, 'pipeline': cpipeline, 'out': 'con' } if not arg.dry_run: r = ck.access(ii) if r['return'] > 0: return r fail = r.get('fail', '') if fail == 'yes': return { 'return': 10, 'error': 'pipeline failed (' + r.get('fail_reason', '') + ')' } if arg.dry_run: ck.out( '---------------------------------------------------------------------------------------' ) ck.out('Experiment count: %d' % experiment_count) ### end pipeline return {'return': 0}
def do(i, arg): # Process arguments. if (arg.accuracy): experiment_type = 'accuracy' num_repetitions = 1 else: experiment_type = 'performance' num_repetitions = arg.repetitions random_name = arg.random_name share_platform = arg.share_platform # Detect basic platform info. ii = {'action': 'detect', 'module_uoa': 'platform', 'out': 'con'} if share_platform: ii['exchange'] = 'yes' r = ck.access(ii) if r['return'] > 0: return r # Keep to prepare ReQuEST meta. platform_dict = copy.deepcopy(r) # Host and target OS params. hos = r['host_os_uoa'] hosd = r['host_os_dict'] tos = r['os_uoa'] tosd = r['os_dict'] tdid = r['device_id'] # Determine platform tags: if one of the known platforms, use its id; otherwise, 'unknown-platform'. # FIXME: only works when the target platform is the same as the host platform. platform_tags = platform_config.get(r['features']['platform']['model'], {'id': 'unknown-platform'})['id'] # Select one of `image-classification-tf*`. r = select_program() if r['return'] > 0: return r program = r['program'] # `lib_type` is used to distinguish result repo records: tf-py, tf-cpp, tflite lib_type = program[len('image-classification-'):] # Select ImageNet dataset. r = select_ImageNet() if r['return'] > 0: return r imagenet_val = r['dataset'] img_dir_val = get_ImageNet_path(imagenet_val) ck.out('ImageNet path: ' + img_dir_val) if (arg.accuracy): # Use as many batches (of size 1), as there are JPEG images in the directory. batch_count = len([ f for f in os.listdir(img_dir_val) if f.endswith('.JPEG') and os.path.isfile(os.path.join(img_dir_val, f)) ]) else: # Use 2 batches, using the first for warm up (excluded from the average). batch_count = 2 ii = { 'action': 'show', 'module_uoa': 'env', 'tags': 'dataset,imagenet,aux' } rx = ck.access(ii) if len(rx['lst']) == 0: return rx imagenet_aux = rx['lst'][0] img_dir_aux = imagenet_aux['meta']['env']['CK_ENV_DATASET_IMAGENET_AUX'] ii = {'action': 'load', 'module_uoa': 'program', 'data_uoa': program} rx = ck.access(ii) if rx['return'] > 0: return rx mm = rx['dict'] # Get compile-time and run-time deps. cdeps = mm.get('compile_deps', {}) rdeps = mm.get('run_deps', {}) # Merge rdeps with cdeps for setting up the pipeline (which uses # common deps), but tag them as "for_run_time". for k in rdeps: cdeps[k] = rdeps[k] cdeps[k]['for_run_time'] = 'yes' if 'lib-tensorflow' in cdeps: library_key = 'lib-tensorflow' elif 'library' in cdeps: library_key = 'library' else: return {'return': 1, 'error': 'no library dependency in program meta'} depl = copy.deepcopy(cdeps[library_key]) if (arg.tos is not None) and (arg.did is not None): tos = arg.tos tdid = arg.did ii = { 'action': 'resolve', 'module_uoa': 'env', 'host_os': hos, 'target_os': tos, 'device_id': tdid, 'out': 'con', 'deps': { 'library': copy.deepcopy(depl) }, 'quiet': 'yes' } r = ck.access(ii) if r['return'] > 0: return r udepl = r['deps']['library'].get( 'choices', []) # All UOAs of env for TensorFlow libs. if len(udepl) == 0: return {'return': 1, 'error': 'no installed TensorFlow libs'} cdeps[library_key]['uoa'] = udepl[0] depm = copy.deepcopy(cdeps['weights']) ii = { 'action': 'resolve', 'module_uoa': 'env', 'host_os': hos, 'target_os': tos, 'device_id': tdid, 'out': 'con', 'deps': { 'weights': copy.deepcopy(depm) }, 'quiet': 'yes' } r = ck.access(ii) if r['return'] > 0: return r udepm = r['deps']['weights'].get( 'choices', []) # All UOAs of env for TensorFlow models. if len(udepm) == 0: return {'return': 1, 'error': 'no installed TensorFlow models'} cdeps[library_key]['uoa'] = udepl[0] cdeps['weights']['uoa'] = udepm[0] cdeps['imagenet-val']['uoa'] = imagenet_val['data_uoa'] cdeps['imagenet-aux']['uoa'] = imagenet_aux['data_uoa'] env = { 'CK_ENV_DATASET_IMAGENET_VAL': img_dir_val, 'CK_BATCH_COUNT': batch_count, 'CK_RESULTS_DIR': 'predictions', 'CK_SKIP_IMAGES': 0 } if arg.accuracy: env['CK_SILENT_MODE'] = 'YES' # Pass additional env vars from input if 'env' in i: for key in i['env']: env[key] = i['env'][key] ii = { 'action': 'pipeline', 'prepare': 'yes', 'dependencies': cdeps, 'module_uoa': 'program', 'data_uoa': program, 'target_os': tos, 'device_id': tdid, 'no_state_check': 'yes', 'no_compiler_description': 'yes', 'skip_calibration': 'yes', 'env': env, 'cpu_freq': 'max', 'gpu_freq': 'max', 'flags': '-O3', 'speed': 'no', 'energy': 'no', 'skip_print_timers': 'yes', 'out': 'con' } r = ck.access(ii) if r['return'] > 0: return r fail = r.get('fail', '') if fail == 'yes': return { 'return': 10, 'error': 'pipeline failed (' + r.get('fail_reason', '') + ')' } ready = r.get('ready', '') if ready != 'yes': return {'return': 11, 'error': 'pipeline not ready'} state = r['state'] tmp_dir = state['tmp_dir'] # Remember resolved deps for this benchmarking session. xcdeps = r.get('dependencies', {}) # Clean pipeline. if 'ready' in r: del (r['ready']) if 'fail' in r: del (r['fail']) if 'return' in r: del (r['return']) experiment_count = 0 pipeline = copy.deepcopy(r) # For each TensorFlow lib.************************************************* for lib_uoa in udepl: # Load TensorFlow lib. ii = {'action': 'load', 'module_uoa': 'env', 'data_uoa': lib_uoa} r = ck.access(ii) if r['return'] > 0: return r lib_name = r['data_name'] lib_tags = lib_type + '-' + r['dict']['customize']['version'] # Skip other libraries if one is explicitly specified. if arg.library_uoa and lib_uoa != arg.library_uoa: continue # Skip some libs with "in [..]" or "not in [..]". if lib_uoa in []: continue # Skip XLA libs. if arg.accuracy and r['dict']['customize']['install_env'].get( 'CK_TF_ENABLE_XLA') == 'YES': continue skip_compile = 'no' # For each TensorFlow model.************************************************* for model_uoa in udepm: # Load TensorFlow model. ii = {'action': 'load', 'module_uoa': 'env', 'data_uoa': model_uoa} r = ck.access(ii) if r['return'] > 0: return r model_name = r['data_name'] # Skip non-MobileNets models. if 'mobilenet' not in r['dict']['tags']: continue # Skip aggregate MobileNets packages. if 'mobilenet-all' in r['dict']['tags']: continue version = r['dict']['env'][ 'CK_ENV_TENSORFLOW_MODEL_MOBILENET_VERSION'] multiplier = float( r['dict']['env'] ['CK_ENV_TENSORFLOW_MODEL_MOBILENET_MULTIPLIER']) resolution = int(r['dict']['env'] ['CK_ENV_TENSORFLOW_MODEL_MOBILENET_RESOLUTION']) record_repo = 'local' record_uoa = '{}-{}-{}-mobilenet-v{}-{:.2f}-{}'.format( experiment_type, platform_tags, lib_tags, version, multiplier, resolution) # Skip the experiment if it already exists. if arg.resume: r = ck.access({ 'action': 'search', 'module_uoa': 'experiment', 'repo_uoa': record_repo, 'data_uoa': record_uoa }) if r['return'] > 0: return r if len(r['lst']) > 0: ck.out('Experiment "%s" already exists, skipping...' % record_uoa) continue # Prepare pipeline. ck.out( '---------------------------------------------------------------------------------------' ) ck.out('%s - %s' % (lib_name, lib_uoa)) ck.out('%s - %s' % (model_name, model_uoa)) ck.out('Experiment - %s:%s' % (record_repo, record_uoa)) experiment_count += 1 # Prepare autotuning input. cpipeline = copy.deepcopy(pipeline) # Reset deps and change UOA. new_deps = { 'library': copy.deepcopy(depl), 'weights': copy.deepcopy(depm) } new_deps['library']['uoa'] = lib_uoa new_deps['weights']['uoa'] = model_uoa jj = { 'action': 'resolve', 'module_uoa': 'env', 'host_os': hos, 'target_os': tos, 'device_id': tdid, 'deps': new_deps } r = ck.access(jj) if r['return'] > 0: return r cpipeline['dependencies'].update(new_deps) cpipeline['no_clean'] = skip_compile cpipeline['no_compile'] = skip_compile # Prepare common meta for ReQuEST tournament. features = copy.deepcopy(cpipeline['features']) platform_dict['features'].update(features) r = ck.access({ 'action': 'prepare_common_meta', 'module_uoa': 'request.asplos18', 'platform_dict': platform_dict, 'deps': cpipeline['dependencies'], 'request_dict': request_dict }) if r['return'] > 0: return r record_dict = r['record_dict'] meta = r['meta'] if random_name: rx = ck.gen_uid({}) if rx['return'] > 0: return rx record_uoa = rx['data_uid'] tags = r['tags'] tags.append(experiment_type) tags.append('explore-mobilenets-' + experiment_type) tags.append(lib_tags) tags.append(platform_tags) tags.append(str(resolution)) tags.append(str(multiplier)) tags.append('mobilenet-v{}'.format(version)) tags.append('mobilenet-v{}-{:.2f}-{}'.format( version, multiplier, resolution)) ii = { 'action': 'autotune', 'module_uoa': 'pipeline', 'data_uoa': 'program', 'choices_order': [ ['##choices#env#CK_BATCH_SIZE'], [ '##choices#env#CK_ENV_TENSORFLOW_MODEL_MOBILENET_VERSION' ], [ '##choices#env#CK_ENV_TENSORFLOW_MODEL_MOBILENET_MULTIPLIER' ], [ '##choices#env#CK_ENV_TENSORFLOW_MODEL_MOBILENET_RESOLUTION' ], ], 'choices_selection': [ { 'type': 'loop', 'start': bs['start'], 'stop': bs['stop'], 'step': bs['step'], 'default': bs['default'] }, { 'type': 'loop', 'choice': [version], 'default': version }, { 'type': 'loop', 'choice': [multiplier], 'default': 1.0 }, { 'type': 'loop', 'choice': [resolution], 'default': 224 }, ], 'features_keys_to_process': ['##choices#*'], 'iterations': -1, 'repetitions': num_repetitions, 'record': 'yes', 'record_failed': 'yes', 'record_params': { 'search_point_by_features': 'yes' }, 'tags': tags, 'meta': meta, 'record_dict': record_dict, 'record_repo': record_repo, 'record_uoa': record_uoa, 'pipeline': cpipeline, 'out': 'con' } if not arg.dry_run: r = ck.access(ii) if r['return'] > 0: return r fail = r.get('fail', '') if fail == 'yes': return { 'return': 10, 'error': 'pipeline failed (' + r.get('fail_reason', '') + ')' } if arg.dry_run: ck.out( '---------------------------------------------------------------------------------------' ) ck.out('Experiment count: %d' % experiment_count) ### end pipeline return {'return': 0}
def do(i, arg): random_name = arg.random_name target = arg.target cmd_key = arg.cmd_key # Detect basic platform info. ii = { 'action': 'detect', 'module_uoa': 'platform', 'target': target, 'out': 'out' } r = ck.access(ii) if r['return'] > 0: return r # Keep to prepare ReQuEST meta. platform_dict = copy.deepcopy(r) # Host and target OS params. hos = r['host_os_uoa'] hosd = r['host_os_dict'] tos = r['os_uoa'] tosd = r['os_dict'] tdid = r['device_id'] ii = { 'action': 'pipeline', 'prepare': 'yes', 'module_uoa': 'program', 'data_uoa': program, 'cmd_key': cmd_key, 'target': target, 'target_os': tos, 'device_id': tdid, 'no_state_check': 'yes', 'no_compiler_description': 'yes', 'skip_calibration': 'yes', 'cpu_freq': 'max', 'gpu_freq': 'max', 'flags': '-O3', 'speed': 'no', 'energy': 'no', 'skip_print_timers': 'yes', 'out': 'con' } r = ck.access(ii) if r['return'] > 0: return r fail = r.get('fail', '') if fail == 'yes': return { 'return': 10, 'error': 'pipeline failed (' + r.get('fail_reason', '') + ')' } ready = r.get('ready', '') if ready != 'yes': return {'return': 11, 'error': 'pipeline not ready'} state = r['state'] tmp_dir = state['tmp_dir'] # Remember resolved deps for this benchmarking session. xdeps = r.get('dependencies', {}) # Clean pipeline. if 'ready' in r: del (r['ready']) if 'fail' in r: del (r['fail']) if 'return' in r: del (r['return']) pipeline = copy.deepcopy(r) for dummy in ['dummy']: for dummy2 in ['dummy2']: record_repo = 'local' record_uoa = 'ck-request-asplos18-iot-farm-' + cmd_key # Prepare pipeline. ck.out( '---------------------------------------------------------------------------------------' ) ck.out('CMD key - ' + cmd_key) ck.out('Experiment - %s:%s' % (record_repo, record_uoa)) # Prepare autotuning input. cpipeline = copy.deepcopy(pipeline) cpipeline['cmd_key'] = cmd_key # Prepare common meta for ReQuEST tournament features = copy.deepcopy(cpipeline['features']) platform_dict['features'].update(features) r = ck.access({ 'action': 'prepare_common_meta', 'module_uoa': 'request.asplos18', 'platform_dict': platform_dict, 'deps': cpipeline['dependencies'], 'request_dict': request_dict }) if r['return'] > 0: return r record_dict = r['record_dict'] meta = r['meta'] if random_name: rx = ck.gen_uid({}) if rx['return'] > 0: return rx record_uoa = rx['data_uid'] tags = r['tags'] tags.append(program) ii = { 'action': 'autotune', 'target': target, 'module_uoa': 'pipeline', 'data_uoa': 'program', 'iterations': 1, 'repetitions': num_repetitions, 'record': 'yes', 'record_failed': 'yes', 'record_params': { 'search_point_by_features': 'yes' }, 'tags': tags, 'meta': meta, 'record_dict': record_dict, 'record_repo': record_repo, 'record_uoa': record_uoa, 'pipeline': cpipeline, 'out': 'con' } r = ck.access(ii) if r['return'] > 0: return r fail = r.get('fail', '') if fail == 'yes': return { 'return': 10, 'error': 'pipeline failed (' + r.get('fail_reason', '') + ')' } skip_compile = 'yes' return {'return': 0}