def validate_onnx_model(platform, device_type, model_file, input_file, mace_out_file, input_names, input_shapes, input_data_formats, output_names, output_shapes, output_data_formats, validation_threshold, input_data_types, backend, log_file): import onnx if backend == "tensorflow": from onnx_tf.backend import prepare print("valivate on onnx tensorflow backend.") elif backend == "caffe2" or backend == "pytorch": from caffe2.python.onnx.backend import prepare print("valivate on onnx caffe2 backend.") else: common.MaceLogger.error( VALIDATION_MODULE, "onnx backend framwork '" + backend + "' is invalid.") if not os.path.isfile(model_file): common.MaceLogger.error( VALIDATION_MODULE, "Input graph file '" + model_file + "' does not exist!") model = onnx.load(model_file) input_dict = {} for i in range(len(input_names)): input_value = load_data(common.formatted_file_name(input_file, input_names[i]), input_data_types[i]) input_value = input_value.reshape(input_shapes[i]) if input_data_formats[i] == common.DataFormat.NHWC and \ len(input_shapes[i]) == 4: input_value = input_value.transpose((0, 3, 1, 2)) input_dict[input_names[i]] = input_value onnx_outputs = [] for i in range(len(output_names)): out_shape = output_shapes[i][:] if output_data_formats[i] == common.DataFormat.NHWC and\ len(out_shape) == 4: out_shape[1], out_shape[2], out_shape[3] = \ out_shape[3], out_shape[1], out_shape[2] onnx_outputs.append( onnx.helper.make_tensor_value_info(output_names[i], onnx.TensorProto.FLOAT, out_shape)) model.graph.output.extend(onnx_outputs) rep = prepare(model) output_values = rep.run(input_dict) for i in range(len(output_names)): out_name = output_names[i] value = output_values[out_name].flatten() output_file_name = common.formatted_file_name(mace_out_file, output_names[i]) mace_out_value = load_data(output_file_name) if output_data_formats[i] == common.DataFormat.NHWC and \ len(output_shapes[i]) == 4: mace_out_value = mace_out_value.reshape(output_shapes[i]) \ .transpose((0, 3, 1, 2)) compare_output(platform, device_type, output_names[i], mace_out_value, value, validation_threshold, log_file)
def validate_with_file(platform, device_type, output_names, output_shapes, mace_out_file, validation_outputs_data, validation_threshold, log_file, output_data_formats): for i in range(len(output_names)): if validation_outputs_data[i].startswith("http://") or \ validation_outputs_data[i].startswith("https://"): validation_file_name = common.formatted_file_name( mace_out_file, output_names[i] + '_validation') six.moves.urllib.request.urlretrieve(validation_outputs_data[i], validation_file_name) else: validation_file_name = validation_outputs_data[i] value = load_data(validation_file_name) out_shape = output_shapes[i] output_file_name = common.formatted_file_name( mace_out_file, output_names[i]) mace_out_value = load_data(output_file_name) mace_out_value, real_output_shape, real_output_data_format = \ get_real_out_value_shape_df(platform, mace_out_value, output_shapes[i], output_data_formats[i]) compare_output(platform, device_type, output_names[i], mace_out_value, value, validation_threshold, log_file, real_output_shape, real_output_data_format)
def validate_tf_model(platform, device_type, model_file, input_file, mace_out_file, input_names, input_shapes, input_data_formats, output_names, output_shapes, output_data_formats, validation_threshold, input_data_types, log_file): import tensorflow as tf if not os.path.isfile(model_file): common.MaceLogger.error( VALIDATION_MODULE, "Input graph file '" + model_file + "' does not exist!") tf.reset_default_graph() input_graph_def = tf.GraphDef() with open(model_file, "rb") as f: data = f.read() input_graph_def.ParseFromString(data) tf.import_graph_def(input_graph_def, name="") with tf.Session() as session: with session.graph.as_default() as graph: tf.import_graph_def(input_graph_def, name="") input_dict = {} for i in range(len(input_names)): input_value = load_data( common.formatted_file_name(input_file, input_names[i]), input_data_types[i]) input_value = input_value.reshape(input_shapes[i]) if input_data_formats[i] == common.DataFormat.NCHW and\ len(input_shapes[i]) == 4: input_value = input_value.transpose((0, 2, 3, 1)) elif input_data_formats[i] == common.DataFormat.OIHW and \ len(input_shapes[i]) == 4: # OIHW -> HWIO input_value = input_value.transpose((2, 3, 1, 0)) input_node = graph.get_tensor_by_name( normalize_tf_tensor_name(input_names[i])) input_dict[input_node] = input_value output_nodes = [] for name in output_names: output_nodes.extend( [graph.get_tensor_by_name( normalize_tf_tensor_name(name))]) output_values = session.run(output_nodes, feed_dict=input_dict) for i in range(len(output_names)): output_file_name = common.formatted_file_name( mace_out_file, output_names[i]) mace_out_value = load_data(output_file_name) mace_out_value, real_out_shape, real_out_data_format = \ get_real_out_value_shape_df(platform, mace_out_value, output_shapes[i], output_data_formats[i]) compare_output(platform, device_type, output_names[i], mace_out_value, output_values[i], validation_threshold, log_file, real_out_shape, real_out_data_format)
def validate_onnx_model(platform, device_type, model_file, input_file, mace_out_file, input_names, input_shapes, input_data_formats, output_names, output_shapes, output_data_formats, validation_threshold, input_data_types, backend, log_file): print("validate on onnxruntime.") import onnx import onnxruntime as onnxrt if not os.path.isfile(model_file): common.MaceLogger.error( VALIDATION_MODULE, "Input graph file '" + model_file + "' does not exist!") model = onnx.load(model_file) remove_initializer_from_input(model) model_outputs = set() for output in model.graph.output: model_outputs.add(output.name) for output_name in output_names: if output_name not in model_outputs: layer_value_info = onnx.helper.ValueInfoProto() layer_value_info.name = output_name model.graph.output.append(layer_value_info) input_dict = {} for i in range(len(input_names)): input_value = load_data(common.formatted_file_name(input_file, input_names[i]), input_data_types[i]) input_value = input_value.reshape(input_shapes[i]) if input_data_formats[i] == common.DataFormat.NHWC and \ len(input_shapes[i]) == 4: input_value = input_value.transpose((0, 3, 1, 2)) input_dict[input_names[i]] = input_value sess = onnxrt.InferenceSession(model.SerializeToString()) output_values = sess.run(output_names, input_dict) for i in range(len(output_names)): value = output_values[i].flatten() output_file_name = common.formatted_file_name(mace_out_file, output_names[i]) mace_out_value = load_data(output_file_name) mace_out_value, real_output_shape, real_output_data_format = \ get_real_out_value_shape_df(platform, mace_out_value, output_shapes[i], output_data_formats[i]) compare_output(platform, device_type, output_names[i], mace_out_value, value, validation_threshold, log_file, real_output_shape, real_output_data_format)
def validate_caffe_model(platform, device_type, model_file, input_file, mace_out_file, weight_file, input_names, input_shapes, input_data_formats, output_names, output_shapes, output_data_formats, validation_threshold, log_file): os.environ['GLOG_minloglevel'] = '1' # suprress Caffe verbose prints import caffe if not os.path.isfile(model_file): common.MaceLogger.error( VALIDATION_MODULE, "Input graph file '" + model_file + "' does not exist!") if not os.path.isfile(weight_file): common.MaceLogger.error( VALIDATION_MODULE, "Input weight file '" + weight_file + "' does not exist!") caffe.set_mode_cpu() net = caffe.Net(model_file, caffe.TEST, weights=weight_file) for i in range(len(input_names)): input_value = load_data( common.formatted_file_name(input_file, input_names[i])) input_value = input_value.reshape(input_shapes[i]) if input_data_formats[i] == common.DataFormat.NHWC and \ len(input_shapes[i]) == 4: input_value = input_value.transpose((0, 3, 1, 2)) input_blob_name = input_names[i] try: if input_names[i] in net.top_names: input_blob_name = net.top_names[input_names[i]][0] except ValueError: pass new_shape = input_value.shape net.blobs[input_blob_name].reshape(*new_shape) for index in range(input_value.shape[0]): net.blobs[input_blob_name].data[index] = input_value[index] net.forward() for i in range(len(output_names)): value = net.blobs[output_names[i]].data output_file_name = common.formatted_file_name( mace_out_file, output_names[i]) mace_out_value = load_data(output_file_name) mace_out_value, real_output_shape, real_output_data_format = \ get_real_out_value_shape_df(platform, mace_out_value, output_shapes[i], output_data_formats[i]) compare_output(platform, device_type, output_names[i], mace_out_value, value, validation_threshold, log_file, real_output_shape, real_output_data_format)
def generate_data(name, shape, input_file, tensor_range): np.random.seed() data = np.random.random(shape) * (tensor_range[1] - tensor_range[0]) \ + tensor_range[0] input_file_name = common.formatted_file_name(input_file, name) print 'Generate input file: ', input_file_name data.astype(np.float32).tofile(input_file_name)
def gen_random_input(model_output_dir, input_nodes, input_shapes, input_files, input_ranges, input_data_types, input_file_name="model_input"): for input_name in input_nodes: formatted_name = common.formatted_file_name( input_file_name, input_name) if os.path.exists("%s/%s" % (model_output_dir, formatted_name)): sh.rm("%s/%s" % (model_output_dir, formatted_name)) input_nodes_str = ",".join(input_nodes) input_shapes_str = ":".join(input_shapes) input_ranges_str = ":".join(input_ranges) input_data_types_str = ",".join(input_data_types) generate_input_data("%s/%s" % (model_output_dir, input_file_name), input_nodes_str, input_shapes_str, input_ranges_str, input_data_types_str) input_file_list = [] if isinstance(input_files, list): input_file_list.extend(input_files) else: input_file_list.append(input_files) if len(input_file_list) != 0: input_name_list = [] if isinstance(input_nodes, list): input_name_list.extend(input_nodes) else: input_name_list.append(input_nodes) if len(input_file_list) != len(input_name_list): raise Exception('If input_files set, the input files should ' 'match the input names.') for i in range(len(input_file_list)): if input_file_list[i] is not None: dst_input_file = model_output_dir + '/' + \ common.formatted_file_name(input_file_name, input_name_list[i]) if input_file_list[i].startswith("http://") or \ input_file_list[i].startswith("https://"): six.moves.urllib.request.urlretrieve(input_file_list[i], dst_input_file) else: sh.cp("-f", input_file_list[i], dst_input_file)
def validate_keras_model(platform, device_type, model_file, input_file, mace_out_file, input_names, input_shapes, input_data_formats, output_names, output_shapes, output_data_formats, validation_threshold, input_data_types, log_file): from tensorflow import keras import tensorflow_model_optimization as tfmot if not os.path.isfile(model_file): common.MaceLogger.error( VALIDATION_MODULE, "Input model file '" + model_file + "' does not exist!") with tfmot.quantization.keras.quantize_scope(): keras_model = keras.models.load_model(model_file, compile=False) input = [] for i in range(len(input_names)): input_value = load_data( common.formatted_file_name(input_file, input_names[i]), input_data_types[i]) input_value = input_value.reshape(input_shapes[i]) if input_data_formats[i] == common.DataFormat.NCHW and \ len(input_shapes[i]) == 4: input_value = input_value.transpose((0, 2, 3, 1)) elif input_data_formats[i] == common.DataFormat.OIHW and \ len(input_shapes[i]) == 4: # OIHW -> HWIO input_value = input_value.transpose((2, 3, 1, 0)) input.append(input_value) output_values = keras_model.predict(input) for i in range(len(output_names)): output_file_name = common.formatted_file_name( mace_out_file, output_names[i]) mace_out_value = load_data(output_file_name) mace_out_value, real_output_shape, real_output_data_format = \ get_real_out_value_shape_df(platform, mace_out_value, output_shapes[i], output_data_formats[i]) compare_output(platform, device_type, output_names[i], mace_out_value, output_values[i], validation_threshold, log_file, real_output_shape, real_output_data_format)
def validate_megengine_model(platform, device_type, model_file, input_file, mace_out_file, input_names, input_shapes, input_data_formats, output_names, output_shapes, output_data_formats, validation_threshold, input_data_types, log_file): import megengine._internal as mgb if not os.path.isfile(model_file): common.MaceLogger.error( VALIDATION_MODULE, "Input graph file '" + model_file + "' does not exist!", ) feed_inputs = [] for i in range(len(input_names)): input_value = load_data( common.formatted_file_name(input_file, input_names[i]), input_data_types[i]) input_value = input_value.reshape(input_shapes[i]) if (input_data_formats[i] == common.DataFormat.NHWC and len(input_shapes[i]) == 4): input_value = input_value.transpose((0, 3, 1, 2)) feed_inputs.append(input_value) cg, _, outputs = mgb.load_comp_graph_from_file(model_file) inputs = mgb.cgtools.get_dep_vars(outputs, "Host2DeviceCopy") inputs = sorted(inputs, key=lambda i: i.name) outputs = list(map(mgb.copy_output, outputs)) if len(outputs) == 1: (outputs,) = outputs func = cg.compile(inputs, outputs) mge_output_value = func(*feed_inputs) for i in range(len(output_names)): output_file_name = \ common.formatted_file_name(mace_out_file, output_names[i]) mace_out_value = load_data(output_file_name) mace_out_value, real_output_shape, real_output_data_format = \ get_real_out_value_shape_df(platform, mace_out_value, output_shapes[i], output_data_formats[i]) compare_output(platform, device_type, output_names[i], mace_out_value, mge_output_value, validation_threshold, log_file, real_output_shape, real_output_data_format)
def gen_random_input(model_output_dir, input_nodes, input_shapes, input_files, input_ranges, input_data_types, input_file_name="model_input"): for input_name in input_nodes: formatted_name = common.formatted_file_name( input_file_name, input_name) if os.path.exists("%s/%s" % (model_output_dir, formatted_name)): sh.rm("%s/%s" % (model_output_dir, formatted_name)) input_nodes_str = ",".join(input_nodes) input_shapes_str = ":".join(input_shapes) input_ranges_str = ":".join(input_ranges) input_data_types_str = ",".join(input_data_types) generate_input_data("%s/%s" % (model_output_dir, input_file_name), input_nodes_str, input_shapes_str, input_ranges_str, input_data_types_str) input_file_list = [] if isinstance(input_files, list): input_file_list.extend(input_files) else: input_file_list.append(input_files) if len(input_file_list) != 0: input_name_list = [] if isinstance(input_nodes, list): input_name_list.extend(input_nodes) else: input_name_list.append(input_nodes) if len(input_file_list) != len(input_name_list): raise Exception('If input_files set, the input files should ' 'match the input names.') for i in range(len(input_file_list)): if input_file_list[i] is not None: dst_input_file = model_output_dir + '/' + \ common.formatted_file_name(input_file_name, input_name_list[i]) if input_file_list[i].startswith("http://") or \ input_file_list[i].startswith("https://"): urllib.urlretrieve(input_file_list[i], dst_input_file) else: sh.cp("-f", input_file_list[i], dst_input_file)
def validate_caffe_model(platform, device_type, model_file, input_file, mace_out_file, weight_file, input_names, input_shapes, output_names, output_shapes): os.environ['GLOG_minloglevel'] = '1' # suprress Caffe verbose prints import caffe if not os.path.isfile(model_file): common.MaceLogger.error( VALIDATION_MODULE, "Input graph file '" + model_file + "' does not exist!") if not os.path.isfile(weight_file): common.MaceLogger.error( VALIDATION_MODULE, "Input weight file '" + weight_file + "' does not exist!") caffe.set_mode_cpu() net = caffe.Net(model_file, caffe.TEST, weights=weight_file) for i in range(len(input_names)): input_value = load_data( common.formatted_file_name(input_file, input_names[i])) input_value = input_value.reshape(input_shapes[i]).transpose( (0, 3, 1, 2)) input_blob_name = input_names[i] try: if input_names[i] in net.top_names: input_blob_name = net.top_names[input_names[i]][0] except ValueError: pass net.blobs[input_blob_name].data[0] = input_value net.forward() for i in range(len(output_names)): value = net.blobs[net.top_names[output_names[i]][0]].data out_shape = output_shapes[i] if len(out_shape) == 4: out_shape[1], out_shape[2], out_shape[3] = \ out_shape[3], out_shape[1], out_shape[2] value = value.reshape(out_shape).transpose((0, 2, 3, 1)) output_file_name = common.formatted_file_name(mace_out_file, output_names[i]) mace_out_value = load_data(output_file_name) compare_output(platform, device_type, output_names[i], mace_out_value, value)
def validate_caffe_model(platform, device_type, model_file, input_file, mace_out_file, weight_file, input_names, input_shapes, output_names, output_shapes): os.environ['GLOG_minloglevel'] = '1' # suprress Caffe verbose prints import caffe if not os.path.isfile(model_file): common.MaceLogger.error( VALIDATION_MODULE, "Input graph file '" + model_file + "' does not exist!") if not os.path.isfile(weight_file): common.MaceLogger.error( VALIDATION_MODULE, "Input weight file '" + weight_file + "' does not exist!") caffe.set_mode_cpu() net = caffe.Net(model_file, caffe.TEST, weights=weight_file) for i in range(len(input_names)): input_value = load_data( common.formatted_file_name(input_file, input_names[i])) input_value = input_value.reshape(input_shapes[i]).transpose((0, 3, 1, 2)) input_blob_name = input_names[i] try: if input_names[i] in net.top_names: input_blob_name = net.top_names[input_names[i]][0] except ValueError: pass net.blobs[input_blob_name].data[0] = input_value net.forward() for i in range(len(output_names)): value = net.blobs[net.top_names[output_names[i]][0]].data out_shape = output_shapes[i] if len(out_shape) == 4: out_shape[1], out_shape[2], out_shape[3] = \ out_shape[3], out_shape[1], out_shape[2] value = value.reshape(out_shape).transpose((0, 2, 3, 1)) output_file_name = common.formatted_file_name( mace_out_file, output_names[i]) mace_out_value = load_data(output_file_name) compare_output(platform, device_type, output_names[i], mace_out_value, value)
def generate_data(name, shape, input_file, tensor_range, input_data_type): np.random.seed() data = np.random.random(shape) * (tensor_range[1] - tensor_range[0]) \ + tensor_range[0] input_file_name = common.formatted_file_name(input_file, name) six.print_('Generate input file: ', input_file_name) if input_data_type == 'float32': np_data_type = np.float32 elif input_data_type == 'int32': np_data_type = np.int32 data.astype(np_data_type).tofile(input_file_name)
def generate_data(name, shape, input_file, tensor_range, input_data_type): np.random.seed() data = np.random.random(shape) * (tensor_range[1] - tensor_range[0]) \ + tensor_range[0] input_file_name = common.formatted_file_name(input_file, name) print 'Generate input file: ', input_file_name if input_data_type == 'float32': np_data_type = np.float32 elif input_data_type == 'int32': np_data_type = np.int32 data.astype(np_data_type).tofile(input_file_name)
def validate_pytorch_model(platform, device_type, model_file, input_file, mace_out_file, input_names, input_shapes, input_data_formats, output_names, output_shapes, output_data_formats, validation_threshold, input_data_types, log_file): import torch loaded_model = torch.jit.load(model_file) pytorch_inputs = [] for i in range(len(input_names)): input_value = load_data( common.formatted_file_name(input_file, input_names[i]), input_data_types[i]) input_value = input_value.reshape(input_shapes[i]) if input_data_formats[i] == common.DataFormat.NHWC and \ len(input_shapes[i]) == 4: input_value = input_value.transpose((0, 3, 1, 2)) input_value = torch.from_numpy(input_value) pytorch_inputs.append(input_value) with torch.no_grad(): pytorch_outputs = loaded_model(*pytorch_inputs) if isinstance(pytorch_outputs, torch.Tensor): pytorch_outputs = [pytorch_outputs] else: if not isinstance(pytorch_outputs, (list, tuple)): print('return type {} unsupported yet'.format( type(pytorch_outputs))) sys.exit(1) for i in range(len(output_names)): value = pytorch_outputs[i].numpy() output_file_name = common.formatted_file_name( mace_out_file, output_names[i]) mace_out_value = load_data(output_file_name) mace_out_value, real_output_shape, real_output_data_format = \ get_real_out_value_shape_df(platform, mace_out_value, output_shapes[i], output_data_formats[i]) compare_output(platform, device_type, output_names[i], mace_out_value, value, validation_threshold, log_file, real_output_shape, real_output_data_format)
def validate_tf_model(platform, device_type, model_file, input_file, mace_out_file, input_names, input_shapes, output_names): import tensorflow as tf if not os.path.isfile(model_file): common.MaceLogger.error( VALIDATION_MODULE, "Input graph file '" + model_file + "' does not exist!") tf.reset_default_graph() input_graph_def = tf.GraphDef() with open(model_file, "rb") as f: data = f.read() input_graph_def.ParseFromString(data) tf.import_graph_def(input_graph_def, name="") with tf.Session() as session: with session.graph.as_default() as graph: tf.import_graph_def(input_graph_def, name="") input_dict = {} for i in range(len(input_names)): input_value = load_data( common.formatted_file_name(input_file, input_names[i])) input_value = input_value.reshape(input_shapes[i]) input_node = graph.get_tensor_by_name( normalize_tf_tensor_name(input_names[i])) input_dict[input_node] = input_value output_nodes = [] for name in output_names: output_nodes.extend([ graph.get_tensor_by_name( normalize_tf_tensor_name(name)) ]) output_values = session.run(output_nodes, feed_dict=input_dict) for i in range(len(output_names)): output_file_name = common.formatted_file_name( mace_out_file, output_names[i]) mace_out_value = load_data(output_file_name) compare_output(platform, device_type, output_names[i], mace_out_value, output_values[i])
def validate_tf_model(platform, device_type, model_file, input_file, mace_out_file, input_names, input_shapes, output_names): import tensorflow as tf if not os.path.isfile(model_file): common.MaceLogger.error( VALIDATION_MODULE, "Input graph file '" + model_file + "' does not exist!") tf.reset_default_graph() input_graph_def = tf.GraphDef() with open(model_file, "rb") as f: data = f.read() input_graph_def.ParseFromString(data) tf.import_graph_def(input_graph_def, name="") with tf.Session() as session: with session.graph.as_default() as graph: tf.import_graph_def(input_graph_def, name="") input_dict = {} for i in range(len(input_names)): input_value = load_data( common.formatted_file_name(input_file, input_names[i])) input_value = input_value.reshape(input_shapes[i]) input_node = graph.get_tensor_by_name( normalize_tf_tensor_name(input_names[i])) input_dict[input_node] = input_value output_nodes = [] for name in output_names: output_nodes.extend( [graph.get_tensor_by_name( normalize_tf_tensor_name(name))]) output_values = session.run(output_nodes, feed_dict=input_dict) for i in range(len(output_names)): output_file_name = common.formatted_file_name( mace_out_file, output_names[i]) mace_out_value = load_data(output_file_name) compare_output(platform, device_type, output_names[i], mace_out_value, output_values[i])
def validate_with_file(platform, device_type, output_names, output_shapes, mace_out_file, validation_outputs_data, validation_threshold, log_file): for i in range(len(output_names)): if validation_outputs_data[i].startswith("http://") or \ validation_outputs_data[i].startswith("https://"): validation_file_name = common.formatted_file_name( mace_out_file, output_names[i] + '_validation') six.moves.urllib.request.urlretrieve(validation_outputs_data[i], validation_file_name) else: validation_file_name = validation_outputs_data[i] value = load_data(validation_file_name) out_shape = output_shapes[i] if len(out_shape) == 4: out_shape[1], out_shape[2], out_shape[3] = \ out_shape[3], out_shape[1], out_shape[2] value = value.reshape(out_shape).transpose((0, 2, 3, 1)) output_file_name = common.formatted_file_name(mace_out_file, output_names[i]) mace_out_value = load_data(output_file_name) compare_output(platform, device_type, output_names[i], mace_out_value, value, validation_threshold, log_file)
def get_output_map(self, target_abi, output_nodes, output_shapes, model_output_dir): output_map = {} for i in range(len(output_nodes)): output_name = output_nodes[i] formatted_name = common.formatted_file_name( "model_out", output_name) if target_abi != "host": if os.path.exists("%s/%s" % (model_output_dir, formatted_name)): sh.rm("-rf", "%s/%s" % (model_output_dir, formatted_name)) self.pull_from_data_dir(formatted_name, model_output_dir) output_file_path = os.path.join(model_output_dir, formatted_name) output_shape = [ int(x) for x in common.split_shape(output_shapes[i]) ] output_map[output_name] = np.fromfile( output_file_path, dtype=np.float32).reshape(output_shape) return output_map
def tuning_run( self, abi, target_dir, target_name, vlog_level, embed_model_data, model_output_dir, input_nodes, output_nodes, input_shapes, output_shapes, mace_model_dir, model_tag, device_type, running_round, restart_round, limit_opencl_kernel_time, tuning, out_of_range_check, model_graph_format, opencl_binary_file, opencl_parameter_file, libmace_dynamic_library_path, omp_num_threads=-1, cpu_affinity_policy=1, gpu_perf_hint=3, gpu_priority_hint=3, input_file_name='model_input', output_file_name='model_out', runtime_failure_ratio=0.0, address_sanitizer=False, link_dynamic=False, quantize_stat=False, ): six.print_("* Run '%s' with round=%s, restart_round=%s, tuning=%s, " "out_of_range_check=%s, omp_num_threads=%s, " "cpu_affinity_policy=%s, gpu_perf_hint=%s, " "gpu_priority_hint=%s" % (model_tag, running_round, restart_round, str(tuning), str(out_of_range_check), omp_num_threads, cpu_affinity_policy, gpu_perf_hint, gpu_priority_hint)) mace_model_path = "" if model_graph_format == ModelFormat.file: mace_model_path = "%s/%s.pb" % (mace_model_dir, model_tag) if self.system == SystemType.host: libmace_dynamic_lib_path = \ os.path.dirname(libmace_dynamic_library_path) p = subprocess.Popen([ "env", "LD_LIBRARY_PATH=%s" % libmace_dynamic_lib_path, "MACE_CPP_MIN_VLOG_LEVEL=%s" % vlog_level, "MACE_RUNTIME_FAILURE_RATIO=%f" % runtime_failure_ratio, "MACE_LOG_TENSOR_RANGE=%d" % (1 if quantize_stat else 0), "%s/%s" % (target_dir, target_name), "--model_name=%s" % model_tag, "--input_node=%s" % ",".join(input_nodes), "--output_node=%s" % ",".join(output_nodes), "--input_shape=%s" % ":".join(input_shapes), "--output_shape=%s" % ":".join(output_shapes), "--input_file=%s/%s" % (model_output_dir, input_file_name), "--output_file=%s/%s" % (model_output_dir, output_file_name), "--model_data_file=%s/%s.data" % (mace_model_dir, model_tag), "--device=%s" % device_type, "--round=%s" % running_round, "--restart_round=%s" % restart_round, "--omp_num_threads=%s" % omp_num_threads, "--cpu_affinity_policy=%s" % cpu_affinity_policy, "--gpu_perf_hint=%s" % gpu_perf_hint, "--gpu_priority_hint=%s" % gpu_priority_hint, "--model_file=%s" % mace_model_path, ], stderr=subprocess.PIPE, stdout=subprocess.PIPE) out, err = p.communicate() self.stdout = err + out six.print_(self.stdout) six.print_("Running finished!\n") elif self.system in [SystemType.android, SystemType.arm_linux]: self.rm(self.data_dir) self.exec_command('mkdir -p {}'.format(self.data_dir)) internal_storage_dir = self.create_internal_storage_dir() for input_name in input_nodes: formatted_name = common.formatted_file_name( input_file_name, input_name) self.push("%s/%s" % (model_output_dir, formatted_name), self.data_dir) if self.system == SystemType.android and address_sanitizer: self.push(sh_commands.find_asan_rt_library(abi), self.data_dir) if not embed_model_data: model_data_path = "%s/%s.data" % (mace_model_dir, model_tag) mace_check( os.path.exists(model_data_path), "Device", 'model data file not found,' ' please convert model first') self.push(model_data_path, self.data_dir) if device_type == common.DeviceType.GPU: if os.path.exists(opencl_binary_file): self.push(opencl_binary_file, self.data_dir) if os.path.exists(opencl_parameter_file): self.push(opencl_parameter_file, self.data_dir) self.push("third_party/nnlib/libhexagon_controller.so", self.data_dir) mace_model_phone_path = "" if model_graph_format == ModelFormat.file: mace_model_phone_path = "%s/%s.pb" % (self.data_dir, model_tag) self.push(mace_model_path, mace_model_phone_path) if link_dynamic: self.push(libmace_dynamic_library_path, self.data_dir) if self.system == SystemType.android: sh_commands.push_depended_so_libs( libmace_dynamic_library_path, abi, self.data_dir, self.address) self.push("%s/%s" % (target_dir, target_name), self.data_dir) stdout_buff = [] process_output = sh_commands.make_output_processor(stdout_buff) cmd = [ "LD_LIBRARY_PATH=%s" % self.data_dir, "MACE_TUNING=%s" % int(tuning), "MACE_OUT_OF_RANGE_CHECK=%s" % int(out_of_range_check), "MACE_CPP_MIN_VLOG_LEVEL=%s" % vlog_level, "MACE_RUN_PARAMETER_PATH=%s/mace_run.config" % self.data_dir, "MACE_INTERNAL_STORAGE_PATH=%s" % internal_storage_dir, "MACE_LIMIT_OPENCL_KERNEL_TIME=%s" % limit_opencl_kernel_time, "MACE_RUNTIME_FAILURE_RATIO=%f" % runtime_failure_ratio, "MACE_LOG_TENSOR_RANGE=%d" % (1 if quantize_stat else 0), ] if self.system == SystemType.android and address_sanitizer: cmd.extend([ "LD_PRELOAD=%s/%s" % (self.data_dir, sh_commands.asan_rt_library_names(abi)) ]) cmd.extend([ "%s/%s" % (self.data_dir, target_name), "--model_name=%s" % model_tag, "--input_node=%s" % ",".join(input_nodes), "--output_node=%s" % ",".join(output_nodes), "--input_shape=%s" % ":".join(input_shapes), "--output_shape=%s" % ":".join(output_shapes), "--input_file=%s/%s" % (self.data_dir, input_file_name), "--output_file=%s/%s" % (self.data_dir, output_file_name), "--model_data_file=%s/%s.data" % (self.data_dir, model_tag), "--device=%s" % device_type, "--round=%s" % running_round, "--restart_round=%s" % restart_round, "--omp_num_threads=%s" % omp_num_threads, "--cpu_affinity_policy=%s" % cpu_affinity_policy, "--gpu_perf_hint=%s" % gpu_perf_hint, "--gpu_priority_hint=%s" % gpu_priority_hint, "--model_file=%s" % mace_model_phone_path, "--opencl_binary_file=%s/%s" % (self.data_dir, os.path.basename(opencl_binary_file)), "--opencl_parameter_file=%s/%s" % (self.data_dir, os.path.basename(opencl_parameter_file)), ]) cmd = ' '.join(cmd) cmd_file_name = "%s-%s-%s" % ('cmd_file', model_tag, str(time.time())) cmd_file = "%s/%s" % (self.data_dir, cmd_file_name) tmp_cmd_file = "%s/%s" % ('/tmp', cmd_file_name) with open(tmp_cmd_file, 'w') as file: file.write(cmd) self.push(tmp_cmd_file, cmd_file) os.remove(tmp_cmd_file) self.exec_command('sh {}'.format(cmd_file), _tty_in=True, _out=process_output, _err_to_out=True) self.stdout = "".join(stdout_buff) if not sh_commands.stdout_success(self.stdout): common.MaceLogger.error("Mace Run", "Mace run failed.") six.print_("Running finished!\n") else: six.print_('Unsupported system %s' % self.system, file=sys.stderr) raise Exception('Wrong device') return self.stdout
def validate_model(abi, device, model_file_path, weight_file_path, docker_image_tag, dockerfile_path, platform, device_type, input_nodes, output_nodes, input_shapes, output_shapes, input_data_formats, output_data_formats, model_output_dir, input_data_types, caffe_env, input_file_name="model_input", output_file_name="model_out", validation_threshold=0.9, backend="tensorflow", validation_outputs_data=[], log_file=""): if not validation_outputs_data: six.print_("* Validate with %s" % platform) else: six.print_("* Validate with file: %s" % validation_outputs_data) if abi != "host": for output_name in output_nodes: formatted_name = common.formatted_file_name( output_file_name, output_name) if os.path.exists("%s/%s" % (model_output_dir, formatted_name)): sh.rm("-rf", "%s/%s" % (model_output_dir, formatted_name)) device.pull_from_data_dir(formatted_name, model_output_dir) if platform == "tensorflow" or platform == "onnx": validate(platform, model_file_path, "", "%s/%s" % (model_output_dir, input_file_name), "%s/%s" % (model_output_dir, output_file_name), device_type, ":".join(input_shapes), ":".join(output_shapes), ",".join(input_data_formats), ",".join(output_data_formats), ",".join(input_nodes), ",".join(output_nodes), validation_threshold, ",".join(input_data_types), backend, validation_outputs_data, log_file) elif platform == "caffe": image_name = "mace-caffe:" + docker_image_tag container_name = "mace_caffe_" + docker_image_tag + "_validator" if caffe_env == common.CaffeEnvType.LOCAL: try: import caffe except ImportError: logging.error('There is no caffe python module.') validate(platform, model_file_path, weight_file_path, "%s/%s" % (model_output_dir, input_file_name), "%s/%s" % (model_output_dir, output_file_name), device_type, ":".join(input_shapes), ":".join(output_shapes), ",".join(input_data_formats), ",".join(output_data_formats), ",".join(input_nodes), ",".join(output_nodes), validation_threshold, ",".join(input_data_types), backend, validation_outputs_data, log_file) elif caffe_env == common.CaffeEnvType.DOCKER: docker_image_id = sh.docker("images", "-q", image_name) if not docker_image_id: six.print_("Build caffe docker") sh.docker("build", "-t", image_name, dockerfile_path) container_id = sh.docker("ps", "-qa", "-f", "name=%s" % container_name) if container_id and not sh.docker("ps", "-qa", "--filter", "status=running", "-f", "name=%s" % container_name): sh.docker("rm", "-f", container_name) container_id = "" if not container_id: six.print_("Run caffe container") sh.docker("run", "-d", "-it", "--name", container_name, image_name, "/bin/bash") for input_name in input_nodes: formatted_input_name = common.formatted_file_name( input_file_name, input_name) sh.docker("cp", "%s/%s" % (model_output_dir, formatted_input_name), "%s:/mace" % container_name) for output_name in output_nodes: formatted_output_name = common.formatted_file_name( output_file_name, output_name) sh.docker("cp", "%s/%s" % (model_output_dir, formatted_output_name), "%s:/mace" % container_name) model_file_name = os.path.basename(model_file_path) weight_file_name = os.path.basename(weight_file_path) sh.docker("cp", "tools/common.py", "%s:/mace" % container_name) sh.docker("cp", "tools/validate.py", "%s:/mace" % container_name) sh.docker("cp", model_file_path, "%s:/mace" % container_name) sh.docker("cp", weight_file_path, "%s:/mace" % container_name) sh.docker("exec", container_name, "python", "-u", "/mace/validate.py", "--platform=caffe", "--model_file=/mace/%s" % model_file_name, "--weight_file=/mace/%s" % weight_file_name, "--input_file=/mace/%s" % input_file_name, "--mace_out_file=/mace/%s" % output_file_name, "--device_type=%s" % device_type, "--input_node=%s" % ",".join(input_nodes), "--output_node=%s" % ",".join(output_nodes), "--input_shape=%s" % ":".join(input_shapes), "--output_shape=%s" % ":".join(output_shapes), "--input_data_format=%s" % ",".join(input_data_formats), "--output_data_format=%s" % ",".join(output_data_formats), "--validation_threshold=%f" % validation_threshold, "--input_data_type=%s" % ",".join(input_data_types), "--backend=%s" % backend, "--validation_outputs_data=%s" % ",".join(validation_outputs_data), "--log_file=%s" % log_file, _fg=True) elif platform == "megengine": validate(platform, model_file_path, "", "%s/%s" % (model_output_dir, input_file_name), "%s/%s" % (model_output_dir, output_file_name), device_type, ":".join(input_shapes), ":".join(output_shapes), ",".join(input_data_formats), ",".join(output_data_formats), ",".join(input_nodes), ",".join(output_nodes), validation_threshold, ",".join(input_data_types), backend, validation_outputs_data, log_file) six.print_("Validation done!\n")
def gen_input(model_output_dir, input_nodes, input_shapes, input_files=None, input_ranges=None, input_data_types=None, input_data_map=None, input_file_name="model_input"): for input_name in input_nodes: formatted_name = common.formatted_file_name(input_file_name, input_name) if os.path.exists("%s/%s" % (model_output_dir, formatted_name)): sh.rm("%s/%s" % (model_output_dir, formatted_name)) input_file_list = [] if isinstance(input_files, list): input_file_list.extend(input_files) else: input_file_list.append(input_files) if input_data_map: for i in range(len(input_nodes)): dst_input_file = model_output_dir + '/' + \ common.formatted_file_name(input_file_name, input_nodes[i]) input_name = input_nodes[i] common.mace_check( input_name in input_data_map, common.ModuleName.RUN, "The preprocessor API in PrecisionValidator" " script should return all inputs of model") if input_data_types[i] == 'float32': input_data = np.array(input_data_map[input_name], dtype=np.float32) elif input_data_types[i] == 'int32': input_data = np.array(input_data_map[input_name], dtype=np.int32) else: common.mace_check( False, common.ModuleName.RUN, 'Do not support input data type %s' % input_data_types[i]) common.mace_check( list(map(int, common.split_shape(input_shapes[i]))) == list( input_data.shape), common.ModuleName.RUN, "The shape return from preprocessor API of" " PrecisionValidator script is not same with" " model deployment file. %s vs %s" % (str(input_shapes[i]), str(input_data.shape))) input_data.tofile(dst_input_file) elif len(input_file_list) != 0: input_name_list = [] if isinstance(input_nodes, list): input_name_list.extend(input_nodes) else: input_name_list.append(input_nodes) common.mace_check( len(input_file_list) == len(input_name_list), common.ModuleName.RUN, 'If input_files set, the input files should ' 'match the input names.') for i in range(len(input_file_list)): if input_file_list[i] is not None: dst_input_file = model_output_dir + '/' + \ common.formatted_file_name(input_file_name, input_name_list[i]) if input_file_list[i].startswith("http://") or \ input_file_list[i].startswith("https://"): six.moves.urllib.request.urlretrieve( input_file_list[i], dst_input_file) else: sh.cp("-f", input_file_list[i], dst_input_file) else: # generate random input files input_nodes_str = ",".join(input_nodes) input_shapes_str = ":".join(input_shapes) input_ranges_str = ":".join(input_ranges) input_data_types_str = ",".join(input_data_types) generate_input_data("%s/%s" % (model_output_dir, input_file_name), input_nodes_str, input_shapes_str, input_ranges_str, input_data_types_str)
def benchmark_model(abi, serialno, benchmark_binary_dir, vlog_level, embed_model_data, model_output_dir, mace_model_dir, input_nodes, output_nodes, input_shapes, output_shapes, model_tag, device_type, phone_data_dir, build_type, opencl_binary_file, shared_library_dir, omp_num_threads=-1, cpu_affinity_policy=1, gpu_perf_hint=3, gpu_priority_hint=3, input_file_name="model_input", linkshared=0): print("* Benchmark for %s" % model_tag) if linkshared == 0: benchmark_model_target = "benchmark_model_static" else: benchmark_model_target = "benchmark_model_shared" mace_model_path = "" if build_type == BuildType.proto: mace_model_path = "%s/%s.pb" % (mace_model_dir, model_tag) if abi == "host": p = subprocess.Popen([ "env", "MACE_CPP_MIN_VLOG_LEVEL=%s" % vlog_level, "%s/%s" % (benchmark_binary_dir, benchmark_model_target), "--model_name=%s" % model_tag, "--input_node=%s" % ",".join(input_nodes), "--output_node=%s" % ",".join(output_nodes), "--input_shape=%s" % ":".join(input_shapes), "--output_shape=%s" % ":".join(output_shapes), "--input_file=%s/%s" % (model_output_dir, input_file_name), "--model_data_file=%s/%s.data" % (mace_model_dir, model_tag), "--device=%s" % device_type, "--omp_num_threads=%s" % omp_num_threads, "--cpu_affinity_policy=%s" % cpu_affinity_policy, "--gpu_perf_hint=%s" % gpu_perf_hint, "--gpu_priority_hint=%s" % gpu_priority_hint, "--model_file=%s" % mace_model_path, ]) p.wait() else: sh.adb("-s", serialno, "shell", "mkdir", "-p", phone_data_dir) internal_storage_dir = create_internal_storage_dir( serialno, phone_data_dir) for input_name in input_nodes: formatted_name = common.formatted_file_name( input_file_name, input_name) adb_push("%s/%s" % (model_output_dir, formatted_name), phone_data_dir, serialno) if not embed_model_data: adb_push("%s/%s.data" % (mace_model_dir, model_tag), phone_data_dir, serialno) if device_type == common.DeviceType.GPU \ and os.path.exists(opencl_binary_file): adb_push(opencl_binary_file, phone_data_dir, serialno) mace_model_phone_path = "" if build_type == BuildType.proto: mace_model_phone_path = "%s/%s.pb" % (phone_data_dir, model_tag) adb_push(mace_model_path, mace_model_phone_path, serialno) if linkshared == 1: adb_push("%s/libmace.so" % shared_library_dir, phone_data_dir, serialno) adb_push("%s/libgnustl_shared.so" % shared_library_dir, phone_data_dir, serialno) adb_push("%s/%s" % (benchmark_binary_dir, benchmark_model_target), phone_data_dir, serialno) adb_cmd = [ "LD_LIBRARY_PATH=%s" % phone_data_dir, "MACE_CPP_MIN_VLOG_LEVEL=%s" % vlog_level, "MACE_RUN_PARAMETER_PATH=%s/mace_run.config" % phone_data_dir, "MACE_INTERNAL_STORAGE_PATH=%s" % internal_storage_dir, "MACE_OPENCL_PROFILING=1", "%s/%s" % (phone_data_dir, benchmark_model_target), "--model_name=%s" % model_tag, "--input_node=%s" % ",".join(input_nodes), "--output_node=%s" % ",".join(output_nodes), "--input_shape=%s" % ":".join(input_shapes), "--output_shape=%s" % ":".join(output_shapes), "--input_file=%s/%s" % (phone_data_dir, input_file_name), "--model_data_file=%s/%s.data" % (phone_data_dir, model_tag), "--device=%s" % device_type, "--omp_num_threads=%s" % omp_num_threads, "--cpu_affinity_policy=%s" % cpu_affinity_policy, "--gpu_perf_hint=%s" % gpu_perf_hint, "--gpu_priority_hint=%s" % gpu_priority_hint, "--model_file=%s" % mace_model_phone_path, "--opencl_binary_file=%s/%s" % (phone_data_dir, os.path.basename(opencl_binary_file)), ] adb_cmd = ' '.join(adb_cmd) adb_cmd_file = "%s/%s" % (phone_data_dir, 'cmd_file') with open('/tmp/mace_cmd_file', 'w') as cmd_file: cmd_file.write(adb_cmd) adb_push('/tmp/mace_cmd_file', adb_cmd_file, serialno) sh.adb("-s", serialno, "shell", "sh", adb_cmd_file, _fg=True) print("Benchmark done!\n")
def benchmark_model(abi, serialno, benchmark_binary_dir, benchmark_binary_name, vlog_level, embed_model_data, model_output_dir, mace_model_dir, input_nodes, output_nodes, input_shapes, output_shapes, model_tag, device_type, phone_data_dir, model_graph_format, opencl_binary_file, opencl_parameter_file, libmace_dynamic_library_path, omp_num_threads=-1, cpu_affinity_policy=1, gpu_perf_hint=3, gpu_priority_hint=3, input_file_name="model_input", link_dynamic=False): print("* Benchmark for %s" % model_tag) mace_model_path = "" if model_graph_format == ModelFormat.file: mace_model_path = "%s/%s.pb" % (mace_model_dir, model_tag) if abi == "host": p = subprocess.Popen( [ "env", "MACE_CPP_MIN_VLOG_LEVEL=%s" % vlog_level, "%s/%s" % (benchmark_binary_dir, benchmark_binary_name), "--model_name=%s" % model_tag, "--input_node=%s" % ",".join(input_nodes), "--output_node=%s" % ",".join(output_nodes), "--input_shape=%s" % ":".join(input_shapes), "--output_shape=%s" % ":".join(output_shapes), "--input_file=%s/%s" % (model_output_dir, input_file_name), "--model_data_file=%s/%s.data" % (mace_model_dir, model_tag), "--device=%s" % device_type, "--omp_num_threads=%s" % omp_num_threads, "--cpu_affinity_policy=%s" % cpu_affinity_policy, "--gpu_perf_hint=%s" % gpu_perf_hint, "--gpu_priority_hint=%s" % gpu_priority_hint, "--model_file=%s" % mace_model_path, ]) p.wait() else: sh.adb("-s", serialno, "shell", "mkdir", "-p", phone_data_dir) internal_storage_dir = create_internal_storage_dir( serialno, phone_data_dir) for input_name in input_nodes: formatted_name = common.formatted_file_name(input_file_name, input_name) adb_push("%s/%s" % (model_output_dir, formatted_name), phone_data_dir, serialno) if not embed_model_data: adb_push("%s/%s.data" % (mace_model_dir, model_tag), phone_data_dir, serialno) if device_type == common.DeviceType.GPU: if os.path.exists(opencl_binary_file): adb_push(opencl_binary_file, phone_data_dir, serialno) if os.path.exists(opencl_parameter_file): adb_push(opencl_parameter_file, phone_data_dir, serialno) mace_model_phone_path = "" if model_graph_format == ModelFormat.file: mace_model_phone_path = "%s/%s.pb" % (phone_data_dir, model_tag) adb_push(mace_model_path, mace_model_phone_path, serialno) if link_dynamic: adb_push(libmace_dynamic_library_path, phone_data_dir, serialno) adb_push("%s/%s" % (benchmark_binary_dir, benchmark_binary_name), phone_data_dir, serialno) adb_cmd = [ "LD_LIBRARY_PATH=%s" % phone_data_dir, "MACE_CPP_MIN_VLOG_LEVEL=%s" % vlog_level, "MACE_RUN_PARAMETER_PATH=%s/mace_run.config" % phone_data_dir, "MACE_INTERNAL_STORAGE_PATH=%s" % internal_storage_dir, "MACE_OPENCL_PROFILING=1", "%s/%s" % (phone_data_dir, benchmark_binary_name), "--model_name=%s" % model_tag, "--input_node=%s" % ",".join(input_nodes), "--output_node=%s" % ",".join(output_nodes), "--input_shape=%s" % ":".join(input_shapes), "--output_shape=%s" % ":".join(output_shapes), "--input_file=%s/%s" % (phone_data_dir, input_file_name), "--model_data_file=%s/%s.data" % (phone_data_dir, model_tag), "--device=%s" % device_type, "--omp_num_threads=%s" % omp_num_threads, "--cpu_affinity_policy=%s" % cpu_affinity_policy, "--gpu_perf_hint=%s" % gpu_perf_hint, "--gpu_priority_hint=%s" % gpu_priority_hint, "--model_file=%s" % mace_model_phone_path, "--opencl_binary_file=%s/%s" % (phone_data_dir, os.path.basename(opencl_binary_file)), "--opencl_parameter_file=%s/%s" % (phone_data_dir, os.path.basename(opencl_parameter_file)), ] adb_cmd = ' '.join(adb_cmd) cmd_file_name = "%s-%s-%s" % ('cmd_file', model_tag, str(time.time())) adb_cmd_file = "%s/%s" % (phone_data_dir, cmd_file_name) tmp_cmd_file = "%s/%s" % ('/tmp', cmd_file_name) with open(tmp_cmd_file, 'w') as cmd_file: cmd_file.write(adb_cmd) adb_push(tmp_cmd_file, adb_cmd_file, serialno) os.remove(tmp_cmd_file) sh.adb( "-s", serialno, "shell", "sh", adb_cmd_file, _fg=True) sh.adb( "-s", serialno, "shell", "rm", adb_cmd_file, _fg=True) print("Benchmark done!\n")
def validate_model(abi, serialno, model_file_path, weight_file_path, platform, device_type, input_nodes, output_nodes, input_shapes, output_shapes, model_output_dir, phone_data_dir, caffe_env, input_file_name="model_input", output_file_name="model_out", validation_threshold=0.9): print("* Validate with %s" % platform) if abi != "host": for output_name in output_nodes: formatted_name = common.formatted_file_name( output_file_name, output_name) if os.path.exists("%s/%s" % (model_output_dir, formatted_name)): sh.rm("-rf", "%s/%s" % (model_output_dir, formatted_name)) adb_pull("%s/%s" % (phone_data_dir, formatted_name), model_output_dir, serialno) if platform == "tensorflow": validate(platform, model_file_path, "", "%s/%s" % (model_output_dir, input_file_name), "%s/%s" % (model_output_dir, output_file_name), device_type, ":".join(input_shapes), ":".join(output_shapes), ",".join(input_nodes), ",".join(output_nodes), validation_threshold) elif platform == "caffe": image_name = "mace-caffe:latest" container_name = "mace_caffe_validator" if caffe_env == common.CaffeEnvType.LOCAL: import imp try: imp.find_module('caffe') except ImportError: logger.error('There is no caffe python module.') validate(platform, model_file_path, weight_file_path, "%s/%s" % (model_output_dir, input_file_name), "%s/%s" % (model_output_dir, output_file_name), device_type, ":".join(input_shapes), ":".join(output_shapes), ",".join(input_nodes), ",".join(output_nodes), validation_threshold) elif caffe_env == common.CaffeEnvType.DOCKER: docker_image_id = sh.docker("images", "-q", image_name) if not docker_image_id: print("Build caffe docker") sh.docker("build", "-t", image_name, "third_party/caffe") container_id = sh.docker("ps", "-qa", "-f", "name=%s" % container_name) if container_id and not sh.docker("ps", "-qa", "--filter", "status=running", "-f", "name=%s" % container_name): sh.docker("rm", "-f", container_name) container_id = "" if not container_id: print("Run caffe container") sh.docker( "run", "-d", "-it", "--name", container_name, image_name, "/bin/bash") for input_name in input_nodes: formatted_input_name = common.formatted_file_name( input_file_name, input_name) sh.docker( "cp", "%s/%s" % (model_output_dir, formatted_input_name), "%s:/mace" % container_name) for output_name in output_nodes: formatted_output_name = common.formatted_file_name( output_file_name, output_name) sh.docker( "cp", "%s/%s" % (model_output_dir, formatted_output_name), "%s:/mace" % container_name) model_file_name = os.path.basename(model_file_path) weight_file_name = os.path.basename(weight_file_path) sh.docker("cp", "tools/common.py", "%s:/mace" % container_name) sh.docker("cp", "tools/validate.py", "%s:/mace" % container_name) sh.docker("cp", model_file_path, "%s:/mace" % container_name) sh.docker("cp", weight_file_path, "%s:/mace" % container_name) sh.docker( "exec", container_name, "python", "-u", "/mace/validate.py", "--platform=caffe", "--model_file=/mace/%s" % model_file_name, "--weight_file=/mace/%s" % weight_file_name, "--input_file=/mace/%s" % input_file_name, "--mace_out_file=/mace/%s" % output_file_name, "--device_type=%s" % device_type, "--input_node=%s" % ",".join(input_nodes), "--output_node=%s" % ",".join(output_nodes), "--input_shape=%s" % ":".join(input_shapes), "--output_shape=%s" % ":".join(output_shapes), "--validation_threshold=%f" % validation_threshold, _fg=True) print("Validation done!\n")
def tuning_run(abi, serialno, target_dir, target_name, vlog_level, embed_model_data, model_output_dir, input_nodes, output_nodes, input_shapes, output_shapes, mace_model_dir, model_tag, device_type, running_round, restart_round, limit_opencl_kernel_time, tuning, out_of_range_check, phone_data_dir, model_graph_format, opencl_binary_file, opencl_parameter_file, libmace_dynamic_library_path, omp_num_threads=-1, cpu_affinity_policy=1, gpu_perf_hint=3, gpu_priority_hint=3, input_file_name="model_input", output_file_name="model_out", runtime_failure_ratio=0.0, address_sanitizer=False, link_dynamic=False): print("* Run '%s' with round=%s, restart_round=%s, tuning=%s, " "out_of_range_check=%s, omp_num_threads=%s, cpu_affinity_policy=%s, " "gpu_perf_hint=%s, gpu_priority_hint=%s" % (model_tag, running_round, restart_round, str(tuning), str(out_of_range_check), omp_num_threads, cpu_affinity_policy, gpu_perf_hint, gpu_priority_hint)) mace_model_path = "" if model_graph_format == ModelFormat.file: mace_model_path = "%s/%s.pb" % (mace_model_dir, model_tag) if abi == "host": libmace_dynamic_lib_path = \ os.path.dirname(libmace_dynamic_library_path) p = subprocess.Popen( [ "env", "LD_LIBRARY_PATH=%s" % libmace_dynamic_lib_path, "MACE_CPP_MIN_VLOG_LEVEL=%s" % vlog_level, "MACE_RUNTIME_FAILURE_RATIO=%f" % runtime_failure_ratio, "%s/%s" % (target_dir, target_name), "--model_name=%s" % model_tag, "--input_node=%s" % ",".join(input_nodes), "--output_node=%s" % ",".join(output_nodes), "--input_shape=%s" % ":".join(input_shapes), "--output_shape=%s" % ":".join(output_shapes), "--input_file=%s/%s" % (model_output_dir, input_file_name), "--output_file=%s/%s" % (model_output_dir, output_file_name), "--model_data_file=%s/%s.data" % (mace_model_dir, model_tag), "--device=%s" % device_type, "--round=%s" % running_round, "--restart_round=%s" % restart_round, "--omp_num_threads=%s" % omp_num_threads, "--cpu_affinity_policy=%s" % cpu_affinity_policy, "--gpu_perf_hint=%s" % gpu_perf_hint, "--gpu_priority_hint=%s" % gpu_priority_hint, "--model_file=%s" % mace_model_path, ], stderr=subprocess.PIPE, stdout=subprocess.PIPE) out, err = p.communicate() stdout = err + out print stdout print("Running finished!\n") else: sh.adb("-s", serialno, "shell", "mkdir", "-p", phone_data_dir) internal_storage_dir = create_internal_storage_dir( serialno, phone_data_dir) for input_name in input_nodes: formatted_name = common.formatted_file_name(input_file_name, input_name) adb_push("%s/%s" % (model_output_dir, formatted_name), phone_data_dir, serialno) if address_sanitizer: adb_push(find_asan_rt_library(abi), phone_data_dir, serialno) if not embed_model_data: adb_push("%s/%s.data" % (mace_model_dir, model_tag), phone_data_dir, serialno) if device_type == common.DeviceType.GPU: if os.path.exists(opencl_binary_file): adb_push(opencl_binary_file, phone_data_dir, serialno) if os.path.exists(opencl_parameter_file): adb_push(opencl_parameter_file, phone_data_dir, serialno) adb_push("third_party/nnlib/libhexagon_controller.so", phone_data_dir, serialno) mace_model_phone_path = "" if model_graph_format == ModelFormat.file: mace_model_phone_path = "%s/%s.pb" % (phone_data_dir, model_tag) adb_push(mace_model_path, mace_model_phone_path, serialno) if link_dynamic: adb_push(libmace_dynamic_library_path, phone_data_dir, serialno) adb_push("%s/%s" % (target_dir, target_name), phone_data_dir, serialno) stdout_buff = [] process_output = make_output_processor(stdout_buff) adb_cmd = [ "LD_LIBRARY_PATH=%s" % phone_data_dir, "MACE_TUNING=%s" % int(tuning), "MACE_OUT_OF_RANGE_CHECK=%s" % int(out_of_range_check), "MACE_CPP_MIN_VLOG_LEVEL=%s" % vlog_level, "MACE_RUN_PARAMETER_PATH=%s/mace_run.config" % phone_data_dir, "MACE_INTERNAL_STORAGE_PATH=%s" % internal_storage_dir, "MACE_LIMIT_OPENCL_KERNEL_TIME=%s" % limit_opencl_kernel_time, "MACE_RUNTIME_FAILURE_RATIO=%f" % runtime_failure_ratio, ] if address_sanitizer: adb_cmd.extend([ "LD_PRELOAD=%s/%s" % (phone_data_dir, asan_rt_library_names(abi)) ]) adb_cmd.extend([ "%s/%s" % (phone_data_dir, target_name), "--model_name=%s" % model_tag, "--input_node=%s" % ",".join(input_nodes), "--output_node=%s" % ",".join(output_nodes), "--input_shape=%s" % ":".join(input_shapes), "--output_shape=%s" % ":".join(output_shapes), "--input_file=%s/%s" % (phone_data_dir, input_file_name), "--output_file=%s/%s" % (phone_data_dir, output_file_name), "--model_data_file=%s/%s.data" % (phone_data_dir, model_tag), "--device=%s" % device_type, "--round=%s" % running_round, "--restart_round=%s" % restart_round, "--omp_num_threads=%s" % omp_num_threads, "--cpu_affinity_policy=%s" % cpu_affinity_policy, "--gpu_perf_hint=%s" % gpu_perf_hint, "--gpu_priority_hint=%s" % gpu_priority_hint, "--model_file=%s" % mace_model_phone_path, "--opencl_binary_file=%s/%s" % (phone_data_dir, os.path.basename(opencl_binary_file)), "--opencl_parameter_file=%s/%s" % (phone_data_dir, os.path.basename(opencl_parameter_file)), ]) adb_cmd = ' '.join(adb_cmd) cmd_file_name = "%s-%s-%s" % ('cmd_file', model_tag, str(time.time())) adb_cmd_file = "%s/%s" % (phone_data_dir, cmd_file_name) tmp_cmd_file = "%s/%s" % ('/tmp', cmd_file_name) with open(tmp_cmd_file, 'w') as cmd_file: cmd_file.write(adb_cmd) adb_push(tmp_cmd_file, adb_cmd_file, serialno) os.remove(tmp_cmd_file) sh.adb( "-s", serialno, "shell", "sh", adb_cmd_file, _tty_in=True, _out=process_output, _err_to_out=True) stdout = "".join(stdout_buff) if not stdout_success(stdout): common.MaceLogger.error("Mace Run", "Mace run failed.") sh.adb( "-s", serialno, "shell", "rm", adb_cmd_file, _fg=True) print("Running finished!\n") return stdout
def benchmark_model(abi, serialno, benchmark_binary_dir, vlog_level, embed_model_data, model_output_dir, mace_model_dir, input_nodes, output_nodes, input_shapes, output_shapes, model_tag, device_type, phone_data_dir, build_type, opencl_binary_file, shared_library_dir, omp_num_threads=-1, cpu_affinity_policy=1, gpu_perf_hint=3, gpu_priority_hint=3, input_file_name="model_input", linkshared=0): print("* Benchmark for %s" % model_tag) if linkshared == 0: benchmark_model_target = "benchmark_model_static" else: benchmark_model_target = "benchmark_model_shared" mace_model_path = "" if build_type == BuildType.proto: mace_model_path = "%s/%s.pb" % (mace_model_dir, model_tag) if abi == "host": p = subprocess.Popen( [ "env", "MACE_CPP_MIN_VLOG_LEVEL=%s" % vlog_level, "%s/%s" % (benchmark_binary_dir, benchmark_model_target), "--model_name=%s" % model_tag, "--input_node=%s" % ",".join(input_nodes), "--output_node=%s" % ",".join(output_nodes), "--input_shape=%s" % ":".join(input_shapes), "--output_shape=%s" % ":".join(output_shapes), "--input_file=%s/%s" % (model_output_dir, input_file_name), "--model_data_file=%s/%s.data" % (mace_model_dir, model_tag), "--device=%s" % device_type, "--omp_num_threads=%s" % omp_num_threads, "--cpu_affinity_policy=%s" % cpu_affinity_policy, "--gpu_perf_hint=%s" % gpu_perf_hint, "--gpu_priority_hint=%s" % gpu_priority_hint, "--model_file=%s" % mace_model_path, ]) p.wait() else: sh.adb("-s", serialno, "shell", "mkdir", "-p", phone_data_dir) internal_storage_dir = create_internal_storage_dir( serialno, phone_data_dir) for input_name in input_nodes: formatted_name = common.formatted_file_name(input_file_name, input_name) adb_push("%s/%s" % (model_output_dir, formatted_name), phone_data_dir, serialno) if not embed_model_data: adb_push("%s/%s.data" % (mace_model_dir, model_tag), phone_data_dir, serialno) if device_type == common.DeviceType.GPU \ and os.path.exists(opencl_binary_file): adb_push(opencl_binary_file, phone_data_dir, serialno) mace_model_phone_path = "" if build_type == BuildType.proto: mace_model_phone_path = "%s/%s.pb" % (phone_data_dir, model_tag) adb_push(mace_model_path, mace_model_phone_path, serialno) if linkshared == 1: adb_push("%s/libmace.so" % shared_library_dir, phone_data_dir, serialno) adb_push("%s/libgnustl_shared.so" % shared_library_dir, phone_data_dir, serialno) adb_push("%s/%s" % (benchmark_binary_dir, benchmark_model_target), phone_data_dir, serialno) adb_cmd = [ "LD_LIBRARY_PATH=%s" % phone_data_dir, "MACE_CPP_MIN_VLOG_LEVEL=%s" % vlog_level, "MACE_RUN_PARAMETER_PATH=%s/mace_run.config" % phone_data_dir, "MACE_INTERNAL_STORAGE_PATH=%s" % internal_storage_dir, "MACE_OPENCL_PROFILING=1", "%s/%s" % (phone_data_dir, benchmark_model_target), "--model_name=%s" % model_tag, "--input_node=%s" % ",".join(input_nodes), "--output_node=%s" % ",".join(output_nodes), "--input_shape=%s" % ":".join(input_shapes), "--output_shape=%s" % ":".join(output_shapes), "--input_file=%s/%s" % (phone_data_dir, input_file_name), "--model_data_file=%s/%s.data" % (phone_data_dir, model_tag), "--device=%s" % device_type, "--omp_num_threads=%s" % omp_num_threads, "--cpu_affinity_policy=%s" % cpu_affinity_policy, "--gpu_perf_hint=%s" % gpu_perf_hint, "--gpu_priority_hint=%s" % gpu_priority_hint, "--model_file=%s" % mace_model_phone_path, "--opencl_binary_file=%s/%s" % (phone_data_dir, os.path.basename(opencl_binary_file)), ] adb_cmd = ' '.join(adb_cmd) cmd_file_name = "%s-%s-%s" % ('cmd_file', model_tag, str(time.time())) adb_cmd_file = "%s/%s" % (phone_data_dir, cmd_file_name) tmp_cmd_file = "%s/%s" % ('/tmp', cmd_file_name) with open(tmp_cmd_file, 'w') as cmd_file: cmd_file.write(adb_cmd) adb_push(tmp_cmd_file, adb_cmd_file, serialno) os.remove(tmp_cmd_file) sh.adb( "-s", serialno, "shell", "sh", adb_cmd_file, _fg=True) sh.adb( "-s", serialno, "shell", "rm", adb_cmd_file, _fg=True) print("Benchmark done!\n")
def tuning_run(abi, serialno, mace_run_dir, vlog_level, embed_model_data, model_output_dir, input_nodes, output_nodes, input_shapes, output_shapes, mace_model_dir, model_tag, device_type, running_round, restart_round, limit_opencl_kernel_time, tuning, out_of_range_check, phone_data_dir, build_type, opencl_binary_file, shared_library_dir, omp_num_threads=-1, cpu_affinity_policy=1, gpu_perf_hint=3, gpu_priority_hint=3, input_file_name="model_input", output_file_name="model_out", runtime_failure_ratio=0.0, address_sanitizer=False, linkshared=0): print("* Run '%s' with round=%s, restart_round=%s, tuning=%s, " "out_of_range_check=%s, omp_num_threads=%s, cpu_affinity_policy=%s, " "gpu_perf_hint=%s, gpu_priority_hint=%s" % (model_tag, running_round, restart_round, str(tuning), str(out_of_range_check), omp_num_threads, cpu_affinity_policy, gpu_perf_hint, gpu_priority_hint)) mace_model_path = "" if build_type == BuildType.proto: mace_model_path = "%s/%s.pb" % (mace_model_dir, model_tag) if linkshared == 0: mace_run_target = "mace_run_static" else: mace_run_target = "mace_run_shared" if abi == "host": p = subprocess.Popen( [ "env", "MACE_CPP_MIN_VLOG_LEVEL=%s" % vlog_level, "MACE_RUNTIME_FAILURE_RATIO=%f" % runtime_failure_ratio, "%s/%s" % (mace_run_dir, mace_run_target), "--model_name=%s" % model_tag, "--input_node=%s" % ",".join(input_nodes), "--output_node=%s" % ",".join(output_nodes), "--input_shape=%s" % ":".join(input_shapes), "--output_shape=%s" % ":".join(output_shapes), "--input_file=%s/%s" % (model_output_dir, input_file_name), "--output_file=%s/%s" % (model_output_dir, output_file_name), "--model_data_file=%s/%s.data" % (mace_model_dir, model_tag), "--device=%s" % device_type, "--round=%s" % running_round, "--restart_round=%s" % restart_round, "--omp_num_threads=%s" % omp_num_threads, "--cpu_affinity_policy=%s" % cpu_affinity_policy, "--gpu_perf_hint=%s" % gpu_perf_hint, "--gpu_priority_hint=%s" % gpu_priority_hint, "--model_file=%s" % mace_model_path, ], stderr=subprocess.PIPE, stdout=subprocess.PIPE) out, err = p.communicate() stdout = err + out print stdout print("Running finished!\n") return stdout else: sh.adb("-s", serialno, "shell", "mkdir", "-p", phone_data_dir) internal_storage_dir = create_internal_storage_dir( serialno, phone_data_dir) for input_name in input_nodes: formatted_name = common.formatted_file_name(input_file_name, input_name) adb_push("%s/%s" % (model_output_dir, formatted_name), phone_data_dir, serialno) if address_sanitizer: adb_push(find_asan_rt_library(abi), phone_data_dir, serialno) if not embed_model_data: adb_push("%s/%s.data" % (mace_model_dir, model_tag), phone_data_dir, serialno) if device_type == common.DeviceType.GPU\ and os.path.exists(opencl_binary_file): adb_push(opencl_binary_file, phone_data_dir, serialno) adb_push("third_party/nnlib/libhexagon_controller.so", phone_data_dir, serialno) mace_model_phone_path = "" if build_type == BuildType.proto: mace_model_phone_path = "%s/%s.pb" % (phone_data_dir, model_tag) adb_push(mace_model_path, mace_model_phone_path, serialno) if linkshared == 1: adb_push("%s/libmace.so" % shared_library_dir, phone_data_dir, serialno) adb_push("%s/libgnustl_shared.so" % shared_library_dir, phone_data_dir, serialno) adb_push("%s/%s" % (mace_run_dir, mace_run_target), phone_data_dir, serialno) stdout_buff = [] process_output = make_output_processor(stdout_buff) adb_cmd = [ "LD_LIBRARY_PATH=%s" % phone_data_dir, "MACE_TUNING=%s" % int(tuning), "MACE_OUT_OF_RANGE_CHECK=%s" % int(out_of_range_check), "MACE_CPP_MIN_VLOG_LEVEL=%s" % vlog_level, "MACE_RUN_PARAMETER_PATH=%s/mace_run.config" % phone_data_dir, "MACE_INTERNAL_STORAGE_PATH=%s" % internal_storage_dir, "MACE_LIMIT_OPENCL_KERNEL_TIME=%s" % limit_opencl_kernel_time, "MACE_RUNTIME_FAILURE_RATIO=%f" % runtime_failure_ratio, ] if address_sanitizer: adb_cmd.extend([ "LD_PRELOAD=%s/%s" % (phone_data_dir, asan_rt_library_names(abi)) ]) adb_cmd.extend([ "%s/%s" % (phone_data_dir, mace_run_target), "--model_name=%s" % model_tag, "--input_node=%s" % ",".join(input_nodes), "--output_node=%s" % ",".join(output_nodes), "--input_shape=%s" % ":".join(input_shapes), "--output_shape=%s" % ":".join(output_shapes), "--input_file=%s/%s" % (phone_data_dir, input_file_name), "--output_file=%s/%s" % (phone_data_dir, output_file_name), "--model_data_file=%s/%s.data" % (phone_data_dir, model_tag), "--device=%s" % device_type, "--round=%s" % running_round, "--restart_round=%s" % restart_round, "--omp_num_threads=%s" % omp_num_threads, "--cpu_affinity_policy=%s" % cpu_affinity_policy, "--gpu_perf_hint=%s" % gpu_perf_hint, "--gpu_priority_hint=%s" % gpu_priority_hint, "--model_file=%s" % mace_model_phone_path, "--opencl_binary_file=%s/%s" % (phone_data_dir, os.path.basename(opencl_binary_file)), ]) adb_cmd = ' '.join(adb_cmd) cmd_file_name = "%s-%s-%s" % ('cmd_file', model_tag, str(time.time())) adb_cmd_file = "%s/%s" % (phone_data_dir, cmd_file_name) tmp_cmd_file = "%s/%s" % ('/tmp', cmd_file_name) with open(tmp_cmd_file, 'w') as cmd_file: cmd_file.write(adb_cmd) adb_push(tmp_cmd_file, adb_cmd_file, serialno) os.remove(tmp_cmd_file) sh.adb( "-s", serialno, "shell", "sh", adb_cmd_file, _tty_in=True, _out=process_output, _err_to_out=True) stdout = "".join(stdout_buff) if not stdout_success(stdout): common.MaceLogger.error("Mace Run", "Mace run failed.") sh.adb( "-s", serialno, "shell", "rm", adb_cmd_file, _fg=True) print("Running finished!\n") return stdout
def validate_model(abi, serialno, model_file_path, weight_file_path, platform, device_type, input_nodes, output_nodes, input_shapes, output_shapes, model_output_dir, phone_data_dir, caffe_env, input_file_name="model_input", output_file_name="model_out"): print("* Validate with %s" % platform) if abi != "host": for output_name in output_nodes: formatted_name = common.formatted_file_name( output_file_name, output_name) if os.path.exists("%s/%s" % (model_output_dir, formatted_name)): sh.rm("-rf", "%s/%s" % (model_output_dir, formatted_name)) adb_pull("%s/%s" % (phone_data_dir, formatted_name), model_output_dir, serialno) if platform == "tensorflow": validate(platform, model_file_path, "", "%s/%s" % (model_output_dir, input_file_name), "%s/%s" % (model_output_dir, output_file_name), device_type, ":".join(input_shapes), ":".join(output_shapes), ",".join(input_nodes), ",".join(output_nodes)) elif platform == "caffe": image_name = "mace-caffe:latest" container_name = "mace_caffe_validator" if caffe_env == common.CaffeEnvType.LOCAL: import imp try: imp.find_module('caffe') except ImportError: logger.error('There is no caffe python module.') validate(platform, model_file_path, weight_file_path, "%s/%s" % (model_output_dir, input_file_name), "%s/%s" % (model_output_dir, output_file_name), device_type, ":".join(input_shapes), ":".join(output_shapes), ",".join(input_nodes), ",".join(output_nodes)) elif caffe_env == common.CaffeEnvType.DOCKER: docker_image_id = sh.docker("images", "-q", image_name) if not docker_image_id: print("Build caffe docker") sh.docker("build", "-t", image_name, "third_party/caffe") container_id = sh.docker("ps", "-qa", "-f", "name=%s" % container_name) if container_id and not sh.docker("ps", "-qa", "--filter", "status=running", "-f", "name=%s" % container_name): sh.docker("rm", "-f", container_name) container_id = "" if not container_id: print("Run caffe container") sh.docker( "run", "-d", "-it", "--name", container_name, image_name, "/bin/bash") for input_name in input_nodes: formatted_input_name = common.formatted_file_name( input_file_name, input_name) sh.docker( "cp", "%s/%s" % (model_output_dir, formatted_input_name), "%s:/mace" % container_name) for output_name in output_nodes: formatted_output_name = common.formatted_file_name( output_file_name, output_name) sh.docker( "cp", "%s/%s" % (model_output_dir, formatted_output_name), "%s:/mace" % container_name) model_file_name = os.path.basename(model_file_path) weight_file_name = os.path.basename(weight_file_path) sh.docker("cp", "tools/common.py", "%s:/mace" % container_name) sh.docker("cp", "tools/validate.py", "%s:/mace" % container_name) sh.docker("cp", model_file_path, "%s:/mace" % container_name) sh.docker("cp", weight_file_path, "%s:/mace" % container_name) sh.docker( "exec", container_name, "python", "-u", "/mace/validate.py", "--platform=caffe", "--model_file=/mace/%s" % model_file_name, "--weight_file=/mace/%s" % weight_file_name, "--input_file=/mace/%s" % input_file_name, "--mace_out_file=/mace/%s" % output_file_name, "--device_type=%s" % device_type, "--input_node=%s" % ",".join(input_nodes), "--output_node=%s" % ",".join(output_nodes), "--input_shape=%s" % ":".join(input_shapes), "--output_shape=%s" % ":".join(output_shapes), _fg=True) print("Validation done!\n")