def load_using_pb(self): self.caffemodel = get_caffe_resolver().NetParameter() self.caffemodel.MergeFromString(open(self.data_path, 'rb').read()) pair = lambda layer: (layer.name, self.normalize_pb_data(layer)) layers = self.caffemodel.layers or self.caffemodel.layer self.params = [pair(layer) for layer in layers if layer.blobs] self.did_use_pb = True
def compute_output_shapes(self, model): sorted_nodes = self.topologically_sorted() (tmp_handle, tmp_prototxt) = tempfile.mkstemp(suffix=".prototxt") with open(tmp_prototxt, 'w') as f: f.write(text_format.MessageToString(model)) self.prototxt = tmp_prototxt if has_pycaffe(): caffe = get_caffe_resolver().caffe net = caffe.Net(tmp_prototxt, caffe.TEST) for key, value in net.blobs.items(): try: node = self.get_node(key) dims = list(value.shape) dims = dims + [1] * (4 - len(dims)) node.output_shape = TensorShape(*dims) except: continue for node in sorted_nodes: if node.output_shape is None: node.output_shape = TensorShape( *NodeKind.compute_output_shape(node)) os.close(tmp_handle) else: for node in sorted_nodes: node.output_shape = TensorShape( *NodeKind.compute_output_shape(node))
def compute_output_shapes(self, model): sorted_nodes = self.topologically_sorted() (tmp_handle, tmp_prototxt) = tempfile.mkstemp(suffix=".prototxt") with open(tmp_prototxt, 'w') as f: f.write(text_format.MessageToString(model)) self.prototxt = tmp_prototxt if has_pycaffe(): caffe = get_caffe_resolver().caffe net = caffe.Net(tmp_prototxt, caffe.TEST) for key, value in net.blobs.items(): try: node = self.get_node(key) dims = list(value.shape) dims = dims + [1] * (4 - len(dims)) node.output_shape = TensorShape(*dims) except: continue for node in sorted_nodes: if node.output_shape is None: node.output_shape = TensorShape(*NodeKind.compute_output_shape(node)) os.close(tmp_handle) os.remove(tmp_prototxt) else: for node in sorted_nodes: node.output_shape = TensorShape(*NodeKind.compute_output_shape(node))
def __init__(self, def_path, data_path, target_toolkit, input_shape=None, phase='test'): self.layer_name_map = {} self.data_injector = None self.is_train_proto = False self.input_shape = input_shape if def_path is None: if self.input_shape is None: raise ConversionError('if the graph prototxt is not provided, the input shape should be provided') self.input_shape = [1] + self.input_shape def_path, self.data_injector = self.gen_prototxt_from_caffemodel(data_path, self.input_shape) self.is_train_proto = True else: model = get_caffe_resolver().NetParameter() with open(def_path, 'r') as f: text_format.Merge(f.read(), model) layers = model.layers or model.layer if len([layer for layer in layers if NodeKind.map_raw_kind(layer.type) in LAYER_IN_TRAIN_PROTO]) > 0: if self.input_shape is None: raise ConversionError('the train_val.prototxt should be provided with the input shape') self.input_shape = [1] + self.input_shape self.is_train_proto = True graph = GraphBuilder(def_path, self.input_shape, self.is_train_proto, phase).build() if self.is_train_proto: def_path = graph.prototxt if data_path is not None: graph = graph.transformed([ self.data_injector if self.data_injector else DataInjector(def_path, data_path), # Load and associate learned parameters BatchNormScaleBiasFuser(), BatchNormPreprocessor() # Pre-process batch normalization data ]) target_toolkit = target_toolkit.lower() if target_toolkit not in ('caffe', 'caffe2'): graph = graph.transformed([DataReshaper({ # Reshape the parameters to TensorFlow's ordering NodeKind.Convolution: (2, 3, 1, 0), # (c_o, c_i, h, w) -> (h, w, c_i, c_o) NodeKind.Deconvolution: (2, 3, 1, 0), # (c_o, c_i, h, w) -> (h, w, c_i, c_o) NodeKind.InnerProduct: (1, 0) # (c_o, c_i) -> (c_i, c_o) }), ParameterNamer() # Convert parameters to dictionaries ]) self.graph = graph # self.graph = NodeRenamer()(graph) print_stderr(self.graph)
def load(self): self.model = get_caffe_resolver().NetParameter() with open(self.model_path, 'r') as f: text_format.Merge(f.read(), self.model) if self.is_train_proto: self.process_train_proto()
def load_using_caffe(self): caffe = get_caffe_resolver().caffe net = caffe.Net(str(self.def_path), str(self.data_path), caffe.TEST) data = lambda blob: blob.data self.params = [(k, list(map(data, v))) for k, v in net.params.items()]
def __init__(self, def_path, data_path, target_toolkit, input_shape=None, phase='test'): self.layer_name_map = {} self.data_injector = None self.is_train_proto = False self.input_shape = input_shape if def_path is None: if self.input_shape is None: raise ConversionError( 'if the graph prototxt is not provided, the input shape should be provided' ) self.input_shape = [1] + self.input_shape def_path, self.data_injector = self.gen_prototxt_from_caffemodel( data_path, self.input_shape) self.is_train_proto = True else: model = get_caffe_resolver().NetParameter() with open(def_path, 'r') as f: text_format.Merge(f.read(), model) layers = model.layers or model.layer if len([ layer for layer in layers if NodeKind.map_raw_kind( layer.type) in LAYER_IN_TRAIN_PROTO ]) > 0: if self.input_shape is None: raise ConversionError( 'the train_val.prototxt should be provided with the input shape' ) self.input_shape = [1] + self.input_shape self.is_train_proto = True graph = GraphBuilder(def_path, self.input_shape, self.is_train_proto, phase).build() if self.is_train_proto: def_path = graph.prototxt if data_path is not None: graph = graph.transformed([ self.data_injector if self.data_injector else DataInjector( def_path, data_path), # Load and associate learned parameters BatchNormScaleBiasFuser(), BatchNormPreprocessor() # Pre-process batch normalization data ]) target_toolkit = target_toolkit.lower() if target_toolkit not in ('caffe', 'caffe2'): graph = graph.transformed([ DataReshaper( { # Reshape the parameters to TensorFlow's ordering NodeKind.Convolution: (2, 3, 1, 0), # (c_o, c_i, h, w) -> (h, w, c_i, c_o) NodeKind.Deconvolution: (2, 3, 1, 0), # (c_o, c_i, h, w) -> (h, w, c_i, c_o) NodeKind.InnerProduct: (1, 0) # (c_o, c_i) -> (c_i, c_o) }), ParameterNamer() # Convert parameters to dictionaries ]) self.graph = graph # self.graph = NodeRenamer()(graph) print(self.graph)