def load_model(cls, path: str):
"""
Deserializes a VGSL model from a CoreML file.
Args:
path (str): CoreML file
Returns:
A TorchVGSLModel instance.
Raises:
KrakenInvalidModelException if the model data is invalid (not a
string, protobuf file, or without appropriate metadata).
FileNotFoundError if the path doesn't point to a file.
"""
try:
mlmodel = MLModel(path)
except TypeError as e:
raise KrakenInvalidModelException(str(e))
except DecodeError as e:
raise KrakenInvalidModelException('Failure parsing model protobuf: {}'.format(str(e)))
if 'vgsl' not in mlmodel.user_defined_metadata:
raise KrakenInvalidModelException('No VGSL spec in model metadata')
vgsl_spec = mlmodel.user_defined_metadata['vgsl']
nn = cls(vgsl_spec)
for name, layer in nn.nn.named_children():
layer.deserialize(name, mlmodel.get_spec())
if 'codec' in mlmodel.user_defined_metadata:
nn.add_codec(PytorchCodec(json.loads(mlmodel.user_defined_metadata['codec'])))
nn.user_metadata = {'accuracy': [], 'seg_type': 'bbox', 'one_channel_mode': '1', 'model_type': None, 'hyper_params': {}} # type: dict[str, str]
if 'kraken_meta' in mlmodel.user_defined_metadata:
nn.user_metadata.update(json.loads(mlmodel.user_defined_metadata['kraken_meta']))
return nn
def load_pronn_model(cls, path: str):
"""
Loads an pronn model to VGSL.
"""
with open(path, 'rb') as fp:
net = pyrnn_pb2.pyrnn()
try:
net.ParseFromString(fp.read())
except Exception:
raise KrakenInvalidModelException('File does not contain valid proto msg')
if not net.IsInitialized():
raise KrakenInvalidModelException('Model incomplete')
# extract codec
codec = PytorchCodec(net.codec)
input = net.ninput
hidden = net.fwdnet.wgi.dim[0]
# extract weights
weightnames = ('wgi', 'wgf', 'wci', 'wgo', 'wip', 'wfp', 'wop')
fwd_w = []
rev_w = []
for w in weightnames:
fwd_ar = getattr(net.fwdnet, w)
rev_ar = getattr(net.revnet, w)
fwd_w.append(torch.Tensor(fwd_ar.value).view(list(fwd_ar.dim)))
rev_w.append(torch.Tensor(rev_ar.value).view(list(rev_ar.dim)))
t = torch.cat(fwd_w[:4])
weight_ih_l0 = t[:, :input+1]
weight_hh_l0 = t[:, input+1:]
t = torch.cat(rev_w[:4])
weight_ih_l0_rev = t[:, :input+1]
weight_hh_l0_rev = t[:, input+1:]
weight_lin = torch.Tensor(net.softmax.w2.value).view(list(net.softmax.w2.dim))
# build vgsl spec and set weights
nn = cls('[1,1,0,{} Lbxo{} O1ca{}]'.format(input, hidden, len(net.codec)))
nn.nn.L_0.layer.weight_ih_l0 = torch.nn.Parameter(weight_ih_l0)
nn.nn.L_0.layer.weight_hh_l0 = torch.nn.Parameter(weight_hh_l0)
nn.nn.L_0.layer.weight_ih_l0_reverse = torch.nn.Parameter(weight_ih_l0_rev)
nn.nn.L_0.layer.weight_hh_l0_reverse = torch.nn.Parameter(weight_hh_l0_rev)
nn.nn.L_0.layer.weight_ip_l0 = torch.nn.Parameter(fwd_w[4])
nn.nn.L_0.layer.weight_fp_l0 = torch.nn.Parameter(fwd_w[5])
nn.nn.L_0.layer.weight_op_l0 = torch.nn.Parameter(fwd_w[6])
nn.nn.L_0.layer.weight_ip_l0_reverse = torch.nn.Parameter(rev_w[4])
nn.nn.L_0.layer.weight_fp_l0_reverse = torch.nn.Parameter(rev_w[5])
nn.nn.L_0.layer.weight_op_l0_reverse = torch.nn.Parameter(rev_w[6])
nn.nn.O_1.lin.weight = torch.nn.Parameter(weight_lin)
nn.add_codec(codec)
return nn
def load_pronn_model(cls, path: str):
"""
Loads an pronn model to VGSL.
"""
with open(path, 'rb') as fp:
net = pyrnn_pb2.pyrnn()
try:
net.ParseFromString(fp.read())
except Exception:
raise KrakenInvalidModelException('File does not contain valid proto msg')
if not net.IsInitialized():
raise KrakenInvalidModelException('Model incomplete')
# extract codec
codec = PytorchCodec(net.codec)
input = net.ninput
hidden = net.fwdnet.wgi.dim[0]
# extract weights
weightnames = ('wgi', 'wgf', 'wci', 'wgo', 'wip', 'wfp', 'wop')
fwd_w = []
rev_w = []
for w in weightnames:
fwd_ar = getattr(net.fwdnet, w)
rev_ar = getattr(net.revnet, w)
fwd_w.append(torch.Tensor(fwd_ar.value).view(list(fwd_ar.dim)))
rev_w.append(torch.Tensor(rev_ar.value).view(list(rev_ar.dim)))
t = torch.cat(fwd_w[:4])
weight_ih_l0 = t[:, :input+1]
weight_hh_l0 = t[:, input+1:]
t = torch.cat(rev_w[:4])
weight_ih_l0_rev = t[:, :input+1]
weight_hh_l0_rev = t[:, input+1:]
weight_lin = torch.Tensor(net.softmax.w2.value).view(list(net.softmax.w2.dim))
# build vgsl spec and set weights
nn = cls('[1,1,0,{} Lbxo{} O1ca{}]'.format(input, hidden, len(net.codec)))
nn.nn.L_0.layer.weight_ih_l0 = torch.nn.Parameter(weight_ih_l0)
nn.nn.L_0.layer.weight_hh_l0 = torch.nn.Parameter(weight_hh_l0)
nn.nn.L_0.layer.weight_ih_l0_reverse = torch.nn.Parameter(weight_ih_l0_rev)
nn.nn.L_0.layer.weight_hh_l0_reverse = torch.nn.Parameter(weight_hh_l0_rev)
nn.nn.L_0.layer.weight_ip_l0 = torch.nn.Parameter(fwd_w[4])
nn.nn.L_0.layer.weight_fp_l0 = torch.nn.Parameter(fwd_w[5])
nn.nn.L_0.layer.weight_op_l0 = torch.nn.Parameter(fwd_w[6])
nn.nn.L_0.layer.weight_ip_l0_reverse = torch.nn.Parameter(rev_w[4])
nn.nn.L_0.layer.weight_fp_l0_reverse = torch.nn.Parameter(rev_w[5])
nn.nn.L_0.layer.weight_op_l0_reverse = torch.nn.Parameter(rev_w[6])
nn.nn.O_1.lin.weight = torch.nn.Parameter(weight_lin)
nn.add_codec(codec)
return nn
def load_model(cls, path: Union[str, pathlib.Path]):
"""
Deserializes a VGSL model from a CoreML file.
Args:
path: CoreML file
Returns:
A TorchVGSLModel instance.
Raises:
KrakenInvalidModelException if the model data is invalid (not a
string, protobuf file, or without appropriate metadata).
FileNotFoundError if the path doesn't point to a file.
"""
if isinstance(path, pathlib.Path):
path = path.as_posix()
try:
mlmodel = MLModel(path)
except TypeError as e:
raise KrakenInvalidModelException(str(e))
except DecodeError as e:
raise KrakenInvalidModelException(
'Failure parsing model protobuf: {}'.format(str(e)))
if 'vgsl' not in mlmodel.user_defined_metadata:
raise KrakenInvalidModelException('No VGSL spec in model metadata')
vgsl_spec = mlmodel.user_defined_metadata['vgsl']
nn = cls(vgsl_spec)
def _deserialize_layers(name, layer):
logger.debug(f'Deserializing layer {name} with type {type(layer)}')
if type(layer) in (layers.MultiParamParallel,
layers.MultiParamSequential):
for name, l in layer.named_children():
_deserialize_layers(name, l)
else:
layer.deserialize(name, mlmodel.get_spec())
_deserialize_layers('', nn.nn)
if 'codec' in mlmodel.user_defined_metadata:
nn.add_codec(
PytorchCodec(json.loads(
mlmodel.user_defined_metadata['codec'])))
nn.user_metadata = {
'accuracy': [],
'seg_type': 'bbox',
'one_channel_mode': '1',
'model_type': None,
'hyper_params': {}
} # type: dict[str, str]
if 'kraken_meta' in mlmodel.user_defined_metadata:
nn.user_metadata.update(
json.loads(mlmodel.user_defined_metadata['kraken_meta']))
return nn
def load_model(cls, path: str):
"""
Deserializes a VGSL model from a CoreML file.
Args:
path (str): CoreML file
"""
mlmodel = MLModel(path)
if 'vgsl' not in mlmodel.user_defined_metadata:
raise ValueError('No VGSL spec in model metadata')
vgsl_spec = mlmodel.user_defined_metadata['vgsl']
nn = cls(vgsl_spec)
for name, layer in nn.nn.named_children():
layer.deserialize(name, mlmodel.get_spec())
if 'codec' in mlmodel.user_defined_metadata:
nn.add_codec(
PytorchCodec(json.loads(
mlmodel.user_defined_metadata['codec'])))
return nn
def load_model(cls, path: str):
"""
Deserializes a VGSL model from a CoreML file.
Args:
path (str): CoreML file
"""
mlmodel = MLModel(path)
if 'vgsl' not in mlmodel.user_defined_metadata:
raise ValueError('No VGSL spec in model metadata')
vgsl_spec = mlmodel.user_defined_metadata['vgsl']
nn = cls(vgsl_spec)
for name, layer in nn.nn.named_children():
layer.deserialize(name, mlmodel.get_spec())
if 'codec' in mlmodel.user_defined_metadata:
nn.add_codec(PytorchCodec(json.loads(mlmodel.user_defined_metadata['codec'])))
if 'kraken_meta' in mlmodel.user_defined_metadata:
nn.user_metadata = json.loads(mlmodel.user_defined_metadata['kraken_meta'])
return nn
def load_clstm_model(cls, path: str):
"""
Loads an CLSTM model to VGSL.
"""
net = clstm_pb2.NetworkProto()
with open(path, 'rb') as fp:
try:
net.ParseFromString(fp.read())
except Exception:
raise KrakenInvalidModelException('File does not contain valid proto msg')
if not net.IsInitialized():
raise KrakenInvalidModelException('Model incomplete')
input = net.ninput
attrib = {a.key: a.value for a in list(net.attribute)}
# mainline clstm model
if len(attrib) > 1:
mode = 'clstm'
else:
mode = 'clstm_compat'
# extract codec
codec = PytorchCodec([''] + [chr(x) for x in net.codec[1:]])
# separate layers
nets = {}
nets['softm'] = [n for n in list(net.sub) if n.kind == 'SoftmaxLayer'][0]
parallel = [n for n in list(net.sub) if n.kind == 'Parallel'][0]
nets['lstm1'] = [n for n in list(parallel.sub) if n.kind.startswith('NPLSTM')][0]
rev = [n for n in list(parallel.sub) if n.kind == 'Reversed'][0]
nets['lstm2'] = rev.sub[0]
hidden = int(nets['lstm1'].attribute[0].value)
weights = {} # type: Dict[str, torch.Tensor]
for n in nets:
weights[n] = {}
for w in list(nets[n].weights):
weights[n][w.name] = torch.Tensor(w.value).view(list(w.dim))
if mode == 'clstm_compat':
weightnames = ('.WGI', '.WGF', '.WCI', '.WGO')
weightname_softm = '.W'
else:
weightnames = ('WGI', 'WGF', 'WCI', 'WGO')
weightname_softm = 'W1'
# input hidden and hidden-hidden weights are in one matrix. also
# CLSTM/ocropy likes 1-augmenting every other tensor so the ih weights
# are input+1 in one dimension.
t = torch.cat(list(w for w in [weights['lstm1'][wn] for wn in weightnames]))
weight_ih_l0 = t[:, :input+1]
weight_hh_l0 = t[:, input+1:]
t = torch.cat(list(w for w in [weights['lstm2'][wn] for wn in weightnames]))
weight_ih_l0_rev = t[:, :input+1]
weight_hh_l0_rev = t[:, input+1:]
weight_lin = weights['softm'][weightname_softm]
if mode == 'clstm_compat':
weight_lin = torch.cat([torch.zeros(len(weight_lin), 1), weight_lin], 1)
# build vgsl spec and set weights
nn = cls('[1,1,0,{} Lbxc{} O1ca{}]'.format(input, hidden, len(net.codec)))
nn.nn.L_0.layer.weight_ih_l0 = torch.nn.Parameter(weight_ih_l0)
nn.nn.L_0.layer.weight_hh_l0 = torch.nn.Parameter(weight_hh_l0)
nn.nn.L_0.layer.weight_ih_l0_reverse = torch.nn.Parameter(weight_ih_l0_rev)
nn.nn.L_0.layer.weight_hh_l0_reverse = torch.nn.Parameter(weight_hh_l0_rev)
nn.nn.O_1.lin.weight = torch.nn.Parameter(weight_lin)
nn.add_codec(codec)
return nn
def load_pyrnn_model(cls, path: str):
"""
Loads an pyrnn model to VGSL.
"""
if not PY2:
raise KrakenInvalidModelException('Loading pickle models is not supported on python 3')
import cPickle
def find_global(mname, cname):
aliases = {
'lstm.lstm': kraken.lib.lstm,
'ocrolib.lstm': kraken.lib.lstm,
'ocrolib.lineest': kraken.lib.lineest,
}
if mname in aliases:
return getattr(aliases[mname], cname)
return getattr(sys.modules[mname], cname)
of = io.open
if path.endswith('.gz'):
of = gzip.open
with io.BufferedReader(of(path, 'rb')) as fp:
unpickler = cPickle.Unpickler(fp)
unpickler.find_global = find_global
try:
net = unpickler.load()
except Exception as e:
raise KrakenInvalidModelException(str(e))
if not isinstance(net, kraken.lib.lstm.SeqRecognizer):
raise KrakenInvalidModelException('Pickle is %s instead of '
'SeqRecognizer' %
type(net).__name__)
# extract codec
codec = PytorchCodec({k: [v] for k, v in net.codec.char2code.items()})
input = net.Ni
parallel, softmax = net.lstm.nets
fwdnet, revnet = parallel.nets
revnet = revnet.net
hidden = fwdnet.WGI.shape[0]
# extract weights
weightnames = ('WGI', 'WGF', 'WCI', 'WGO', 'WIP', 'WFP', 'WOP')
fwd_w = []
rev_w = []
for w in weightnames:
fwd_w.append(torch.Tensor(getattr(fwdnet, w)))
rev_w.append(torch.Tensor(getattr(revnet, w)))
t = torch.cat(fwd_w[:4])
weight_ih_l0 = t[:, :input+1]
weight_hh_l0 = t[:, input+1:]
t = torch.cat(rev_w[:4])
weight_ih_l0_rev = t[:, :input+1]
weight_hh_l0_rev = t[:, input+1:]
weight_lin = torch.Tensor(softmax.W2)
# build vgsl spec and set weights
nn = cls('[1,1,0,{} Lbxo{} O1ca{}]'.format(input, hidden, len(net.codec.code2char)))
nn.nn.L_0.layer.weight_ih_l0 = torch.nn.Parameter(weight_ih_l0)
nn.nn.L_0.layer.weight_hh_l0 = torch.nn.Parameter(weight_hh_l0)
nn.nn.L_0.layer.weight_ih_l0_reverse = torch.nn.Parameter(weight_ih_l0_rev)
nn.nn.L_0.layer.weight_hh_l0_reverse = torch.nn.Parameter(weight_hh_l0_rev)
nn.nn.L_0.layer.weight_ip_l0 = torch.nn.Parameter(fwd_w[4])
nn.nn.L_0.layer.weight_fp_l0 = torch.nn.Parameter(fwd_w[5])
nn.nn.L_0.layer.weight_op_l0 = torch.nn.Parameter(fwd_w[6])
nn.nn.L_0.layer.weight_ip_l0_reverse = torch.nn.Parameter(rev_w[4])
nn.nn.L_0.layer.weight_fp_l0_reverse = torch.nn.Parameter(rev_w[5])
nn.nn.L_0.layer.weight_op_l0_reverse = torch.nn.Parameter(rev_w[6])
nn.nn.O_1.lin.weight = torch.nn.Parameter(weight_lin)
nn.add_codec(codec)
return nn
def load_clstm_model(cls, path: str):
"""
Loads an CLSTM model to VGSL.
"""
net = clstm_pb2.NetworkProto()
with open(path, 'rb') as fp:
try:
net.ParseFromString(fp.read())
except Exception:
raise KrakenInvalidModelException('File does not contain valid proto msg')
if not net.IsInitialized():
raise KrakenInvalidModelException('Model incomplete')
input = net.ninput
attrib = {a.key: a.value for a in list(net.attribute)}
# mainline clstm model
if len(attrib) > 1:
mode = 'clstm'
else:
mode = 'clstm_compat'
# extract codec
codec = PytorchCodec([u''] + [chr(x) for x in net.codec[1:]])
# separate layers
nets = {}
nets['softm'] = [n for n in list(net.sub) if n.kind == 'SoftmaxLayer'][0]
parallel = [n for n in list(net.sub) if n.kind == 'Parallel'][0]
nets['lstm1'] = [n for n in list(parallel.sub) if n.kind.startswith('NPLSTM')][0]
rev = [n for n in list(parallel.sub) if n.kind == 'Reversed'][0]
nets['lstm2'] = rev.sub[0]
hidden = int(nets['lstm1'].attribute[0].value)
weights = {} # type: Dict[str, torch.Tensor]
for n in nets:
weights[n] = {}
for w in list(nets[n].weights):
weights[n][w.name] = torch.Tensor(w.value).view(list(w.dim))
if mode == 'clstm_compat':
weightnames = ('.WGI', '.WGF', '.WCI', '.WGO')
weightname_softm = '.W'
else:
weightnames = ('WGI', 'WGF', 'WCI', 'WGO')
weightname_softm = 'W1'
# input hidden and hidden-hidden weights are in one matrix. also
# CLSTM/ocropy likes 1-augmenting every other tensor so the ih weights
# are input+1 in one dimension.
t = torch.cat(list(w for w in [weights['lstm1'][wn] for wn in weightnames]))
weight_ih_l0 = t[:, :input+1]
weight_hh_l0 = t[:, input+1:]
t = torch.cat(list(w for w in [weights['lstm2'][wn] for wn in weightnames]))
weight_ih_l0_rev = t[:, :input+1]
weight_hh_l0_rev = t[:, input+1:]
weight_lin = weights['softm'][weightname_softm]
if mode == 'clstm_compat':
weight_lin = torch.cat([torch.zeros(len(weight_lin), 1), weight_lin], 1)
# build vgsl spec and set weights
nn = cls('[1,1,0,{} Lbxc{} O1ca{}]'.format(input, hidden, len(net.codec)))
nn.nn.L_0.layer.weight_ih_l0 = torch.nn.Parameter(weight_ih_l0)
nn.nn.L_0.layer.weight_hh_l0 = torch.nn.Parameter(weight_hh_l0)
nn.nn.L_0.layer.weight_ih_l0_reverse = torch.nn.Parameter(weight_ih_l0_rev)
nn.nn.L_0.layer.weight_hh_l0_reverse = torch.nn.Parameter(weight_hh_l0_rev)
nn.nn.O_1.lin.weight = torch.nn.Parameter(weight_lin)
nn.add_codec(codec)
return nn
def load_pyrnn_model(cls, path: str):
"""
Loads an pyrnn model to VGSL.
"""
if not PY2:
raise KrakenInvalidModelException('Loading pickle models is not supported on python 3')
import cPickle
def find_global(mname, cname):
aliases = {
'lstm.lstm': kraken.lib.lstm,
'ocrolib.lstm': kraken.lib.lstm,
'ocrolib.lineest': kraken.lib.lineest,
}
if mname in aliases:
return getattr(aliases[mname], cname)
return getattr(sys.modules[mname], cname)
of = io.open
if path.endswith(u'.gz'):
of = gzip.open
with io.BufferedReader(of(path, 'rb')) as fp:
unpickler = cPickle.Unpickler(fp)
unpickler.find_global = find_global
try:
net = unpickler.load()
except Exception as e:
raise KrakenInvalidModelException(str(e))
if not isinstance(net, kraken.lib.lstm.SeqRecognizer):
raise KrakenInvalidModelException('Pickle is %s instead of '
'SeqRecognizer' %
type(net).__name__)
# extract codec
codec = PytorchCodec({k: [v] for k, v in net.codec.char2code.items()})
input = net.Ni
parallel, softmax = net.lstm.nets
fwdnet, revnet = parallel.nets
revnet = revnet.net
hidden = fwdnet.WGI.shape[0]
# extract weights
weightnames = ('WGI', 'WGF', 'WCI', 'WGO', 'WIP', 'WFP', 'WOP')
fwd_w = []
rev_w = []
for w in weightnames:
fwd_w.append(torch.Tensor(getattr(fwdnet, w)))
rev_w.append(torch.Tensor(getattr(revnet, w)))
t = torch.cat(fwd_w[:4])
weight_ih_l0 = t[:, :input+1]
weight_hh_l0 = t[:, input+1:]
t = torch.cat(rev_w[:4])
weight_ih_l0_rev = t[:, :input+1]
weight_hh_l0_rev = t[:, input+1:]
weight_lin = torch.Tensor(softmax.W2)
# build vgsl spec and set weights
nn = cls('[1,1,0,{} Lbxo{} O1ca{}]'.format(input, hidden, len(net.codec.code2char)))
nn.nn.L_0.layer.weight_ih_l0 = torch.nn.Parameter(weight_ih_l0)
nn.nn.L_0.layer.weight_hh_l0 = torch.nn.Parameter(weight_hh_l0)
nn.nn.L_0.layer.weight_ih_l0_reverse = torch.nn.Parameter(weight_ih_l0_rev)
nn.nn.L_0.layer.weight_hh_l0_reverse = torch.nn.Parameter(weight_hh_l0_rev)
nn.nn.L_0.layer.weight_ip_l0 = torch.nn.Parameter(fwd_w[4])
nn.nn.L_0.layer.weight_fp_l0 = torch.nn.Parameter(fwd_w[5])
nn.nn.L_0.layer.weight_op_l0 = torch.nn.Parameter(fwd_w[6])
nn.nn.L_0.layer.weight_ip_l0_reverse = torch.nn.Parameter(rev_w[4])
nn.nn.L_0.layer.weight_fp_l0_reverse = torch.nn.Parameter(rev_w[5])
nn.nn.L_0.layer.weight_op_l0_reverse = torch.nn.Parameter(rev_w[6])
nn.nn.O_1.lin.weight = torch.nn.Parameter(weight_lin)
nn.add_codec(codec)
return nn
