def raw_to_proto(self, raw):
"""doc"""
ids = [
s if isinstance(s, int) else self.vocab.get(s, self.unk_id)
for s in self.tokenizer(raw)
]
fe = feature_pb2.Feature(int64_list=feature_pb2.Int64List(value=ids))
return fe
def raw_to_proto(self, raw):
"""doc"""
if self.vocab is None:
ids = [int(raw)]
else:
ids = [self.vocab[raw]]
fe = feature_pb2.Feature(int64_list=feature_pb2.Int64List(value=ids))
return fe
def raw_to_proto(self, raw):
"""doc"""
ids = [int(s) for s in raw.split(b' ')]
fe = feature_pb2.Feature(int64_list=feature_pb2.Int64List(value=ids))
return fe