def test_float_repr(self):
res = t.Float()
self.assertEqual(repr(res), '<Float>')
res = t.Float(gte=1)
self.assertEqual(repr(res), '<Float(gte=1)>')
res = t.Float(lte=10)
self.assertEqual(repr(res), '<Float(lte=10)>')
res = t.Float(gte=1, lte=10)
self.assertEqual(repr(res), '<Float(gte=1, lte=10)>')
def test_float(self):
res = t.Float().check(1.0)
self.assertEqual(res, 1.0)
res = extract_error(t.Float(), 1 + 3j)
self.assertEqual(res, 'value is not float')
res = extract_error(t.Float(), 1)
self.assertEqual(res, 1.0)
res = t.Float(gte=2).check(3.0)
self.assertEqual(res, 3.0)
res = extract_error(t.Float(gte=2), 1.0)
self.assertEqual(res, 'value is less than 2')
res = t.Float(lte=10).check(5.0)
self.assertEqual(res, 5.0)
res = extract_error(t.Float(lte=3), 5.0)
self.assertEqual(res, 'value is greater than 3')
res = t.Float().check("5.0")
self.assertEqual(res, 5.0)
def build_trafaret(sa_type, **kwargs):
if isinstance(sa_type, sa.sql.sqltypes.Enum):
trafaret = t.Enum(*sa_type.enums, **kwargs)
# check for Text should be before String
elif isinstance(sa_type, sa.sql.sqltypes.Text):
trafaret = t.String(**kwargs)
elif isinstance(sa_type, sa.sql.sqltypes.String):
trafaret = t.String(max_length=sa_type.length, **kwargs)
elif isinstance(sa_type, sa.sql.sqltypes.Integer):
trafaret = t.Int(**kwargs)
elif isinstance(sa_type, sa.sql.sqltypes.Float):
trafaret = t.Float(**kwargs)
elif isinstance(sa_type, sa.sql.sqltypes.DateTime):
trafaret = DateTime(**kwargs) # RFC3339
elif isinstance(sa_type, sa.sql.sqltypes.Date):
trafaret = DateTime(**kwargs) # RFC3339
elif isinstance(sa_type, sa.sql.sqltypes.Boolean):
trafaret = t.StrBool(**kwargs)
# Add PG related JSON and ARRAY
elif isinstance(sa_type, postgresql.JSON):
trafaret = AnyDict | t.List(AnyDict)
# Add PG related JSON and ARRAY
elif isinstance(sa_type, postgresql.ARRAY):
item_trafaret = build_trafaret(sa_type.item_type)
trafaret = t.List(item_trafaret)
elif isinstance(sa_type, postgresql.INET):
# TODO: depend on "ipaddress" module?
trafaret = t.String(**kwargs)
else:
type_ = str(sa_type)
msg = 'Validator for type {} not implemented'.format(type_)
raise NotImplementedError(msg)
return trafaret
def construct(arg):
'''
Shortcut syntax to define trafarets.
- int, str, float and bool will return t.Int, t.String, t.Float and t.Bool
- one element list will return t.List
- tuple or list with several args will return t.Tuple
- dict will return t.Dict. If key has '?' at the and it will be optional and '?' will be removed
- any callable will be t.Call
- otherwise it will be returned as is
construct is recursive and will try construct all lists, tuples and dicts args
'''
if isinstance(arg, t.Trafaret):
return arg
elif isinstance(arg, tuple) or (isinstance(arg, list) and len(arg) > 1):
return t.Tuple(*(construct(a) for a in arg))
elif isinstance(arg, list):
# if len(arg) == 1
return t.List(construct(arg[0]))
elif isinstance(arg, dict):
return t.Dict({
construct_key(key): construct(value)
for key, value in arg.items()
})
elif isinstance(arg, str):
return t.Atom(arg)
elif isinstance(arg, type):
if arg is int:
return t.Int()
elif arg is float:
return t.Float()
elif arg is str:
return t.String()
elif arg is bool:
return t.Bool()
else:
return t.Type(arg)
elif callable(arg):
return t.Call(arg)
else:
return arg
def test_list_many_values(self):
contract = Contract(
[Template(t.Int()),
Template(t.String()),
Template(t.Float())])
result = contract([1, 'Test', 12.5])
self.assertEqual(result, [1, 'Test', 12.5])
with self.assertRaises(ValueError):
result = contract(['error', 'Test', 12.5])
with self.assertRaises(ValueError):
result = contract([1, 666, 12.5])
with self.assertRaises(ValueError):
result = contract([1, 'Test', 'error'])
with self.assertRaises(ValueError):
result = contract([1, 'Test'])
with self.assertRaises(ValueError):
result = contract([1, 'Test', 12.5, 'error'])
with self.assertRaises(ValueError):
result = contract(42)
def test_list_many_values_with_default(self):
contract = AsyncContract([
AsyncTemplate(t.Int(), 42),
AsyncTemplate(t.String(), 'null'),
AsyncTemplate(t.Float(), 1.5)
])
result = self.loop.run_until_complete(contract([1, 'Test', 12.5]))
self.assertEqual(result, [1, 'Test', 12.5])
result = self.loop.run_until_complete(contract(['error', 'Test',
12.5]))
self.assertEqual(result, [42, 'Test', 12.5])
result = self.loop.run_until_complete(contract([1, 666, 12.5]))
self.assertEqual(result, [1, 'null', 12.5])
result = self.loop.run_until_complete(contract([1, 'Test', 'error']))
self.assertEqual(result, [1, 'Test', 1.5])
with self.assertRaises(ValueError):
result = self.loop.run_until_complete(contract([1, 'Test']))
with self.assertRaises(ValueError):
result = self.loop.run_until_complete(
contract([1, 'Test', 12.5, 'error']))
with self.assertRaises(ValueError):
result = self.loop.run_until_complete(contract(42))
class TfObjectDetectionModel(object):
"""Implementation for TF Object Detection API Inference"""
# model's input tensor name from official website
# https://github.com/tensorflow/models/blob/master/research/object_detection/object_detection_tutorial.ipynb
input_tensors = {'images': "image_tensor:0"}
# model's output tensors names from official website
# https://github.com/tensorflow/models/blob/master/research/object_detection/object_detection_tutorial.ipynb
output_tensors = {
"labels": "detection_classes:0",
"boxes": "detection_boxes:0",
"scores": "detection_scores:0"
}
# default configuration
_config_schema = t.Dict(
{
# path or name of the frozen weights file (*.pb)
t.Key('weights', default="data/models/*.pb"):
t.String(min_length=4),
t.Key('width', default=300):
t.Int(gt=0), # input tensor width
t.Key('height', default=300):
t.Int(gt=0), # output tensor width
t.Key('threshold', default=0.5):
t.Float(gte=0.0,
lte=1.0), # confidence threshold for detected objects
# labels dict or file
t.Key('labels', default={1: 'person'}):
t.Or(t.Dict({}, allow_extra='*'), t.String(min_length=4)),
# device to execute graph
t.Key('device', default='GPU|CPU'):
t.Regexp(r'GPU\|CPU|CPU(?:\:0)?|GPU(?:\:\d)?') >> _parse_device,
t.Key('log_device_placement', default=False):
t.Bool, # TF specific
t.Key('per_process_gpu_memory_fraction', default=0.0):
t.Float(gte=0.0, lte=1.0), # TF specific
},
allow_extra='*')
def __init__(self, **kwargs):
# validate config
try:
self.config = self._config_schema.check(kwargs or {})
except t.DataError as err:
raise ValueError('Wrong model configuration for {}: {}'.format(
self, err))
self._session = None # tf.Session
self._inputs = None # typ.Dict[str, tf.Tensor]
self._outputs = None # typ.Dict[str, tf.Tensor]
self._labels = None # typ.Dict[int, str]
def __str__(self) -> str:
return self.__class__.__name__
def __repr__(self) -> str:
return '<{}>'.format(self)
def __enter__(self):
self.startup()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.shutdown()
@property
def log(self) -> logging.Logger:
return log
def startup(self):
self.log.info("Starting %s ...", self)
self._labels = self.config['labels'] if isinstance(
self.config['labels'], dict) else load_labels_from_file(
self.config['labels'])
if len(self._labels) <= 0:
raise ValueError(f"Labels can't be empty {self._labels}")
tf_config = create_config(
self.config['device'],
log_device_placement=self.config['log_device_placement'],
per_process_gpu_memory_fraction=self.
config['per_process_gpu_memory_fraction'])
graph = import_graph(parse_graph_def(self.config['weights']),
self.config['device'])
self.log.debug("Model (%s) placed on %s", self.config['weights'],
self.config['device'])
self._session = tf.Session(graph=graph, config=tf_config)
self._inputs = {
alias: graph.get_tensor_by_name(name)
for alias, name in self.input_tensors.items()
}
self._outputs = {
alias: graph.get_tensor_by_name(name)
for alias, name in self.output_tensors.items()
}
# warm up
self.log.info("Warming up %s ...", self)
self.process_single(np.zeros((2, 2, 3), dtype=np.uint8))
def shutdown(self):
""" Releases model when object deleted """
self.log.info("Shutdown %s ...", self)
if self._session is None:
return
try:
self._session.close()
self._session = None
except tf.OpError as err:
self.log.error('%s close TF session error: %s. Skipping...', self,
err)
self.log.info("%s Destroyed successfully", self)
def process_single(self, image: np.ndarray) -> typ.List[dict]:
return self.process_batch([image])[0]
def process_batch(self, images: typ.List[np.ndarray]) -> typ.List[dict]:
preprocessed = np.stack([self._preprocess(image) for image in images])
result = self._session.run(
self._outputs, feed_dict={self._inputs['images']: preprocessed})
detections_per_image = []
for image, scores, boxes, labels in zip(images, result['scores'],
result['boxes'],
result['labels']):
detections = []
for score, box, label in zip(scores, boxes, labels):
class_name = self._labels.get(int(label))
if not class_name or score < self.config['threshold']:
continue
# scale boxes wrt initial image size
ymin, xmin, ymax, xmax = (np.tile(image.shape[:2], 2) *
box).astype(np.int32).tolist()
width, height = xmax - xmin, ymax - ymin
detections.append({
'confidence': float(score),
'bounding_box': [xmin, ymin, width, height],
'class_name': class_name,
})
detections_per_image.append(detections)
return detections_per_image
def _preprocess(self, image: np.ndarray) -> np.ndarray:
return cv2.resize(image, (self.config['width'], self.config['height']),
interpolation=cv2.INTER_NEAREST)
def test_is_valid(self):
string_trafaret = t.Float()
assert string_trafaret.is_valid(1.5) == True
assert string_trafaret.is_valid('foo') == False
def __init__(self, *, gte=None, lte=None, gt=None, lt=None, **kwargs):
super(Float, self).__init__(**kwargs)
self._trafaret = t.Float(gte=gte, lte=lte, gt=gt, lt=lt)
if self.allow_none:
self._trafaret |= t.Null()
import datetime
from time import gmtime
import trafaret as t
### Sample simple validation
print t.Int().check(3)
print t.Float().check(2.3)
### Sample complex validation
print t.Dict({'name': t.String}).check({'name': 'Jeff'})
### Sample simple conversion
print t.Int().check('3')
print t.Int().check(3.0)
print t.Float().check('2.3')
### Sample UNIX time to datetime conversion
def from_ts(ts):
return datetime.datetime(*gmtime(ts)[:5])
print(t.Int() >> from_ts).check(1234234234)
### Sample dict to datetime conversion
date = t.Dict(year=t.Int, month=t.Int,
day=t.Int) >> (lambda d: datetime.datetime(**d))
task = t.Dict({
'key': t.String(),
t.Key('timestamp', optional=True): date,
qstr = urlencode(payload)
url = path + qstr
return url
import trafaret as t
from bson.objectid import ObjectId
product_t = t.Dict({
t.Key('_id'): ObjectId,
t.Key('product_name'): t.String(),
t.Key('discription'): t.String(),
t.Key('price'): t.Float(),
t.Key('img_url'): t.String(),
t.Key('product_url'): t.String()
})
lite_product_t = t.Dict({
t.Key('_id'): ObjectId,
t.Key('product_name'): t.String(),
})
products_pagination_t = t.Dict({
t.Key('total'): t.Int,
t.Key('products'): t.List(product_t)
})
wish_t = t.Dict({
from __future__ import print_function
import datetime
from time import gmtime
import trafaret as t
### Sample simple validation
print(t.Int().check(3))
print(t.Float().check(2.3))
### Sample complex validation
print(t.Dict({'name': t.String}).check({'name': 'Jeff'}))
### Sample simple conversion
print(t.Int().check('3'))
print(t.Int().check(3.0))
print(t.Float().check('2.3'))
### Sample UNIX time to datetime conversion
def from_ts(ts):
return datetime.datetime(*gmtime(ts)[:5])
print((t.Int() >> from_ts).check(1234234234))
### Sample dict to datetime conversion
date = t.Dict(year=t.Int, month=t.Int,
day=t.Int) >> (lambda d: datetime.datetime(**d))
task = t.Dict({
class DeepSpeechModel:
# default configuration
_config_schema = t.Dict(
{
# path or name of the frozen model file (*.pb, *.pbmm)
t.Key('model', default="data/models/deepspeech/output_graph.pbmm"):
t.String(min_length=4),
# Path to the language model binary file
t.Key('language_model', default="data/models/deepspeech/lm.binary"):
t.String(min_length=4),
# trie file (build from the same vocabulary as the language model binary)
# https://www.youtube.com/watch?v=-urNrIAQnNo
t.Key('trie', default="data/models/deepspeech/trie"):
t.String(min_length=4),
# The alpha hyperparameter of the CTC decoder. Language Model weight.
t.Key('lm_alpha', default=0.75):
t.Float(),
# The beta hyperparameter of the CTC decoder. Word insertion weight.
t.Key('lm_beta', default=1.85):
t.Float(),
# A larger beam width value generates better results at the cost of decoding time.
t.Key('beam_width', default=500):
t.Int(gt=0),
},
allow_extra='*')
def __init__(self, **kwargs):
# validate config
try:
self.config = self._config_schema.check(kwargs or {})
except t.DataError as err:
raise ValueError('Wrong model configuration for {}: {}'.format(
self, err))
self.model = None
def __str__(self) -> str:
return self.__class__.__name__
def __repr__(self) -> str:
return '<{}>'.format(self)
def __enter__(self):
self.startup()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.shutdown()
@property
def log(self) -> logging.Logger:
return log
def startup(self):
self.log.info("Starting %s ...", self)
if not os.path.isfile(self.config['model']):
raise FileNotFoundError(f"Invalid filename {self.config['model']}")
self.model = Model(self.config['model'], self.config['beam_width'])
if os.path.isfile(self.config['language_model']) and \
os.path.isfile(self.config['trie']):
self.model.enableDecoderWithLM(self.config['language_model'],
self.config['trie'],
self.config['lm_alpha'],
self.config['lm_beta'])
self.log.info("Warming up %s ...", self)
self.model.stt(np.zeros(160, dtype=np.int16))
def shutdown(self):
""" Releases model when object deleted """
self.log.info("Shutdown %s ...", self)
self.model = None
self.log.info("%s Destroyed successfully", self)
def process(self, audio: np.ndarray) -> str:
return self.model.stt(audio)
class TfObjectDetectionModel(object):
"""Implementation for TF Object Detection API Inference"""
# model's input tensor name from official website
# https://github.com/tensorflow/models/blob/master/research/object_detection/object_detection_tutorial.ipynb
input_tensors = {'images': "image_tensor:0"}
# model's output tensors names from official website
# https://github.com/tensorflow/models/blob/master/research/object_detection/object_detection_tutorial.ipynb
output_tensors = {
"labels": "detection_classes:0",
"boxes": "detection_boxes:0",
"scores": "detection_scores:0",
"masks": "detection_masks:0"
}
# default configuration
_config_schema = t.Dict(
{
# path or name of the frozen weights file (*.pb)
t.Key('weights', default="data/models/*.pb"):
t.String(min_length=4),
t.Key('width', default=300):
t.Int(gt=0), # input tensor width
t.Key('height', default=300):
t.Int(gt=0), # output tensor width
t.Key('threshold', default=0.5):
t.Float(gte=0.0,
lte=1.0), # confidence threshold for detected objects
# labels dict or file
t.Key('labels', default={1: 'person'}):
t.Or(t.Dict({}, allow_extra='*'), t.String(min_length=4)),
# device to execute graph
t.Key('device', default='GPU|CPU'):
t.Regexp(r'GPU\|CPU|CPU(?:\:0)?|GPU(?:\:\d)?') >> _parse_device,
t.Key('log_device_placement', default=False):
t.Bool, # TF specific
t.Key('per_process_gpu_memory_fraction', default=0.0):
t.Float(gte=0.0, lte=1.0), # TF specific
},
allow_extra='*')
def __init__(self, **kwargs):
# validate config
try:
self.config = self._config_schema.check(kwargs or {})
except t.DataError as err:
raise ValueError('Wrong model configuration for {}: {}'.format(
self, err))
self._session = None # tf.Session
self._inputs = None # typ.Dict[str, tf.Tensor]
self._outputs = None # typ.Dict[str, tf.Tensor]
self._labels = None # typ.Dict[int, str]
def __str__(self) -> str:
return self.__class__.__name__
def __repr__(self) -> str:
return '<{}>'.format(self)
def __enter__(self):
self.startup()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.shutdown()
@property
def log(self) -> logging.Logger:
return log
def startup(self):
self.log.info("Starting %s ...", self)
self._labels = self.config['labels'] if isinstance(
self.config['labels'], dict) else load_labels_from_file(
self.config['labels'])
if len(self._labels) <= 0:
raise ValueError(f"Labels can't be empty {self._labels}")
tf_config = create_config(
self.config['device'],
log_device_placement=self.config['log_device_placement'],
per_process_gpu_memory_fraction=self.
config['per_process_gpu_memory_fraction'])
graph = import_graph(parse_graph_def(self.config['weights']),
self.config['device'])
has_masks = self.output_tensors['masks'].replace(":0", "") in set(
[n.name for n in graph.as_graph_def().node])
self.log.debug("Model (%s) placed on %s", self.config['weights'],
self.config['device'])
self._session = tf.Session(graph=graph, config=tf_config)
output_tensors = self.output_tensors
if not has_masks:
output_tensors = deepcopy(self.output_tensors)
output_tensors.pop('masks')
self._inputs = {
alias: graph.get_tensor_by_name(name)
for alias, name in self.input_tensors.items()
}
self._outputs = {
alias: graph.get_tensor_by_name(name)
for alias, name in output_tensors.items()
}
# warm up
self.log.info("Warming up %s ...", self)
self.process_single(np.zeros((2, 2, 3), dtype=np.uint8))
def shutdown(self):
""" Releases model when object deleted """
self.log.info("Shutdown %s ...", self)
if self._session is None:
return
try:
self._session.close()
self._session = None
except tf.OpError as err:
self.log.error('%s close TF session error: %s. Skipping...', self,
err)
self.log.info("%s Destroyed successfully", self)
def process_single(self, image: np.ndarray) -> typ.List[typ.List[dict]]:
"""Run inference on single image
Returns: list of detection* per image (ex: process_batch())
"""
return self.process_batch([image])[0]
def process_batch(
self, images: typ.List[np.ndarray]) -> typ.List[typ.List[dict]]:
"""
Returns: list of detection* per image
*detection: dict
| Field name | Type | Description |
|--------------|-------------------------------|------------------------------------|
| confidence | float | class score |
| bounding_box | typ.Tuple[int, int, int, int] | x, y, width, height wrt to image |
| class_name | str | human readable class name |
| mask | np.ndarray[np.bool] | with shape (box_width, box_height) |
"""
preprocessed = np.stack([self._preprocess(image) for image in images])
result = self._session.run(
self._outputs, feed_dict={self._inputs['images']: preprocessed})
detections_per_image = []
has_masks = 'masks' in result
for i, image in enumerate(images):
detections = []
boxes = result['boxes'][i]
scores, labels = result['scores'][i], result['labels'][i]
for j in range(len(scores)):
score = scores[j]
class_name = self._labels.get(int(labels[j]))
if not class_name or score < self.config['threshold']:
continue
# resize bounding box wrt to image size
ymin, xmin, ymax, xmax = (np.tile(image.shape[:2], 2) *
boxes[j]).astype(np.int32).tolist()
width, height = xmax - xmin, ymax - ymin
obj = {
'confidence': float(score),
'bounding_box': [xmin, ymin, width, height],
'class_name': class_name,
}
if has_masks:
# resize mask wrt to bounding box size
mask = cv2.resize(result['masks'][i][j], (width, height),
interpolation=cv2.INTER_NEAREST)
obj['mask'] = mask >= self.config[
'threshold'] # threshold mask
detections.append(obj)
detections_per_image.append(detections)
return detections_per_image
def _preprocess(self, image: np.ndarray) -> np.ndarray:
return cv2.resize(image, (self.config['width'], self.config['height']),
interpolation=cv2.INTER_NEAREST)
@classmethod
def from_config_file(cls, filename: str) -> "TfObjectDetectionModel":
"""
:param filename: filename to model config
"""
return cls(**load_config(filename))
just,
Any,
All,
Not,
Pattern,
unique_strings_list,
ensure_list,
)
from .decimal import Decimal
from .format import format_trafaret
__VERSION__ = (0, 2, 1)
check_number = t.OnError(t.Float() | Decimal(), 'Not a number')
json_schema_type = (
t.Atom('null') & just(t.Null())
| t.Atom('boolean') & just(t.Bool())
| t.Atom('object') & just(t.Type(dict))
| t.Atom('array') & just(t.Type(list))
| t.Atom('number') & just(check_number)
| t.Atom('integer') & just(t.Int())
| t.Atom('string') & just(t.String())
)
def multipleOf(multiplier):
def check(value):
if value % multiplier != 0:
