def test_float_repr(self): res = t.Float() self.assertEqual(repr(res), '<Float>') res = t.Float(gte=1) self.assertEqual(repr(res), '<Float(gte=1)>') res = t.Float(lte=10) self.assertEqual(repr(res), '<Float(lte=10)>') res = t.Float(gte=1, lte=10) self.assertEqual(repr(res), '<Float(gte=1, lte=10)>')
def test_float(self): res = t.Float().check(1.0) self.assertEqual(res, 1.0) res = extract_error(t.Float(), 1 + 3j) self.assertEqual(res, 'value is not float') res = extract_error(t.Float(), 1) self.assertEqual(res, 1.0) res = t.Float(gte=2).check(3.0) self.assertEqual(res, 3.0) res = extract_error(t.Float(gte=2), 1.0) self.assertEqual(res, 'value is less than 2') res = t.Float(lte=10).check(5.0) self.assertEqual(res, 5.0) res = extract_error(t.Float(lte=3), 5.0) self.assertEqual(res, 'value is greater than 3') res = t.Float().check("5.0") self.assertEqual(res, 5.0)
def build_trafaret(sa_type, **kwargs): if isinstance(sa_type, sa.sql.sqltypes.Enum): trafaret = t.Enum(*sa_type.enums, **kwargs) # check for Text should be before String elif isinstance(sa_type, sa.sql.sqltypes.Text): trafaret = t.String(**kwargs) elif isinstance(sa_type, sa.sql.sqltypes.String): trafaret = t.String(max_length=sa_type.length, **kwargs) elif isinstance(sa_type, sa.sql.sqltypes.Integer): trafaret = t.Int(**kwargs) elif isinstance(sa_type, sa.sql.sqltypes.Float): trafaret = t.Float(**kwargs) elif isinstance(sa_type, sa.sql.sqltypes.DateTime): trafaret = DateTime(**kwargs) # RFC3339 elif isinstance(sa_type, sa.sql.sqltypes.Date): trafaret = DateTime(**kwargs) # RFC3339 elif isinstance(sa_type, sa.sql.sqltypes.Boolean): trafaret = t.StrBool(**kwargs) # Add PG related JSON and ARRAY elif isinstance(sa_type, postgresql.JSON): trafaret = AnyDict | t.List(AnyDict) # Add PG related JSON and ARRAY elif isinstance(sa_type, postgresql.ARRAY): item_trafaret = build_trafaret(sa_type.item_type) trafaret = t.List(item_trafaret) elif isinstance(sa_type, postgresql.INET): # TODO: depend on "ipaddress" module? trafaret = t.String(**kwargs) else: type_ = str(sa_type) msg = 'Validator for type {} not implemented'.format(type_) raise NotImplementedError(msg) return trafaret
def construct(arg): ''' Shortcut syntax to define trafarets. - int, str, float and bool will return t.Int, t.String, t.Float and t.Bool - one element list will return t.List - tuple or list with several args will return t.Tuple - dict will return t.Dict. If key has '?' at the and it will be optional and '?' will be removed - any callable will be t.Call - otherwise it will be returned as is construct is recursive and will try construct all lists, tuples and dicts args ''' if isinstance(arg, t.Trafaret): return arg elif isinstance(arg, tuple) or (isinstance(arg, list) and len(arg) > 1): return t.Tuple(*(construct(a) for a in arg)) elif isinstance(arg, list): # if len(arg) == 1 return t.List(construct(arg[0])) elif isinstance(arg, dict): return t.Dict({ construct_key(key): construct(value) for key, value in arg.items() }) elif isinstance(arg, str): return t.Atom(arg) elif isinstance(arg, type): if arg is int: return t.Int() elif arg is float: return t.Float() elif arg is str: return t.String() elif arg is bool: return t.Bool() else: return t.Type(arg) elif callable(arg): return t.Call(arg) else: return arg
def test_list_many_values(self): contract = Contract( [Template(t.Int()), Template(t.String()), Template(t.Float())]) result = contract([1, 'Test', 12.5]) self.assertEqual(result, [1, 'Test', 12.5]) with self.assertRaises(ValueError): result = contract(['error', 'Test', 12.5]) with self.assertRaises(ValueError): result = contract([1, 666, 12.5]) with self.assertRaises(ValueError): result = contract([1, 'Test', 'error']) with self.assertRaises(ValueError): result = contract([1, 'Test']) with self.assertRaises(ValueError): result = contract([1, 'Test', 12.5, 'error']) with self.assertRaises(ValueError): result = contract(42)
def test_list_many_values_with_default(self): contract = AsyncContract([ AsyncTemplate(t.Int(), 42), AsyncTemplate(t.String(), 'null'), AsyncTemplate(t.Float(), 1.5) ]) result = self.loop.run_until_complete(contract([1, 'Test', 12.5])) self.assertEqual(result, [1, 'Test', 12.5]) result = self.loop.run_until_complete(contract(['error', 'Test', 12.5])) self.assertEqual(result, [42, 'Test', 12.5]) result = self.loop.run_until_complete(contract([1, 666, 12.5])) self.assertEqual(result, [1, 'null', 12.5]) result = self.loop.run_until_complete(contract([1, 'Test', 'error'])) self.assertEqual(result, [1, 'Test', 1.5]) with self.assertRaises(ValueError): result = self.loop.run_until_complete(contract([1, 'Test'])) with self.assertRaises(ValueError): result = self.loop.run_until_complete( contract([1, 'Test', 12.5, 'error'])) with self.assertRaises(ValueError): result = self.loop.run_until_complete(contract(42))
class TfObjectDetectionModel(object): """Implementation for TF Object Detection API Inference""" # model's input tensor name from official website # https://github.com/tensorflow/models/blob/master/research/object_detection/object_detection_tutorial.ipynb input_tensors = {'images': "image_tensor:0"} # model's output tensors names from official website # https://github.com/tensorflow/models/blob/master/research/object_detection/object_detection_tutorial.ipynb output_tensors = { "labels": "detection_classes:0", "boxes": "detection_boxes:0", "scores": "detection_scores:0" } # default configuration _config_schema = t.Dict( { # path or name of the frozen weights file (*.pb) t.Key('weights', default="data/models/*.pb"): t.String(min_length=4), t.Key('width', default=300): t.Int(gt=0), # input tensor width t.Key('height', default=300): t.Int(gt=0), # output tensor width t.Key('threshold', default=0.5): t.Float(gte=0.0, lte=1.0), # confidence threshold for detected objects # labels dict or file t.Key('labels', default={1: 'person'}): t.Or(t.Dict({}, allow_extra='*'), t.String(min_length=4)), # device to execute graph t.Key('device', default='GPU|CPU'): t.Regexp(r'GPU\|CPU|CPU(?:\:0)?|GPU(?:\:\d)?') >> _parse_device, t.Key('log_device_placement', default=False): t.Bool, # TF specific t.Key('per_process_gpu_memory_fraction', default=0.0): t.Float(gte=0.0, lte=1.0), # TF specific }, allow_extra='*') def __init__(self, **kwargs): # validate config try: self.config = self._config_schema.check(kwargs or {}) except t.DataError as err: raise ValueError('Wrong model configuration for {}: {}'.format( self, err)) self._session = None # tf.Session self._inputs = None # typ.Dict[str, tf.Tensor] self._outputs = None # typ.Dict[str, tf.Tensor] self._labels = None # typ.Dict[int, str] def __str__(self) -> str: return self.__class__.__name__ def __repr__(self) -> str: return '<{}>'.format(self) def __enter__(self): self.startup() return self def __exit__(self, exc_type, exc_val, exc_tb): self.shutdown() @property def log(self) -> logging.Logger: return log def startup(self): self.log.info("Starting %s ...", self) self._labels = self.config['labels'] if isinstance( self.config['labels'], dict) else load_labels_from_file( self.config['labels']) if len(self._labels) <= 0: raise ValueError(f"Labels can't be empty {self._labels}") tf_config = create_config( self.config['device'], log_device_placement=self.config['log_device_placement'], per_process_gpu_memory_fraction=self. config['per_process_gpu_memory_fraction']) graph = import_graph(parse_graph_def(self.config['weights']), self.config['device']) self.log.debug("Model (%s) placed on %s", self.config['weights'], self.config['device']) self._session = tf.Session(graph=graph, config=tf_config) self._inputs = { alias: graph.get_tensor_by_name(name) for alias, name in self.input_tensors.items() } self._outputs = { alias: graph.get_tensor_by_name(name) for alias, name in self.output_tensors.items() } # warm up self.log.info("Warming up %s ...", self) self.process_single(np.zeros((2, 2, 3), dtype=np.uint8)) def shutdown(self): """ Releases model when object deleted """ self.log.info("Shutdown %s ...", self) if self._session is None: return try: self._session.close() self._session = None except tf.OpError as err: self.log.error('%s close TF session error: %s. Skipping...', self, err) self.log.info("%s Destroyed successfully", self) def process_single(self, image: np.ndarray) -> typ.List[dict]: return self.process_batch([image])[0] def process_batch(self, images: typ.List[np.ndarray]) -> typ.List[dict]: preprocessed = np.stack([self._preprocess(image) for image in images]) result = self._session.run( self._outputs, feed_dict={self._inputs['images']: preprocessed}) detections_per_image = [] for image, scores, boxes, labels in zip(images, result['scores'], result['boxes'], result['labels']): detections = [] for score, box, label in zip(scores, boxes, labels): class_name = self._labels.get(int(label)) if not class_name or score < self.config['threshold']: continue # scale boxes wrt initial image size ymin, xmin, ymax, xmax = (np.tile(image.shape[:2], 2) * box).astype(np.int32).tolist() width, height = xmax - xmin, ymax - ymin detections.append({ 'confidence': float(score), 'bounding_box': [xmin, ymin, width, height], 'class_name': class_name, }) detections_per_image.append(detections) return detections_per_image def _preprocess(self, image: np.ndarray) -> np.ndarray: return cv2.resize(image, (self.config['width'], self.config['height']), interpolation=cv2.INTER_NEAREST)
def test_is_valid(self): string_trafaret = t.Float() assert string_trafaret.is_valid(1.5) == True assert string_trafaret.is_valid('foo') == False
def __init__(self, *, gte=None, lte=None, gt=None, lt=None, **kwargs): super(Float, self).__init__(**kwargs) self._trafaret = t.Float(gte=gte, lte=lte, gt=gt, lt=lt) if self.allow_none: self._trafaret |= t.Null()
import datetime from time import gmtime import trafaret as t ### Sample simple validation print t.Int().check(3) print t.Float().check(2.3) ### Sample complex validation print t.Dict({'name': t.String}).check({'name': 'Jeff'}) ### Sample simple conversion print t.Int().check('3') print t.Int().check(3.0) print t.Float().check('2.3') ### Sample UNIX time to datetime conversion def from_ts(ts): return datetime.datetime(*gmtime(ts)[:5]) print(t.Int() >> from_ts).check(1234234234) ### Sample dict to datetime conversion date = t.Dict(year=t.Int, month=t.Int, day=t.Int) >> (lambda d: datetime.datetime(**d)) task = t.Dict({ 'key': t.String(), t.Key('timestamp', optional=True): date,
qstr = urlencode(payload) url = path + qstr return url import trafaret as t from bson.objectid import ObjectId product_t = t.Dict({ t.Key('_id'): ObjectId, t.Key('product_name'): t.String(), t.Key('discription'): t.String(), t.Key('price'): t.Float(), t.Key('img_url'): t.String(), t.Key('product_url'): t.String() }) lite_product_t = t.Dict({ t.Key('_id'): ObjectId, t.Key('product_name'): t.String(), }) products_pagination_t = t.Dict({ t.Key('total'): t.Int, t.Key('products'): t.List(product_t) }) wish_t = t.Dict({
from __future__ import print_function import datetime from time import gmtime import trafaret as t ### Sample simple validation print(t.Int().check(3)) print(t.Float().check(2.3)) ### Sample complex validation print(t.Dict({'name': t.String}).check({'name': 'Jeff'})) ### Sample simple conversion print(t.Int().check('3')) print(t.Int().check(3.0)) print(t.Float().check('2.3')) ### Sample UNIX time to datetime conversion def from_ts(ts): return datetime.datetime(*gmtime(ts)[:5]) print((t.Int() >> from_ts).check(1234234234)) ### Sample dict to datetime conversion date = t.Dict(year=t.Int, month=t.Int, day=t.Int) >> (lambda d: datetime.datetime(**d)) task = t.Dict({
class DeepSpeechModel: # default configuration _config_schema = t.Dict( { # path or name of the frozen model file (*.pb, *.pbmm) t.Key('model', default="data/models/deepspeech/output_graph.pbmm"): t.String(min_length=4), # Path to the language model binary file t.Key('language_model', default="data/models/deepspeech/lm.binary"): t.String(min_length=4), # trie file (build from the same vocabulary as the language model binary) # https://www.youtube.com/watch?v=-urNrIAQnNo t.Key('trie', default="data/models/deepspeech/trie"): t.String(min_length=4), # The alpha hyperparameter of the CTC decoder. Language Model weight. t.Key('lm_alpha', default=0.75): t.Float(), # The beta hyperparameter of the CTC decoder. Word insertion weight. t.Key('lm_beta', default=1.85): t.Float(), # A larger beam width value generates better results at the cost of decoding time. t.Key('beam_width', default=500): t.Int(gt=0), }, allow_extra='*') def __init__(self, **kwargs): # validate config try: self.config = self._config_schema.check(kwargs or {}) except t.DataError as err: raise ValueError('Wrong model configuration for {}: {}'.format( self, err)) self.model = None def __str__(self) -> str: return self.__class__.__name__ def __repr__(self) -> str: return '<{}>'.format(self) def __enter__(self): self.startup() return self def __exit__(self, exc_type, exc_val, exc_tb): self.shutdown() @property def log(self) -> logging.Logger: return log def startup(self): self.log.info("Starting %s ...", self) if not os.path.isfile(self.config['model']): raise FileNotFoundError(f"Invalid filename {self.config['model']}") self.model = Model(self.config['model'], self.config['beam_width']) if os.path.isfile(self.config['language_model']) and \ os.path.isfile(self.config['trie']): self.model.enableDecoderWithLM(self.config['language_model'], self.config['trie'], self.config['lm_alpha'], self.config['lm_beta']) self.log.info("Warming up %s ...", self) self.model.stt(np.zeros(160, dtype=np.int16)) def shutdown(self): """ Releases model when object deleted """ self.log.info("Shutdown %s ...", self) self.model = None self.log.info("%s Destroyed successfully", self) def process(self, audio: np.ndarray) -> str: return self.model.stt(audio)
class TfObjectDetectionModel(object): """Implementation for TF Object Detection API Inference""" # model's input tensor name from official website # https://github.com/tensorflow/models/blob/master/research/object_detection/object_detection_tutorial.ipynb input_tensors = {'images': "image_tensor:0"} # model's output tensors names from official website # https://github.com/tensorflow/models/blob/master/research/object_detection/object_detection_tutorial.ipynb output_tensors = { "labels": "detection_classes:0", "boxes": "detection_boxes:0", "scores": "detection_scores:0", "masks": "detection_masks:0" } # default configuration _config_schema = t.Dict( { # path or name of the frozen weights file (*.pb) t.Key('weights', default="data/models/*.pb"): t.String(min_length=4), t.Key('width', default=300): t.Int(gt=0), # input tensor width t.Key('height', default=300): t.Int(gt=0), # output tensor width t.Key('threshold', default=0.5): t.Float(gte=0.0, lte=1.0), # confidence threshold for detected objects # labels dict or file t.Key('labels', default={1: 'person'}): t.Or(t.Dict({}, allow_extra='*'), t.String(min_length=4)), # device to execute graph t.Key('device', default='GPU|CPU'): t.Regexp(r'GPU\|CPU|CPU(?:\:0)?|GPU(?:\:\d)?') >> _parse_device, t.Key('log_device_placement', default=False): t.Bool, # TF specific t.Key('per_process_gpu_memory_fraction', default=0.0): t.Float(gte=0.0, lte=1.0), # TF specific }, allow_extra='*') def __init__(self, **kwargs): # validate config try: self.config = self._config_schema.check(kwargs or {}) except t.DataError as err: raise ValueError('Wrong model configuration for {}: {}'.format( self, err)) self._session = None # tf.Session self._inputs = None # typ.Dict[str, tf.Tensor] self._outputs = None # typ.Dict[str, tf.Tensor] self._labels = None # typ.Dict[int, str] def __str__(self) -> str: return self.__class__.__name__ def __repr__(self) -> str: return '<{}>'.format(self) def __enter__(self): self.startup() return self def __exit__(self, exc_type, exc_val, exc_tb): self.shutdown() @property def log(self) -> logging.Logger: return log def startup(self): self.log.info("Starting %s ...", self) self._labels = self.config['labels'] if isinstance( self.config['labels'], dict) else load_labels_from_file( self.config['labels']) if len(self._labels) <= 0: raise ValueError(f"Labels can't be empty {self._labels}") tf_config = create_config( self.config['device'], log_device_placement=self.config['log_device_placement'], per_process_gpu_memory_fraction=self. config['per_process_gpu_memory_fraction']) graph = import_graph(parse_graph_def(self.config['weights']), self.config['device']) has_masks = self.output_tensors['masks'].replace(":0", "") in set( [n.name for n in graph.as_graph_def().node]) self.log.debug("Model (%s) placed on %s", self.config['weights'], self.config['device']) self._session = tf.Session(graph=graph, config=tf_config) output_tensors = self.output_tensors if not has_masks: output_tensors = deepcopy(self.output_tensors) output_tensors.pop('masks') self._inputs = { alias: graph.get_tensor_by_name(name) for alias, name in self.input_tensors.items() } self._outputs = { alias: graph.get_tensor_by_name(name) for alias, name in output_tensors.items() } # warm up self.log.info("Warming up %s ...", self) self.process_single(np.zeros((2, 2, 3), dtype=np.uint8)) def shutdown(self): """ Releases model when object deleted """ self.log.info("Shutdown %s ...", self) if self._session is None: return try: self._session.close() self._session = None except tf.OpError as err: self.log.error('%s close TF session error: %s. Skipping...', self, err) self.log.info("%s Destroyed successfully", self) def process_single(self, image: np.ndarray) -> typ.List[typ.List[dict]]: """Run inference on single image Returns: list of detection* per image (ex: process_batch()) """ return self.process_batch([image])[0] def process_batch( self, images: typ.List[np.ndarray]) -> typ.List[typ.List[dict]]: """ Returns: list of detection* per image *detection: dict | Field name | Type | Description | |--------------|-------------------------------|------------------------------------| | confidence | float | class score | | bounding_box | typ.Tuple[int, int, int, int] | x, y, width, height wrt to image | | class_name | str | human readable class name | | mask | np.ndarray[np.bool] | with shape (box_width, box_height) | """ preprocessed = np.stack([self._preprocess(image) for image in images]) result = self._session.run( self._outputs, feed_dict={self._inputs['images']: preprocessed}) detections_per_image = [] has_masks = 'masks' in result for i, image in enumerate(images): detections = [] boxes = result['boxes'][i] scores, labels = result['scores'][i], result['labels'][i] for j in range(len(scores)): score = scores[j] class_name = self._labels.get(int(labels[j])) if not class_name or score < self.config['threshold']: continue # resize bounding box wrt to image size ymin, xmin, ymax, xmax = (np.tile(image.shape[:2], 2) * boxes[j]).astype(np.int32).tolist() width, height = xmax - xmin, ymax - ymin obj = { 'confidence': float(score), 'bounding_box': [xmin, ymin, width, height], 'class_name': class_name, } if has_masks: # resize mask wrt to bounding box size mask = cv2.resize(result['masks'][i][j], (width, height), interpolation=cv2.INTER_NEAREST) obj['mask'] = mask >= self.config[ 'threshold'] # threshold mask detections.append(obj) detections_per_image.append(detections) return detections_per_image def _preprocess(self, image: np.ndarray) -> np.ndarray: return cv2.resize(image, (self.config['width'], self.config['height']), interpolation=cv2.INTER_NEAREST) @classmethod def from_config_file(cls, filename: str) -> "TfObjectDetectionModel": """ :param filename: filename to model config """ return cls(**load_config(filename))
just, Any, All, Not, Pattern, unique_strings_list, ensure_list, ) from .decimal import Decimal from .format import format_trafaret __VERSION__ = (0, 2, 1) check_number = t.OnError(t.Float() | Decimal(), 'Not a number') json_schema_type = ( t.Atom('null') & just(t.Null()) | t.Atom('boolean') & just(t.Bool()) | t.Atom('object') & just(t.Type(dict)) | t.Atom('array') & just(t.Type(list)) | t.Atom('number') & just(check_number) | t.Atom('integer') & just(t.Int()) | t.Atom('string') & just(t.String()) ) def multipleOf(multiplier): def check(value): if value % multiplier != 0: