def __init__(self,
*,
allow_blank=False,
regex=None,
choices=None,
min_length=None,
max_length=None,
**kwargs):
super(String, self).__init__(**kwargs)
if regex is not None:
self._trafaret = t.Regexp(regex)
else:
self._trafaret = t.String(allow_blank=allow_blank,
min_length=min_length,
max_length=max_length)
self.choices = None
if choices and is_collection(choices):
if isinstance(choices, type(Enum)):
self.choices = choices
self._trafaret &= t.Enum(*choices.__members__.keys())
else:
self._trafaret &= t.Enum(*choices)
if self.allow_none:
self._trafaret |= t.Null()
def __init__(self,
request: web.Request,
prev_cursor=None,
next_cursor=None,
cursor_regex: str = None):
super(Cursor, self).__init__(request)
self.cursor = request.query.get('page[cursor]', self.FIRST)
if isinstance(self.cursor, str):
if cursor_regex is not None:
try:
self.cursor = t.Regexp(cursor_regex).check(self.cursor)
except t.DataError:
raise HTTPBadRequest(detail='The cursor is invalid.',
source_parameter='page[cursor]')
self.cursor = make_sentinel(var_name=str(self.cursor))
self.prev_cursor = \
make_sentinel(var_name=str(prev_cursor)) if prev_cursor else None
self.next_cursor = \
make_sentinel(var_name=str(next_cursor)) if next_cursor else None
self.limit = request.query.get('page[limit]', DEFAULT_LIMIT)
try:
self.limit = t.Int(gt=0).check(self.limit)
except t.DataError:
raise HTTPBadRequest(detail='The limit must be an integer > 0.',
source_parameter='page[limit]')
class TestList(unittest.TestCase):
TRAFARET = T.Dict({
"hosts": T.List(T.String() & T.Regexp("\w+:\d+")),
})
def test_ok(self):
self.assertEqual(
get_err(
self.TRAFARET, u"""\
hosts:
- bear:8080
- cat:7070
"""), None)
def test_err(self):
self.assertEqual(
get_err(
self.TRAFARET, u"""\
hosts:
- bear:8080
- cat:x
"""), "config.yaml:3: hosts[1]: "
"does not match pattern \\w+:\\d+\n")
from models_library.basic_types import PortInt, VersionTag
from servicelib.application_keys import APP_CLIENT_SESSION_KEY, APP_CONFIG_KEY
CONFIG_SECTION_NAME = "catalog"
_default_values = {
"host": os.environ.get("CATALOG_HOST", "catalog"),
"port": int(os.environ.get("CATALOG_PORT", 8000)),
}
schema = T.Dict({
T.Key("enabled", default=True, optional=True): T.Bool(),
T.Key("host", default=_default_values["host"]): T.String(),
T.Key("port", default=_default_values["port"]): T.ToInt(),
T.Key("version", default="v0"):
T.Regexp(regexp=r"^v\d+"), # catalog API version basepath
})
class CatalogSettings(BaseSettings):
enabled: bool = True
host: str = "catalog"
port: PortInt = 8000
vtag: VersionTag = Field("v0",
alias="version",
description="Catalog service API's version tag")
class Config:
prefix = "CATALOG_"
import trafaret as t
from aiohttp.web import HTTPBadRequest
from .fields import Fields
_email_scheme = t.Dict({t.Key(Fields.EMAIL): t.Email})
_password_scheme = t.Dict(
{t.Key(Fields.PASSWORD): t.Regexp(regexp=r'^[A-Za-z0-9_]{4,64}$')})
_token_scheme = t.Dict({
t.Key(Fields.TOKEN):
t.Regexp(regexp=r'^[A-Za-z0-9-_=]+\.[A-Za-z0-9-_=]+\.[A-Za-z0-9-_=]+$'),
})
_ref_token_scheme = t.Dict(
{t.Key(Fields.REF_TOKEN): t.Regexp(regexp=r'^[0-9a-f]{32}$')})
def _fetcher(data, scheme, key, exc_msg):
scheme = scheme.allow_extra('*')
try:
data = scheme(data)
except t.DataError as exc:
print(exc)
raise HTTPBadRequest(reason=exc_msg)
return data[key]
def fetch_email(json):
return _fetcher(json, _email_scheme, Fields.EMAIL, "Invalid email")
import pydash as _
import trafaret as t
import datetime
from functools import partial
from jinja2.utils import import_string
from trafaret.contrib.object_id import MongoId
from trafaret.contrib.rfc_3339 import DateTime
Optional = partial(t.Key, optional=True)
SimpleType = t.IntRaw | t.Bool | t.String | t.FloatRaw
DateTimeType = DateTime | t.Type(datetime.datetime)
NumericType = t.Float | t.Int >> (lambda val: float(val))
URLType = t.Regexp(r'^([a-z]{2,5}:)?(\/\/?)?[a-z][a-z0-9\.\-\/]+$')
OptionValue = t.String(
allow_blank=True) | t.Bool | t.Float | t.Int | t.Type(dict)
Optional = partial(t.Key, optional=True)
SimpleDoc = t.Dict({
t.Key('id', optional=True) >> '_id': MongoId,
Optional('_id'): MongoId
})
TimestampDoc = SimpleDoc + t.Dict({
Optional('created', default=datetime.datetime.now):
DateTimeType | t.Null,
Optional('modified'):
DateTimeType
})
set(g_dev_type))
objs_per_group[group_id]['g_smp'] += c_info['smp']
objs_per_group[group_id]['g_gpu_mem_allocated'] += c_info[
'gpu_mem_allocated']
objs_per_group[group_id]['g_gpu_allocated'] += c_info[
'gpu_allocated']
objs_per_group[group_id]['c_infos'].append(c_info)
return list(objs_per_group.values())
@atomic
@server_status_required(READ_ALLOWED)
@superadmin_required
@check_api_params(t.Dict({
tx.MultiKey('group_ids'): t.List(t.String) | t.Null,
t.Key('month'): t.Regexp(r'^\d{6}', re.ASCII),
}),
loads=_json_loads)
async def usage_per_month(request: web.Request, params: Any) -> web.Response:
'''
Return usage statistics of terminated containers for a specified month.
The date/time comparison is done using the configured timezone.
:param group_ids: If not None, query containers only in those groups.
:param month: The year-month to query usage statistics. ex) "202006" to query for Jun 2020
'''
log.info('USAGE_PER_MONTH (g:[{}], month:{})',
','.join(params['group_ids']), params['month'])
local_tz = request.app['config']['system']['timezone']
try:
start_date = datetime.strptime(params['month'],
class TfObjectDetectionModel(object):
"""Implementation for TF Object Detection API Inference"""
# model's input tensor name from official website
# https://github.com/tensorflow/models/blob/master/research/object_detection/object_detection_tutorial.ipynb
input_tensors = {'images': "image_tensor:0"}
# model's output tensors names from official website
# https://github.com/tensorflow/models/blob/master/research/object_detection/object_detection_tutorial.ipynb
output_tensors = {
"labels": "detection_classes:0",
"boxes": "detection_boxes:0",
"scores": "detection_scores:0",
"masks": "detection_masks:0"
}
# default configuration
_config_schema = t.Dict(
{
# path or name of the frozen weights file (*.pb)
t.Key('weights', default="data/models/*.pb"):
t.String(min_length=4),
t.Key('width', default=300):
t.Int(gt=0), # input tensor width
t.Key('height', default=300):
t.Int(gt=0), # output tensor width
t.Key('threshold', default=0.5):
t.Float(gte=0.0,
lte=1.0), # confidence threshold for detected objects
# labels dict or file
t.Key('labels', default={1: 'person'}):
t.Or(t.Dict({}, allow_extra='*'), t.String(min_length=4)),
# device to execute graph
t.Key('device', default='GPU|CPU'):
t.Regexp(r'GPU\|CPU|CPU(?:\:0)?|GPU(?:\:\d)?') >> _parse_device,
t.Key('log_device_placement', default=False):
t.Bool, # TF specific
t.Key('per_process_gpu_memory_fraction', default=0.0):
t.Float(gte=0.0, lte=1.0), # TF specific
},
allow_extra='*')
def __init__(self, **kwargs):
# validate config
try:
self.config = self._config_schema.check(kwargs or {})
except t.DataError as err:
raise ValueError('Wrong model configuration for {}: {}'.format(
self, err))
self._session = None # tf.Session
self._inputs = None # typ.Dict[str, tf.Tensor]
self._outputs = None # typ.Dict[str, tf.Tensor]
self._labels = None # typ.Dict[int, str]
def __str__(self) -> str:
return self.__class__.__name__
def __repr__(self) -> str:
return '<{}>'.format(self)
def __enter__(self):
self.startup()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.shutdown()
@property
def log(self) -> logging.Logger:
return log
def startup(self):
self.log.info("Starting %s ...", self)
self._labels = self.config['labels'] if isinstance(
self.config['labels'], dict) else load_labels_from_file(
self.config['labels'])
if len(self._labels) <= 0:
raise ValueError(f"Labels can't be empty {self._labels}")
tf_config = create_config(
self.config['device'],
log_device_placement=self.config['log_device_placement'],
per_process_gpu_memory_fraction=self.
config['per_process_gpu_memory_fraction'])
graph = import_graph(parse_graph_def(self.config['weights']),
self.config['device'])
has_masks = self.output_tensors['masks'].replace(":0", "") in set(
[n.name for n in graph.as_graph_def().node])
self.log.debug("Model (%s) placed on %s", self.config['weights'],
self.config['device'])
self._session = tf.Session(graph=graph, config=tf_config)
output_tensors = self.output_tensors
if not has_masks:
output_tensors = deepcopy(self.output_tensors)
output_tensors.pop('masks')
self._inputs = {
alias: graph.get_tensor_by_name(name)
for alias, name in self.input_tensors.items()
}
self._outputs = {
alias: graph.get_tensor_by_name(name)
for alias, name in output_tensors.items()
}
# warm up
self.log.info("Warming up %s ...", self)
self.process_single(np.zeros((2, 2, 3), dtype=np.uint8))
def shutdown(self):
""" Releases model when object deleted """
self.log.info("Shutdown %s ...", self)
if self._session is None:
return
try:
self._session.close()
self._session = None
except tf.OpError as err:
self.log.error('%s close TF session error: %s. Skipping...', self,
err)
self.log.info("%s Destroyed successfully", self)
def process_single(self, image: np.ndarray) -> typ.List[typ.List[dict]]:
"""Run inference on single image
Returns: list of detection* per image (ex: process_batch())
"""
return self.process_batch([image])[0]
def process_batch(
self, images: typ.List[np.ndarray]) -> typ.List[typ.List[dict]]:
"""
Returns: list of detection* per image
*detection: dict
| Field name | Type | Description |
|--------------|-------------------------------|------------------------------------|
| confidence | float | class score |
| bounding_box | typ.Tuple[int, int, int, int] | x, y, width, height wrt to image |
| class_name | str | human readable class name |
| mask | np.ndarray[np.bool] | with shape (box_width, box_height) |
"""
preprocessed = np.stack([self._preprocess(image) for image in images])
result = self._session.run(
self._outputs, feed_dict={self._inputs['images']: preprocessed})
detections_per_image = []
has_masks = 'masks' in result
for i, image in enumerate(images):
detections = []
boxes = result['boxes'][i]
scores, labels = result['scores'][i], result['labels'][i]
for j in range(len(scores)):
score = scores[j]
class_name = self._labels.get(int(labels[j]))
if not class_name or score < self.config['threshold']:
continue
# resize bounding box wrt to image size
ymin, xmin, ymax, xmax = (np.tile(image.shape[:2], 2) *
boxes[j]).astype(np.int32).tolist()
width, height = xmax - xmin, ymax - ymin
obj = {
'confidence': float(score),
'bounding_box': [xmin, ymin, width, height],
'class_name': class_name,
}
if has_masks:
# resize mask wrt to bounding box size
mask = cv2.resize(result['masks'][i][j], (width, height),
interpolation=cv2.INTER_NEAREST)
obj['mask'] = mask >= self.config[
'threshold'] # threshold mask
detections.append(obj)
detections_per_image.append(detections)
return detections_per_image
def _preprocess(self, image: np.ndarray) -> np.ndarray:
return cv2.resize(image, (self.config['width'], self.config['height']),
interpolation=cv2.INTER_NEAREST)
@classmethod
def from_config_file(cls, filename: str) -> "TfObjectDetectionModel":
"""
:param filename: filename to model config
"""
return cls(**load_config(filename))
class TfObjectDetectionModel(object):
"""Implementation for TF Object Detection API Inference"""
# model's input tensor name from official website
# https://github.com/tensorflow/models/blob/master/research/object_detection/object_detection_tutorial.ipynb
input_tensors = {'images': "image_tensor:0"}
# model's output tensors names from official website
# https://github.com/tensorflow/models/blob/master/research/object_detection/object_detection_tutorial.ipynb
output_tensors = {
"labels": "detection_classes:0",
"boxes": "detection_boxes:0",
"scores": "detection_scores:0"
}
# default configuration
_config_schema = t.Dict(
{
# path or name of the frozen weights file (*.pb)
t.Key('weights', default="data/models/*.pb"):
t.String(min_length=4),
t.Key('width', default=300):
t.Int(gt=0), # input tensor width
t.Key('height', default=300):
t.Int(gt=0), # output tensor width
t.Key('threshold', default=0.5):
t.Float(gte=0.0,
lte=1.0), # confidence threshold for detected objects
# labels dict or file
t.Key('labels', default={1: 'person'}):
t.Or(t.Dict({}, allow_extra='*'), t.String(min_length=4)),
# device to execute graph
t.Key('device', default='GPU|CPU'):
t.Regexp(r'GPU\|CPU|CPU(?:\:0)?|GPU(?:\:\d)?') >> _parse_device,
t.Key('log_device_placement', default=False):
t.Bool, # TF specific
t.Key('per_process_gpu_memory_fraction', default=0.0):
t.Float(gte=0.0, lte=1.0), # TF specific
},
allow_extra='*')
def __init__(self, **kwargs):
# validate config
try:
self.config = self._config_schema.check(kwargs or {})
except t.DataError as err:
raise ValueError('Wrong model configuration for {}: {}'.format(
self, err))
self._session = None # tf.Session
self._inputs = None # typ.Dict[str, tf.Tensor]
self._outputs = None # typ.Dict[str, tf.Tensor]
self._labels = None # typ.Dict[int, str]
def __str__(self) -> str:
return self.__class__.__name__
def __repr__(self) -> str:
return '<{}>'.format(self)
def __enter__(self):
self.startup()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.shutdown()
@property
def log(self) -> logging.Logger:
return log
def startup(self):
self.log.info("Starting %s ...", self)
self._labels = self.config['labels'] if isinstance(
self.config['labels'], dict) else load_labels_from_file(
self.config['labels'])
if len(self._labels) <= 0:
raise ValueError(f"Labels can't be empty {self._labels}")
tf_config = create_config(
self.config['device'],
log_device_placement=self.config['log_device_placement'],
per_process_gpu_memory_fraction=self.
config['per_process_gpu_memory_fraction'])
graph = import_graph(parse_graph_def(self.config['weights']),
self.config['device'])
self.log.debug("Model (%s) placed on %s", self.config['weights'],
self.config['device'])
self._session = tf.Session(graph=graph, config=tf_config)
self._inputs = {
alias: graph.get_tensor_by_name(name)
for alias, name in self.input_tensors.items()
}
self._outputs = {
alias: graph.get_tensor_by_name(name)
for alias, name in self.output_tensors.items()
}
# warm up
self.log.info("Warming up %s ...", self)
self.process_single(np.zeros((2, 2, 3), dtype=np.uint8))
def shutdown(self):
""" Releases model when object deleted """
self.log.info("Shutdown %s ...", self)
if self._session is None:
return
try:
self._session.close()
self._session = None
except tf.OpError as err:
self.log.error('%s close TF session error: %s. Skipping...', self,
err)
self.log.info("%s Destroyed successfully", self)
def process_single(self, image: np.ndarray) -> typ.List[dict]:
return self.process_batch([image])[0]
def process_batch(self, images: typ.List[np.ndarray]) -> typ.List[dict]:
preprocessed = np.stack([self._preprocess(image) for image in images])
result = self._session.run(
self._outputs, feed_dict={self._inputs['images']: preprocessed})
detections_per_image = []
for image, scores, boxes, labels in zip(images, result['scores'],
result['boxes'],
result['labels']):
detections = []
for score, box, label in zip(scores, boxes, labels):
class_name = self._labels.get(int(label))
if not class_name or score < self.config['threshold']:
continue
# scale boxes wrt initial image size
ymin, xmin, ymax, xmax = (np.tile(image.shape[:2], 2) *
box).astype(np.int32).tolist()
width, height = xmax - xmin, ymax - ymin
detections.append({
'confidence': float(score),
'bounding_box': [xmin, ymin, width, height],
'class_name': class_name,
})
detections_per_image.append(detections)
return detections_per_image
def _preprocess(self, image: np.ndarray) -> np.ndarray:
return cv2.resize(image, (self.config['width'], self.config['height']),
interpolation=cv2.INTER_NEAREST)
from aiohttp import ClientSession, web
from servicelib.application_keys import APP_CLIENT_SESSION_KEY, APP_CONFIG_KEY
CONFIG_SECTION_NAME = "catalog"
_default_values = {
"host": os.environ.get("CATALOG_HOST", "catalog"),
"port": int(os.environ.get("CATALOG_PORT", 8000)),
}
schema = T.Dict(
{
T.Key("enabled", default=True, optional=True): T.Bool(),
T.Key("host", default=_default_values["host"]): T.String(),
T.Key("port", default=_default_values["port"]): T.ToInt(),
T.Key("version", default="v0"): T.Regexp(
regexp=r"^v\d+"
), # catalog API version basepath
}
)
def get_config(app: web.Application) -> Dict:
return app[APP_CONFIG_KEY][CONFIG_SECTION_NAME]
def get_client_session(app: web.Application) -> ClientSession:
return app[APP_CLIENT_SESSION_KEY]
def test_upper(self):
trafaret = t.Regexp('\w+-\w+') & str.upper
self.assertEqual(trafaret('abc-Abc'), 'ABC-ABC')
CONFIG_FILENAME = 'batch_scoring.ini'
def verify_objectid(value):
"""Verify if id_ is a proper ObjectId. """
try:
t.Regexp(regexp='^[A-Fa-f0-9]{24}$').check(value)
except t.DataError:
raise ValueError('id {} not a valid project/model id'.format(value))
config_validator = t.Dict({
OptKey('host'):
t.String,
OptKey('project_id'):
t.Regexp(regexp='^[A-Fa-f0-9]{24}$'),
OptKey('model_id'):
t.Regexp(regexp='^[A-Fa-f0-9]{24}$'),
OptKey('deployment_id'):
t.Regexp(regexp='^[A-Fa-f0-9]{24}$'),
OptKey('import_id'):
t.String,
OptKey('n_retry'):
t.Int,
OptKey('keep_cols'):
t.String,
OptKey('n_concurrent'):
t.Int,
OptKey('dataset'):
t.String,
OptKey('n_samples'):
def verify_objectid(value):
"""Verify if id_ is a proper ObjectId. """
try:
t.Regexp(regexp='^[A-Fa-f0-9]{24}$').check(value)
except t.DataError:
raise ValueError('id {} not a valid project/model id'.format(value))
- config-file schema
- settings
"""
from typing import Dict
import trafaret as T
from aiohttp import ClientSession, web
from servicelib.application_keys import APP_CLIENT_SESSION_KEY, APP_CONFIG_KEY
CONFIG_SECTION_NAME = "storage"
schema = T.Dict(
{
T.Key("enabled", default=True, optional=True): T.Bool(),
T.Key("host", default="storage"): T.String(),
T.Key("port", default=11111): T.ToInt(),
T.Key("version", default="v0"): T.Regexp(
regexp=r"^v\d+"
), # storage API version basepath
}
)
def get_config(app: web.Application) -> Dict:
return app[APP_CONFIG_KEY][CONFIG_SECTION_NAME]
def get_client_session(app: web.Application) -> ClientSession:
return app[APP_CLIENT_SESSION_KEY]
tx.AliasedKey(['clusterSize', 'cluster_size'], default=None):
t.Null | t.Int[1:],
tx.AliasedKey(['scalingGroup', 'scaling_group'], default=None):
t.Null | t.String,
t.Key('resources', default=None):
t.Null | t.Mapping(t.String, t.Any),
t.Key('resource_opts', default=None):
t.Null | t.Mapping(t.String, t.Any),
})
@server_status_required(ALL_ALLOWED)
@auth_required
@check_api_params(t.Dict({
t.Key('clientSessionToken') >> 'sess_id':
t.Regexp(r'^(?=.{4,64}$)\w[\w.-]*\w$', re.ASCII),
tx.AliasedKey(['image', 'lang']):
t.String,
tx.AliasedKey(['group', 'groupName', 'group_name'], default='default'):
t.String,
tx.AliasedKey(['domain', 'domainName', 'domain_name'], default='default'):
t.String,
t.Key('config', default=dict):
t.Mapping(t.String, t.Any),
t.Key('tag', default=None):
t.Null | t.String,
}),
loads=_json_loads)
async def create(request: web.Request, params: Any) -> web.Response:
if params['domain'] is None:
params['domain'] = request['user']['domain_name']
""" director subsystem's configuration
- config-file schema
- settings
"""
import trafaret as T
APP_DIRECTOR_API_KEY = __name__ + ".director_api"
CONFIG_SECTION_NAME = "director"
# TODO: deprecate trafaret schema
schema = T.Dict(
{
T.Key("enabled", default=True, optional=True): T.Bool(),
T.Key(
"host",
default="director",
): T.String(),
T.Key("port", default=8001): T.ToInt(),
T.Key("version", default="v0"): T.Regexp(
regexp=r"^v\d+"
), # director API version basepath
}
)
keywords = (
t.Key('enum', optional=True, trafaret=t.List(t.Any) & (lambda consts: t.Or(*(t.Atom(cnst) for cnst in consts)))),
t.Key('const', optional=True, trafaret=t.Any() & then(t.Atom)),
t.Key('type', optional=True, trafaret=ensure_list(json_schema_type) & then(Any)),
# number validation
t.Key('multipleOf', optional=True, trafaret=t.Float(gt=0) & then(multipleOf)),
t.Key('maximum', optional=True, trafaret=t.Float() & (lambda maximum: t.Float(lte=maximum))),
t.Key('exclusiveMaximum', optional=True, trafaret=t.Float() & (lambda maximum: t.Float(lt=maximum))),
t.Key('minimum', optional=True, trafaret=t.Float() & (lambda minimum: t.Float(gte=minimum))),
t.Key('exclusiveMinimum', optional=True, trafaret=t.Float() & (lambda minimum: t.Float(gt=minimum))),
# string
t.Key('maxLength', optional=True, trafaret=t.Int(gte=0) & (lambda length: t.String(max_length=length))),
t.Key('minLength', optional=True, trafaret=t.Int(gte=0) & (lambda length: t.String(min_length=length))),
t.Key('pattern', optional=True, trafaret=Pattern() & (lambda pattern: t.Regexp(pattern))),
# array
t.Key('maxItems', optional=True, trafaret=t.Int(gte=0) & (lambda length: t.List(t.Any, max_length=length))),
t.Key('minItems', optional=True, trafaret=t.Int(gte=0) & (lambda length: t.List(t.Any, min_length=length))),
t.Key(
'uniqueItems',
optional=True,
trafaret=t.Bool() & (lambda need_check: t.List(t.Any) & uniq if need_check else t.Any)
),
# object
t.Key(
'maxProperties',
optional=True,
trafaret=(
query_domain_dotfiles,
verify_dotfile_name,
MAXIMUM_DOTFILE_SIZE,
)
log = BraceStyleAdapter(logging.getLogger('ai.backend.gateway.dotfile'))
@server_status_required(READ_ALLOWED)
@admin_required
@check_api_params(
t.Dict({
t.Key('domain'): t.String,
t.Key('data'): t.String(max_length=MAXIMUM_DOTFILE_SIZE),
t.Key('path'): t.String,
t.Key('permission'): t.Regexp(r'^[0-7]{3}$', re.ASCII),
}))
async def create(request: web.Request, params: Any) -> web.Response:
log.info('CREATE DOTFILE (domain: {0})', params['domain'])
if not request['is_superadmin'] and request['user'][
'domain_name'] != params['domain']:
raise GenericForbidden(
'Domain admins cannot create dotfiles of other domains')
dbpool = request.app['dbpool']
async with dbpool.acquire() as conn, conn.begin():
dotfiles, leftover_space = await query_domain_dotfiles(
conn, params['domain'])
if dotfiles is None:
raise DomainNotFound('Input domain is not found')
if leftover_space == 0:
def test_auto_call(self):
import functools
to_int_10000 = functools.partial(int, '10000')
trafaret = t.Regexp('2|10|16') & t.Int & to_int_10000
self.assertEqual(trafaret('10'), 10000)
def test_regexp(self):
trafaret = t.Regexp('cat')
assert trafaret('cat1212'), 'cat'
from servicelib.application_keys import APP_CONFIG_KEY, APP_CLIENT_SESSION_KEY
from yarl import URL
APP_DIRECTOR_API_KEY = __name__ + ".director_api"
CONFIG_SECTION_NAME = "director"
schema = T.Dict({
T.Key("enabled", default=True, optional=True): T.Bool(),
T.Key(
"host",
default="director",
): T.String(),
T.Key("port", default=8001): T.ToInt(),
T.Key("version", default="v0"):
T.Regexp(regexp=r"^v\d+"), # storage API version basepath
})
def build_api_url(config: Dict) -> URL:
api_baseurl = URL.build(scheme="http",
host=config["host"],
port=config["port"]).with_path(config["version"])
return api_baseurl
def get_config(app: web.Application) -> Dict:
return app[APP_CONFIG_KEY][CONFIG_SECTION_NAME]
def get_client_session(app: web.Application) -> ClientSession:
import logging
import os
import pathlib
import trafaret as t
import yaml
logger = logging.getLogger(__name__)
BASE_DIR = pathlib.Path(__file__).parent.parent
DEFAULT_CONFIG = 'config.yml'
CITIZENS_COLLECTION = 'requests_dump'
config_template = t.Dict({
t.Key('mongo'): t.Dict({
t.Key('host'): t.Regexp(regexp=r'^\d+.\d+.\d+.\d+$'),
t.Key('port'): t.Int(gte=0),
t.Key('database'): t.String(),
t.Key('max_pool_size'): t.Int(gte=0),
}),
t.Key('host'): t.Regexp(regexp=r'^\d+.\d+.\d+.\d+$'),
t.Key('port'): t.Int(gte=0),
t.Key('proxy-port'): t.Int(gte=0),
})
def get_config(mode=DEFAULT_CONFIG):
config_path = os.path.join(BASE_DIR, 'config', mode)
with open(config_path, 'rt') as f:
config = yaml.load(f, Loader=yaml.FullLoader)
def test_upper(self):
trafaret = t.Regexp(r'\w+-\w+') & str.upper
assert trafaret('abc-Abc') == 'ABC-ABC'
from servicelib.aiohttp.application_keys import APP_CONFIG_KEY
APP_DIRECTOR_API_KEY = __name__ + ".director_api"
CONFIG_SECTION_NAME = "director"
# TODO: deprecate trafaret schema
schema = T.Dict({
T.Key("enabled", default=True, optional=True): T.Bool(),
T.Key(
"host",
default="director",
): T.String(),
T.Key("port", default=8001): T.ToInt(),
T.Key("version", default="v0"):
T.Regexp(regexp=r"^v\d+"), # director API version basepath
})
class DirectorSettings(BaseSettings):
enabled: bool = True
host: str = "director"
port: PortInt = 8001
vtag: VersionTag = Field("v0",
alias="version",
description="Director service API's version tag")
url: Optional[AnyHttpUrl] = None
@validator("url", pre=True)
@classmethod
def test_callable(self):
import functools
to_int_10000 = functools.partial(int, '10000')
trafaret = t.Regexp('2|10|16') & t.ToInt & t.Call(to_int_10000)
assert trafaret('10') == 10000
from yarl import URL
APP_DIRECTOR_SESSION_KEY = __name__ + ".director_session"
APP_DIRECTOR_API_KEY = __name__ + ".director_api"
CONFIG_SECTION_NAME = 'director'
schema = T.Dict({
T.Key("enabled", default=True, optional=True): T.Bool(),
T.Key(
"host",
default="director",
): T.String(),
T.Key("port", default=8001): T.Int(),
T.Key("version", default="v0"):
T.Regexp(regexp=r'^v\d+') # storage API version basepath
})
def build_api_url(config: Dict) -> URL:
api_baseurl = URL.build(scheme='http',
host=config['host'],
port=config['port']).with_path(config["version"])
return api_baseurl
def get_config(app: web.Application) -> Dict:
return app[APP_CONFIG_KEY][CONFIG_SECTION_NAME]
def get_client_session(app: web.Application) -> ClientSession:
def test_regexp(self):
trafaret = t.Regexp('cat')
self.assertEqual(trafaret('cat1212'), 'cat')
CONFIG_FILENAME = 'batch_scoring.ini'
def verify_objectid(value):
"""Verify if id_ is a proper ObjectId. """
try:
t.Regexp(regexp='^[A-Fa-f0-9]{24}$').check(value)
except t.DataError:
raise ValueError('id {} not a valid project/model id'.format(value))
config_validator = t.Dict({
OptKey('host'): t.String,
OptKey('project_id'): t.Regexp(regexp='^[A-Fa-f0-9]{24}$'),
OptKey('model_id'): t.Regexp(regexp='^[A-Fa-f0-9]{24}$'),
OptKey('deployment_id'): t.Regexp(regexp='^[A-Fa-f0-9]{24}$'),
OptKey('import_id'): t.String,
OptKey('n_retry'): t.Int,
OptKey('keep_cols'): t.String,
OptKey('n_concurrent'): t.Int,
OptKey('dataset'): t.String,
OptKey('n_samples'): t.Int,
OptKey('delimiter'): t.String,
OptKey('out'): t.String,
OptKey('user'): t.String,
OptKey('password'): t.String,
OptKey('datarobot_key'): t.String,
OptKey('timeout'): t.Int,
OptKey('api_token'): t.String,
