def __init__(self,
*,
allow_blank=False,
regex=None,
choices=None,
min_length=None,
max_length=None,
**kwargs):
super(String, self).__init__(**kwargs)
if regex is not None:
self._trafaret = t.Regexp(regex)
else:
self._trafaret = t.String(allow_blank=allow_blank,
min_length=min_length,
max_length=max_length)
self.choices = None
if choices and is_collection(choices):
if isinstance(choices, type(Enum)):
self.choices = choices
self._trafaret &= t.Enum(*choices.__members__.keys())
else:
self._trafaret &= t.Enum(*choices)
if self.allow_none:
self._trafaret |= t.Null()
class Config(
namedtuple(
'BaseConfig',
'token, mongo, loglevel, sslchain, sslprivkey,reaction_threshold,sample_df'
)):
__slots__ = ()
mongo_uri_re = re.compile(
r'^(?:mongodb)://' # mongodb://
r'(?:\S+(?::\S*)?@)?' # user and password
r'(?:(?:[A-Z0-9](?:[A-Z0-9-_]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' # domain...
r'localhost|mongodb|' # localhost...
r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip
r'(?::\d+)?' # optional port
r'(?:/?|[/?]\S+)$',
re.IGNORECASE)
default = {
'token': 'randomtoken',
'mongo': {
'uri': 'mongodb://mongodb',
'db': 'talkbot'
},
'loglevel': 'DEBUG',
'reaction_threshold': 4,
'sample_df': "sample.csv",
}
trafaret = t.Dict({
'token':
t.String,
'mongo':
t.Dict({
'uri': t.String(regex=mongo_uri_re),
'db': t.String
}),
'loglevel':
t.Enum('DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'),
'sslchain':
FilePath(),
'sslprivkey':
FilePath(),
'sample_df':
FilePath(),
'reaction_threshold':
t.Int
})
@classmethod
def load_config(cls, dict_object):
default = cls.default.copy()
default.update(dict_object)
valid_conf = cls.trafaret.check(default)
Mongo = namedtuple('BaseMongoConfig', 'uri, db')
valid_conf['mongo'] = Mongo(**valid_conf['mongo'])
return cls(**valid_conf)
def to_dict(self):
return self._asdict()
def test_enum(self):
trafaret = t.Enum("foo", "bar", 1)
self.assertEqual(repr(trafaret), "<Enum('foo', 'bar', 1)>")
res = trafaret.check("foo")
res = trafaret.check(1)
res = extract_error(trafaret, 2)
self.assertEqual(res, "value doesn't match any variant")
def create_schema():
"""
Build schema for the configuration's file
by aggregating all the subsystem configurations
"""
schema = T.Dict({
"version":
T.String(),
"main":
T.Dict({
"host": T.IP,
"port": T.Int(),
"log_level": T.Enum(*logging._nameToLevel.keys()), # pylint: disable=protected-access
"enabled_development_mode": T.Bool(),
}),
rest_config.CONFIG_SECTION_NAME:
rest_config.schema,
## Add here more configurations
})
section_names = [k.name for k in schema.keys]
assert len(section_names) == len(set(
section_names)), "Found repeated section names in %s" % section_names
return schema
def document_schema():
choices = ['a', 'b', 'c']
schema = t.Dict({
t.Key('_id'):
MongoId,
t.Key('title'):
t.String(max_length=200),
t.Key('category'):
t.String(max_length=200),
t.Key('body'):
t.String,
t.Key('views'):
t.Int,
t.Key('average_note'):
t.Float,
# t.Key('pictures'): t.Dict({}).allow_extra('*'),
t.Key('published_at'):
DateTime,
# t.Key('tags'): t.List(t.Int),
t.Key('status'):
t.Enum(*choices),
t.Key('visible'):
t.StrBool,
})
return schema
class BugHistoryValidator(TrafaretValidator):
name = t.String(max_length=30)
description = t.String(max_length=240)
team_id = t.Int()
priority = t.Enum(*[e.name for e in Priority])
time_created = t.String()
time_closed = t.String()
def create_schema() -> T.Dict:
"""
Build schema for the configuration's file
by aggregating all the subsystem configurations
"""
# pylint: disable=protected-access
schema = T.Dict(
{
"version": T.String(),
"main": T.Dict(
{
"host": T.IP,
"port": T.ToInt(),
"client_outdir": T.String(),
"log_level": T.Enum(*logging._nameToLevel.keys()),
"testing": T.Bool(),
T.Key("studies_access_enabled", default=False): T.Or(
T.Bool(), T.ToInt
),
}
),
addon_section(tracing.tracing_section_name, optional=True): tracing.schema,
db_config.CONFIG_SECTION_NAME: db_config.schema,
director_config.CONFIG_SECTION_NAME: director_config.schema,
rest_config.CONFIG_SECTION_NAME: rest_config.schema,
projects_config.CONFIG_SECTION_NAME: projects_config.schema,
email_config.CONFIG_SECTION_NAME: email_config.schema,
storage_config.CONFIG_SECTION_NAME: storage_config.schema,
addon_section(
login_config.CONFIG_SECTION_NAME, optional=True
): login_config.schema,
addon_section(
socketio_config.CONFIG_SECTION_NAME, optional=True
): socketio_config.schema,
session_config.CONFIG_SECTION_NAME: session_config.schema,
activity_config.CONFIG_SECTION_NAME: activity_config.schema,
resource_manager_config.CONFIG_SECTION_NAME: resource_manager_config.schema,
# BELOW HERE minimal sections until more options are needed
addon_section("diagnostics", optional=True): minimal_addon_schema(),
addon_section("users", optional=True): minimal_addon_schema(),
addon_section("groups", optional=True): minimal_addon_schema(),
addon_section("tags", optional=True): minimal_addon_schema(),
addon_section("publications", optional=True): minimal_addon_schema(),
addon_section("catalog", optional=True): catalog_config.schema,
addon_section("products", optional=True): minimal_addon_schema(),
addon_section("computation", optional=True): minimal_addon_schema(),
addon_section("director-v2", optional=True): minimal_addon_schema(),
addon_section("studies_access", optional=True): minimal_addon_schema(),
addon_section("studies_dispatcher", optional=True): minimal_addon_schema(),
}
)
section_names = [k.name for k in schema.keys]
# fmt: off
assert len(section_names) == len(set(section_names)), f"Found repeated section names in {section_names}" # nosec
# fmt: on
return schema
class TeamReleaseStatusValidator(TrafaretValidator):
team_id = t.Int
release_id = t.Int
status = t.Enum(*[e.name for e in Status])
comment = t.String(250)
username = t.String
password = t.String
time_delay = t.Int
def find_objects_in_radius():
'''
Function for implementation API endpoint 'api/objects/find/in_radius'.
> Request:
- body:
:param nodes: list<str>,
Ids nodes
:param metrics: str
Type direction: 'to', 'from', 'to-from'
:param max_dist: float
> Response:
(Success)
- body:
<id_node>: list<<id_object>>
...
(Failed)
- body:
:param detail: str
status: int
'''
logger.setLevel(logging.INFO)
logger.info("Request on API Gateway 'api/objects/find/in_radius'")
# Validation of body request
validator = trafaret.Dict({
trafaret.Key('nodes'): trafaret.List(trafaret.String),
trafaret.Key('metrics'): trafaret.Enum("to", "from", "to-from"),
trafaret.Key('max_dist'): trafaret.Float
})
try:
validated_data = validator.check(request.json)
except trafaret.DataError:
return jsonify({'details': f"Error of body request: {trafaret.extract_error(validator, request.json)}"}), 400
# Getting info for graph
id_objects = _graph__get_all_id_objects()
id_nodes = _graph__get_id_nodes(validated_data['nodes'])
type_dir = {"to": 1, "from": 2, "to-from": 3}[validated_data['metrics']]
max_dist = validated_data['max_dist']
# Results: list<(int, list<int>)>. Array of pair (id_node, array id_object)
results = algorithmsWrapper.task_1_1_b(id_objects, id_nodes, type_dir, max_dist)
# Converting results from graph to real
with open(os.path.join(PATH_DATA, 'matching_from_graph.json'), 'r') as file:
matching_from_graph = ast.literal_eval(file.read())
response_data = {}
for result in results:
response_data[matching_from_graph[result[0]]] = list(matching_from_graph[id_object] for id_object in result[1])
return jsonify(response_data), 200
class ElasticacheValidator(BaseValidator):
_schema = tr.Dict({
tr.Key('region'): tr.Enum(*amazon_regions),
tr.Key('snapshot_id'): tr.String,
})
_schema_db = _schema + tr.Dict({tr.Key('database_id'): tr.String})
def create_validate(self, params):
self._schema_db(params)
def restore_validate(self, params):
self._schema_db(params)
def delete_validate(self, params):
self._schema(params)
def action_validate(self, choices, **kwargs):
action_schema = self.tr.Dict({
self.tr.Key('type'):
t.String,
self.tr.Key('action'):
t.Enum(*choices),
self.tr.Key('description', optional=True):
t.String(max_length=200),
self.tr.Key('parameters'):
t.Dict().allow_extra("*"),
self.tr.Key('filters', optional=True):
t.List(t.Dict().allow_extra("*"))
})
action_schema(kwargs)
def build_trafaret(sa_type, **kwargs):
if isinstance(sa_type, sa.sql.sqltypes.Enum):
trafaret = t.Enum(*sa_type.enums, **kwargs)
# check for Text should be before String
elif isinstance(sa_type, sa.sql.sqltypes.Text):
trafaret = t.String(**kwargs)
elif isinstance(sa_type, sa.sql.sqltypes.String):
trafaret = t.String(max_length=sa_type.length, **kwargs)
elif isinstance(sa_type, sa.sql.sqltypes.Integer):
trafaret = t.Int(**kwargs)
elif isinstance(sa_type, sa.sql.sqltypes.Float):
trafaret = t.Float(**kwargs)
elif isinstance(sa_type, sa.sql.sqltypes.DateTime):
trafaret = DateTime(**kwargs) # RFC3339
elif isinstance(sa_type, sa.sql.sqltypes.Date):
trafaret = DateTime(**kwargs) # RFC3339
elif isinstance(sa_type, sa.sql.sqltypes.Boolean):
trafaret = t.StrBool(**kwargs)
# Add PG related JSON and ARRAY
elif isinstance(sa_type, postgresql.JSON):
trafaret = AnyDict | t.List(AnyDict)
# Add PG related JSON and ARRAY
elif isinstance(sa_type, postgresql.ARRAY):
item_trafaret = build_trafaret(sa_type.item_type)
trafaret = t.List(item_trafaret)
elif isinstance(sa_type, postgresql.INET):
# TODO: depend on "ipaddress" module?
trafaret = t.String(**kwargs)
else:
type_ = str(sa_type)
msg = 'Validator for type {} not implemented'.format(type_)
raise NotImplementedError(msg)
return trafaret
def create_schema():
"""
Build schema for the configuration's file
by aggregating all the subsystem configurations
"""
schema = T.Dict({
"version":
T.String(),
"main":
T.Dict({
"host": T.IP,
"port": T.Int(),
"client_outdir": T.String(),
"log_level": T.Enum(*logging._nameToLevel.keys()), # pylint: disable=protected-access
"testing": T.Bool(),
}),
db_config.CONFIG_SECTION_NAME:
db_config.schema,
director_config.CONFIG_SECTION_NAME:
director_config.schema,
rest_config.CONFIG_SECTION_NAME:
rest_config.schema,
projects_config.CONFIG_SECTION_NAME:
projects_config.schema,
email_config.CONFIG_SECTION_NAME:
email_config.schema,
computation_config.CONFIG_SECTION_NAME:
computation_config.schema,
storage_config.CONFIG_SECTION_NAME:
storage_config.schema,
T.Key(login_config.CONFIG_SECTION_NAME, optional=True):
login_config.schema
#s3_config.CONFIG_SECTION_NAME: s3_config.schema
#TODO: enable when sockets are refactored
})
section_names = [k.name for k in schema.keys]
assert len(section_names) == len(set(
section_names)), "Found repeated section names in %s" % section_names
return schema
def filters_validate(self, **kwargs):
regex_filter_schema = t.Dict({
t.Key('pattern'): t.String,
t.Key('type'): t.String
})
age_filter_schema = t.Dict({
t.Key('term'):
t.String,
t.Key('type'):
t.String,
t.Key('unit'):
t.Enum(*['hours', 'days', 'weeks', 'months']),
t.Key('count'):
t.Int
})
checks = {'regex': regex_filter_schema, 'age': age_filter_schema}
for filter in kwargs['filters']:
checks[filter['type']](filter)
def test_repr(self):
trafaret = t.Enum("foo", "bar", 1)
assert repr(trafaret), "<Enum('foo', 'bar', 1)>"
from .types import VolumeInfo
_max_cpu_count = os.cpu_count()
local_config_iv = t.Dict({
t.Key('storage-proxy'):
t.Dict({
t.Key('node-id'):
t.String,
t.Key('num-proc', default=_max_cpu_count):
t.Int[1:_max_cpu_count],
t.Key('pid-file', default=os.devnull):
tx.Path(type='file', allow_nonexisting=True, allow_devnull=True),
t.Key('event-loop', default='asyncio'):
t.Enum('asyncio', 'uvloop'),
t.Key('scandir-limit', default=1000):
t.Int[0:],
t.Key('max-upload-size', default="100g"):
tx.BinarySize,
t.Key('secret'):
t.String, # used to generate JWT tokens
t.Key('session-expire'):
tx.TimeDuration,
}),
t.Key('logging'):
logging_config_iv,
t.Key('api'):
t.Dict({
t.Key('client'):
t.Dict({
def test_enum(self):
trafaret = t.Enum("foo", "bar", 1)
trafaret.check("foo")
trafaret.check(1)
res = extract_error(trafaret, 2)
assert res == "value doesn't match any variant"
]
})
@server_status_required(ALL_ALLOWED)
@admin_required
@check_api_params(
t.Dict({
t.Key('src'): t.String,
t.Key('target'): t.String,
t.Key('launchOptions', default={}): t.Dict({
t.Key('scalingGroup', default='default'): t.String,
t.Key('group', default='default'): t.String,
}).allow_extra('*'),
t.Key('brand'): t.String,
t.Key('baseDistro'): t.Enum('ubuntu', 'centos'),
t.Key('minCPU', default=1): t.Int[1:],
t.Key('minMemory', default='64m'): tx.BinarySize,
t.Key('preferredSharedMemory', default='64m'): tx.BinarySize,
t.Key('supportedAccelerators'): t.List(t.String),
t.Key('runtimeType'): t.Enum('python'),
t.Key('runtimePath'): tx.Path(type='file', allow_nonexisting=True, resolve=False),
t.Key('CPUCountEnvs'): t.List(t.String),
t.Key('servicePorts', default=[]): t.List(t.Dict({
t.Key('name'): t.String,
t.Key('protocol'): t.Enum('http', 'tcp', 'pty'),
t.Key('ports'): t.List(t.Int[1:65535], min_length=1),
})),
}).allow_extra('*'))
async def import_image(request: web.Request, params: Any) -> web.Response:
'''
class ImageFeaturizer:
"""
This object can load images, rescale, crop, and vectorize them into a
uniform batch, and then featurize the images for use with custom classifiers.
Methods
------------------
__init__(depth, auto_sample,
downsample_size):
--------------------------------
Initialize the ImageFeaturizer. Build the featurizer model with the
depth and feature downsampling specified by the inputs.
load_and_featurize_data(image_column_headers, image_path,
csv_path, new_csv_name, scaled_size, grayscale):
--------------------------------
Loads image directory and/or csv into the model, and
featurizes the images
load_data(image_column_headers, image_path, csv_path,
new_csv_name, scaled_size, grayscale):
--------------------------------
Loads image directory and/or csv into the model, and vectorize the
images for input into the featurizer
featurize():
--------------------------------
Featurize the loaded data, append the features to the csv, and
return the full dataframe
"""
@t.guard(depth=t.Int(gte=1, lte=4),
auto_sample=t.Bool,
downsample_size=t.Int(gte=0),
model=t.Enum(*supported_model_types.keys()))
def __init__(self,
depth=1,
auto_sample=False,
downsample_size=0,
model='squeezenet'
):
"""
Initializer.
Loads an initial InceptionV3 pretrained network, decapitates it and
downsamples according to user specifications.
Parameters:
----------
depth : int
How deep to decapitate the model. Deeper means less specific but
also less complex
auto_sample : bool
If True, feature layer is automatically downsampled to the right size.
downsample_size: int
The number of features to downsample the featurizer to
Returns:
--------
None. Initializes and saves the featurizer object attributes.
"""
# BUILDING THE MODEL #
logging.info("Building the featurizer.")
featurizer = build_featurizer(depth, auto_sample,
downsample_size, model_str=model.lower())
# Saving initializations of model
self.depth = depth
self.auto_sample = auto_sample
self.downsample_size = downsample_size
self.num_features = featurizer.layers[-1].output_shape[-1]
# Save the model
self.model_name = model.lower()
self.featurizer = featurizer
self.visualize = featurizer.summary
# Initializing preprocessing variables for after we load and featurize the images
self.data = np.zeros((1))
self.features = np.zeros((1))
self.full_dataframe = pd.DataFrame()
self.csv_path = ''
self.image_list = ''
self.image_column_headers = ''
self.image_path = ''
# Image scaling and cropping
self.scaled_size = (0, 0)
self.crop_size = (0, 0)
self.number_crops = 0
self.isotropic_scaling = False
def load_and_featurize_data(self,
image_column_headers,
image_path='',
csv_path='',
new_csv_name='featurizer_csv/generated_images_csv',
grayscale=False,
save_features=False,
omit_time=False,
omit_model=False,
omit_depth=False,
omit_output=False
# crop_size = (299, 299),
# number_crops = 0,
# random_crop = False,
# isotropic_scaling = True
):
"""
Load image directory and/or csv, and vectorize the images for input into the featurizer.
Then, featurize the data.
Parameters:
----------
image_column_headers : str
the name of the column holding the image data, if a csv exists,
or what the name of the column will be, if generating the csv
from a directory
image_path : str
the path to the folder containing the images. If using URLs, leave blank
csv_path : str
the path to the csv. If just using a directory, leave blank, and
specify the path for the generated csv in new_csv_name.
If csv exists, this is the path where the featurized csv will be
generated.
new_csv_name : str
the path to the new csv, if one is being generated from a directory.
If no csv exists, this is the path where the featurized csv will
be generated
grayscale : bool
Decides if image is grayscale or not. May get deprecated. Don't
think it works on the InceptionV3 model due to input size.
### These features haven't been implemented yet.
# isotropic_scaling : bool
# if True, images are scaled keeping proportions and then cropped
#
# crop_size: tuple
# if the image gets cropped, decides the size of the crop
#
# random_crop: bool
# If False, only take the center crop. If True, take random crop
#
Returns:
--------
full_dataframe :
Dataframe containing the features appended to the original csv.
Also writes csvs containing the features only and the full dataframe
to the same path as the csv containing the list of names
"""
self.load_data(image_column_headers, image_path, csv_path, new_csv_name, grayscale)
return self.featurize(save_features=save_features, omit_time=omit_time,
omit_model=omit_model, omit_depth=omit_depth, omit_output=omit_output)
def load_data(self,
image_column_headers,
image_path='',
csv_path='',
new_csv_name='featurizer_csv/generated_images_csv',
batch_size=1000,
grayscale=False
# crop_size = (299, 299),
# number_crops = 0,
# random_crop = False,
# isotropic_scaling = True
):
"""
Load image directory and/or csv, and vectorize the images for input into the featurizer.
Parameters:
----------
image_column_headers : str
the name of the column holding the image data, if a csv exists,
or what the name of the column will be, if generating the csv
from a directory
image_path : str
the path to the folder containing the images. If using URLs, leave blank
csv_path : str
the path to the csv. If just using a directory, leave blank, and
specify the path for the generated csv in new_csv_name.
If csv exists, this is the path where the featurized csv will be
generated.
new_csv_name : str
the path to the new csv, if one is being generated from a directory.
If no csv exists, this is the path where the featurized csv will
be generated
grayscale : bool
Decides if image is grayscale or not. May get deprecated. Don't
think it works on the InceptionV3 model due to input size.
### These features haven't been implemented yet.
# isotropic_scaling : bool
# if True, images are scaled keeping proportions and then cropped
#
# crop_size: tuple
# if the image gets cropped, decides the size of the crop
#
# random_crop: bool
# If False, only take the center crop. If True, take random crop
#
"""
size_dict = {'squeezenet': (227, 227), 'vgg16': (224, 224), 'vgg19': (224, 224),
'resnet50': (224, 224), 'inceptionv3': (299, 299), 'xception': (299, 299)}
scaled_size = size_dict[self.model_name]
# Convert column header to list if it's passed a single string
if isinstance(image_column_headers, str):
image_column_headers = [image_column_headers]
# If new csv_path is being generated, make sure the folder exists.
if (csv_path == ''):
# Raise error if multiple image columns are passed in without a csv
if len(image_column_headers) > 1:
raise ValueError('If building the csv from a directory, featurizer can only '
'create a single image column. If two image columns are needed, '
'please create a csv to pass in.')
# Create the filepath to the new csv
path_to_new_csv = os.path.dirname(new_csv_name)
if not os.path.isdir(path_to_new_csv) and path_to_new_csv != '':
os.makedirs(os.path.dirname(new_csv_name))
# Add backslash to end of image path if it is not there
if image_path != '' and image_path[-1] != "/":
image_path = '{}/'.format(image_path)
# Save the full image tensor, the path to the csv, and the list of image paths
(image_data, csv_path, list_of_image_paths) = \
preprocess_data(image_column_headers[0], self.model_name, image_path, csv_path,
new_csv_name, scaled_size, grayscale)
full_image_list = [list_of_image_paths]
full_image_data = np.expand_dims(image_data, axis=0)
if len(image_column_headers) > 1:
for column in image_column_headers[1:]:
(image_data, csv_path, list_of_image_paths) = \
preprocess_data(column, self.model_name, image_path, csv_path,
new_csv_name, scaled_size, grayscale)
full_image_data = np.concatenate((full_image_data,
np.expand_dims(image_data, axis=0)))
full_image_list.append(list_of_image_paths)
# Save all of the necessary data to the featurizer
self.data = full_image_data
self.csv_path = csv_path
self.image_list = full_image_list
self.image_column_headers = image_column_headers
self.scaled_size = scaled_size
self.image_path = image_path
@t.guard(save_features=t.Bool, omit_time=t.Bool, omit_model=t.Bool,
omit_depth=t.Bool, omit_output=t.Bool)
def featurize(self, save_features=False, omit_time=False, omit_model=False,
omit_depth=False, omit_output=False):
"""
Featurize the loaded data, returning the dataframe and writing the features
and the full combined data to csv
Parameters
----------
None, just operates on the loaded data
Returns
-------
full_dataframe : pandas.DataFrame
Dataframe containing the features appended to the original csv.
Also writes csvs containing the features only and the full dataframe
to the same path as the csv containing the list of names
"""
# Check data has been loaded, and that the data was vectorized correctly
if np.array_equal(self.data, np.zeros((1))):
raise IOError('Must load data into the model first. Call load_data.')
assert len(self.image_column_headers) == self.data.shape[0]
logging.info("Trying to featurize data.")
# Initialize featurized data vector with appropriate size
self.features = np.zeros((self.data.shape[1],
self.num_features * len(self.image_column_headers)))
# Save csv_names
csv_name, ext = os.path.splitext(self.csv_path)
# For each image column, perform the full featurization and add the features to the csv
for column in range(self.data.shape[0]):
# Create the correct csv path if we have multiple image columns
if column == 0:
csv_path = "{}{}".format(csv_name, ext)
else:
named_path = _named_path_finder(csv_name, self.model_name, self.depth,
self.num_features, omit_model, omit_depth,
omit_output, omit_time)
# Save the name and extension separately, for robust naming
csv_path = '{}_full{}'.format(named_path, ext)
# Featurize the data, and save it to the appropriate columns
self.features[:,
self.num_features * column:self.num_features * column +
self.num_features] \
= partial_features = featurize_data(self.featurizer, self.data[column])
# Save the full dataframe to the csv
full_dataframe = _features_to_csv(self.data[column], partial_features, csv_path,
self.image_column_headers[column], self.image_list,
model_str=self.model_name, model_depth=self.depth,
model_output=self.num_features,
omit_model=omit_model, omit_time=omit_time,
omit_depth=omit_depth, omit_output=omit_output,
save_features=save_features, continued_column=column)
self.full_dataframe = full_dataframe
return full_dataframe
OptKey('le'): SimpleType,
OptKey('ne'): SimpleType,
OptKey('eq'): SimpleType,
OptKey('like'): SimpleType,
})
ASC = 'ASC'
DESC = 'DESC'
ListQuery = t.Dict({
OptKey('_page', default=1): t.Int[1:],
OptKey('_perPage', default=30): t.Int[1:],
OptKey('_sortField'): t.String,
OptKey('_sortDir', default=DESC): t.Enum(DESC, ASC),
OptKey('_filters'): t.Mapping(t.String, Filter | SimpleType)
})
LoginForm = t.Dict({
"username": t.String,
"password": t.String,
})
def validate_query_structure(query):
"""Validate query arguments in list request.
:param query: mapping with pagination and filtering information
"""
class AccountValidator(TrafaretValidator):
name = t.String(max_length=50)
nickname = t.String(max_length=20)
password = t.String(max_length=20)
role = t.Enum(*[e.name for e in Role])
""" rest subsystem's configuration
- constants
- config-file schema
"""
import trafaret as T
from servicelib.application_keys import APP_OPENAPI_SPECS_KEY
APP_OPENAPI_SPECS_KEY = APP_OPENAPI_SPECS_KEY # pylint: disable=self-assigning-variable,bad-option-value
CONFIG_SECTION_NAME = 'rest'
schema = T.Dict({
"version": T.Enum("{{cookiecutter.openapi_specs_version}}"),
"location": T.Or(T.String,
T.URL), # either path or url should contain version in it
})
import trafaret as T
from .rest_config import schema as rest_schema
app_schema = T.Dict({
T.Key("host", default="0.0.0.0"):
T.IP,
"port":
T.Int(),
"log_level":
T.Enum("DEBUG", "WARNING", "INFO", "ERROR", "CRITICAL", "FATAL", "NOTSET"),
"watched_git_repositories":
T.List(
T.Dict({
"id":
T.String(),
"url":
T.URL,
T.Key("username", optional=True, default=""):
T.Or(T.String(allow_blank=True), T.Null),
T.Key("password", optional=True, default=""):
T.Or(T.String(allow_blank=True), T.Null),
T.Key("branch", default="master", optional=True):
T.String(allow_blank=True),
T.Key("tags", default="", optional=True):
T.String(allow_blank=True),
"pull_only_files":
T.Bool(),
"paths":
T.List(T.String()),
}),
@server_status_required(READ_ALLOWED)
@auth_required
@check_api_params(
t.Dict({
tx.AliasedKey(['name', 'sessionName'], default='*') >> 'session_name':
t.String,
t.Key('ownerAccessKey', default=None) >> 'owner_access_key':
t.Null | t.String,
t.Key('sessionId', default=None) >> 'session_id':
t.Null | tx.UUID,
# NOTE: if set, sessionId overrides sessionName and ownerAccessKey parameters.
tx.AliasedKey(['group', 'groupName'], default='*') >> 'group_name':
t.String,
t.Key('scope', default='*'):
t.Enum('*', 'session', 'kernel'),
}))
@adefer
async def push_session_events(
defer,
request: web.Request,
params: Mapping[str, Any],
) -> web.StreamResponse:
app = request.app
session_name = params['session_name']
session_id = params['session_id']
scope = params['scope']
user_role = request['user']['role']
user_uuid = request['user']['uuid']
access_key = params['owner_access_key']
if access_key is None:
EVENT_SUGGESTION_TRAFARET = t.Dict({
'title':
t.String,
'agenda':
t.String,
'social':
t.String(allow_blank=True) | t.Null,
'place':
t.String(allow_blank=True),
'registration_url':
t.URL(allow_blank=True) | t.Null,
'image_url':
t.URL(allow_blank=True) | t.String(max_length=0, allow_blank=True),
'level':
t.Enum('NONE', 'TRAINEE', 'JUNIOR', 'MIDDLE', 'SENIOR'),
'when_start':
t.String,
'when_end': (t.String(allow_blank=True) >>
(lambda x: None if not x else x)) | t.Null,
'only_date':
t.StrBool(),
'team':
t.String() >> get_team_by_name,
'submitter_email':
t.Email(),
'secret':
t.String
}).make_optional('secret', 'social')
import trafaret as t
STORE_LOCATION_SCHEMA = t.Enum('S3', 'gcs', 'azure', 'rackspace', 'dropbox')
def validate_upload_tags(d):
t.List(t.String, max_length=10).check(list(d.keys()))
t.Mapping(t.String(max_length=128), t.String(max_length=256)).check(d)
return d
STORE_SCHEMA = t.Dict(
t.Key('filename', optional=True, trafaret=t.String),
t.Key('mimetype', optional=True, trafaret=t.String),
t.Key('location', optional=True, trafaret=t.String),
t.Key('path', optional=True, trafaret=t.String),
t.Key('container', optional=True, trafaret=t.String),
t.Key('region', optional=True, trafaret=t.String),
t.Key('access', optional=True, trafaret=t.String),
t.Key('base64decode', optional=True, trafaret=t.Bool),
t.Key('workflows', optional=True, trafaret=t.List(t.String)),
t.Key('upload_tags', optional=True, trafaret=validate_upload_tags),
)
},
'xception': {
'label': 'Xception',
'class': Xception,
'kwargs': {},
'depth': {
1: 1,
2: 8,
3: 18,
4: 28
}
}
}
@t.guard(model_str=t.Enum(*supported_model_types.keys()),
loaded_weights=t.String(allow_blank=True))
def _initialize_model(model_str, loaded_weights=''):
"""
Initialize the InceptionV3 model with the saved weights, or
if the weight file can't be found, load them automatically through Keras.
Parameters:
----------
model_str : str
String deciding which model to use for the featurizer
Returns:
-------
model : keras.models.Model
The initialized model loaded with pre-trained weights
"""
logging.info('Loading/downloading {model_label} model weights. '
def main(cli_ctx, config_path, debug):
volume_config_iv = t.Dict({
t.Key('etcd'):
t.Dict({
t.Key('namespace'): t.String,
t.Key('addr'): tx.HostPortPair(allow_blank_host=False)
}).allow_extra('*'),
t.Key('logging'):
t.Any, # checked in ai.backend.common.logging
t.Key('agent'):
t.Dict({
t.Key('mode'): t.Enum('scratch', 'vfolder'),
t.Key('rpc-listen-addr'): tx.HostPortPair(allow_blank_host=True),
t.Key('user-uid'): t.Int,
t.Key('user-gid'): t.Int
}),
t.Key('storage'):
t.Dict({
t.Key('mode'): t.Enum('xfs', 'btrfs'),
t.Key('path'): t.String
})
}).allow_extra('*')
# Determine where to read configuration.
raw_cfg, cfg_src_path = config.read_from_file(config_path, 'agent')
config.override_with_env(raw_cfg, ('etcd', 'namespace'),
'BACKEND_NAMESPACE')
config.override_with_env(raw_cfg, ('etcd', 'addr'), 'BACKEND_ETCD_ADDR')
config.override_with_env(raw_cfg, ('etcd', 'user'), 'BACKEND_ETCD_USER')
config.override_with_env(raw_cfg, ('etcd', 'password'),
'BACKEND_ETCD_PASSWORD')
config.override_with_env(raw_cfg, ('agent', 'rpc-listen-addr', 'host'),
'BACKEND_AGENT_HOST_OVERRIDE')
config.override_with_env(raw_cfg, ('agent', 'rpc-listen-addr', 'port'),
'BACKEND_AGENT_PORT')
if debug:
config.override_key(raw_cfg, ('debug', 'enabled'), True)
config.override_key(raw_cfg, ('logging', 'level'), 'DEBUG')
config.override_key(raw_cfg, ('logging', 'pkg-ns', 'ai.backend'),
'DEBUG')
try:
cfg = config.check(raw_cfg, volume_config_iv)
cfg['_src'] = cfg_src_path
except config.ConfigurationError as e:
print(
'ConfigurationError: Validation of agent configuration has failed:',
file=sys.stderr)
print(pformat(e.invalid_data), file=sys.stderr)
raise click.Abort()
rpc_host = cfg['agent']['rpc-listen-addr'].host
if (isinstance(rpc_host, BaseIPAddress)
and (rpc_host.is_unspecified or rpc_host.is_link_local)):
print(
'ConfigurationError: '
'Cannot use link-local or unspecified IP address as the RPC listening host.',
file=sys.stderr)
raise click.Abort()
if os.getuid() != 0:
print('Storage agent can only be run as root', file=sys.stderr)
raise click.Abort()
if cli_ctx.invoked_subcommand is None:
setproctitle('Backend.AI: Storage Agent')
logger = Logger(cfg['logging'])
with logger:
log.info('Backend.AI Storage Agent', VERSION)
log_config = logging.getLogger('ai.backend.agent.config')
if debug:
log_config.debug('debug mode enabled.')
if 'debug' in cfg and cfg['debug']['enabled']:
print('== Agent configuration ==')
pprint(cfg)
aiotools.start_server(server_main,
num_workers=1,
use_threading=True,
args=(cfg, ))
log.info('exit.')
return 0
class ProductId(TrafaretPoweredAttribute):
trafaret = trafaret.Enum('F171', 'F172', 'F173')
if row is None:
raise GenericNotFound('No such user group or '
'you are not the member of the group.')
group_role = 'user'
resp_data = {
'global_role': 'superadmin' if request['is_superadmin'] else 'user',
'domain_role': 'admin' if request['is_admin'] else 'user',
'group_role': group_role,
}
return web.json_response(resp_data)
@atomic
@check_api_params(
t.Dict({
t.Key('type'): t.Enum('keypair', 'jwt'),
t.Key('domain'): t.String,
t.Key('username'): t.String,
t.Key('password'): t.String,
}))
async def authorize(request: web.Request, params: Any) -> web.Response:
if params['type'] != 'keypair':
# other types are not implemented yet.
raise InvalidAPIParameters('Unsupported authorization type')
log.info(
'AUTH.AUTHORIZE(d:{0[domain]}, u:{0[username]}, passwd:****, type:{0[type]})',
params)
dbpool = request.app['dbpool']
# [Hooking point for AUTHORIZE with the FIRST_COMPLETED requirement]
# The hook handlers should accept the whole ``params`` dict, and optional
