def __init__(self, *, allow_blank=False, regex=None, choices=None, min_length=None, max_length=None, **kwargs): super(String, self).__init__(**kwargs) if regex is not None: self._trafaret = t.Regexp(regex) else: self._trafaret = t.String(allow_blank=allow_blank, min_length=min_length, max_length=max_length) self.choices = None if choices and is_collection(choices): if isinstance(choices, type(Enum)): self.choices = choices self._trafaret &= t.Enum(*choices.__members__.keys()) else: self._trafaret &= t.Enum(*choices) if self.allow_none: self._trafaret |= t.Null()
class Config( namedtuple( 'BaseConfig', 'token, mongo, loglevel, sslchain, sslprivkey,reaction_threshold,sample_df' )): __slots__ = () mongo_uri_re = re.compile( r'^(?:mongodb)://' # mongodb:// r'(?:\S+(?::\S*)?@)?' # user and password r'(?:(?:[A-Z0-9](?:[A-Z0-9-_]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' # domain... r'localhost|mongodb|' # localhost... r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip r'(?::\d+)?' # optional port r'(?:/?|[/?]\S+)$', re.IGNORECASE) default = { 'token': 'randomtoken', 'mongo': { 'uri': 'mongodb://mongodb', 'db': 'talkbot' }, 'loglevel': 'DEBUG', 'reaction_threshold': 4, 'sample_df': "sample.csv", } trafaret = t.Dict({ 'token': t.String, 'mongo': t.Dict({ 'uri': t.String(regex=mongo_uri_re), 'db': t.String }), 'loglevel': t.Enum('DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'), 'sslchain': FilePath(), 'sslprivkey': FilePath(), 'sample_df': FilePath(), 'reaction_threshold': t.Int }) @classmethod def load_config(cls, dict_object): default = cls.default.copy() default.update(dict_object) valid_conf = cls.trafaret.check(default) Mongo = namedtuple('BaseMongoConfig', 'uri, db') valid_conf['mongo'] = Mongo(**valid_conf['mongo']) return cls(**valid_conf) def to_dict(self): return self._asdict()
def test_enum(self): trafaret = t.Enum("foo", "bar", 1) self.assertEqual(repr(trafaret), "<Enum('foo', 'bar', 1)>") res = trafaret.check("foo") res = trafaret.check(1) res = extract_error(trafaret, 2) self.assertEqual(res, "value doesn't match any variant")
def create_schema(): """ Build schema for the configuration's file by aggregating all the subsystem configurations """ schema = T.Dict({ "version": T.String(), "main": T.Dict({ "host": T.IP, "port": T.Int(), "log_level": T.Enum(*logging._nameToLevel.keys()), # pylint: disable=protected-access "enabled_development_mode": T.Bool(), }), rest_config.CONFIG_SECTION_NAME: rest_config.schema, ## Add here more configurations }) section_names = [k.name for k in schema.keys] assert len(section_names) == len(set( section_names)), "Found repeated section names in %s" % section_names return schema
def document_schema(): choices = ['a', 'b', 'c'] schema = t.Dict({ t.Key('_id'): MongoId, t.Key('title'): t.String(max_length=200), t.Key('category'): t.String(max_length=200), t.Key('body'): t.String, t.Key('views'): t.Int, t.Key('average_note'): t.Float, # t.Key('pictures'): t.Dict({}).allow_extra('*'), t.Key('published_at'): DateTime, # t.Key('tags'): t.List(t.Int), t.Key('status'): t.Enum(*choices), t.Key('visible'): t.StrBool, }) return schema
class BugHistoryValidator(TrafaretValidator): name = t.String(max_length=30) description = t.String(max_length=240) team_id = t.Int() priority = t.Enum(*[e.name for e in Priority]) time_created = t.String() time_closed = t.String()
def create_schema() -> T.Dict: """ Build schema for the configuration's file by aggregating all the subsystem configurations """ # pylint: disable=protected-access schema = T.Dict( { "version": T.String(), "main": T.Dict( { "host": T.IP, "port": T.ToInt(), "client_outdir": T.String(), "log_level": T.Enum(*logging._nameToLevel.keys()), "testing": T.Bool(), T.Key("studies_access_enabled", default=False): T.Or( T.Bool(), T.ToInt ), } ), addon_section(tracing.tracing_section_name, optional=True): tracing.schema, db_config.CONFIG_SECTION_NAME: db_config.schema, director_config.CONFIG_SECTION_NAME: director_config.schema, rest_config.CONFIG_SECTION_NAME: rest_config.schema, projects_config.CONFIG_SECTION_NAME: projects_config.schema, email_config.CONFIG_SECTION_NAME: email_config.schema, storage_config.CONFIG_SECTION_NAME: storage_config.schema, addon_section( login_config.CONFIG_SECTION_NAME, optional=True ): login_config.schema, addon_section( socketio_config.CONFIG_SECTION_NAME, optional=True ): socketio_config.schema, session_config.CONFIG_SECTION_NAME: session_config.schema, activity_config.CONFIG_SECTION_NAME: activity_config.schema, resource_manager_config.CONFIG_SECTION_NAME: resource_manager_config.schema, # BELOW HERE minimal sections until more options are needed addon_section("diagnostics", optional=True): minimal_addon_schema(), addon_section("users", optional=True): minimal_addon_schema(), addon_section("groups", optional=True): minimal_addon_schema(), addon_section("tags", optional=True): minimal_addon_schema(), addon_section("publications", optional=True): minimal_addon_schema(), addon_section("catalog", optional=True): catalog_config.schema, addon_section("products", optional=True): minimal_addon_schema(), addon_section("computation", optional=True): minimal_addon_schema(), addon_section("director-v2", optional=True): minimal_addon_schema(), addon_section("studies_access", optional=True): minimal_addon_schema(), addon_section("studies_dispatcher", optional=True): minimal_addon_schema(), } ) section_names = [k.name for k in schema.keys] # fmt: off assert len(section_names) == len(set(section_names)), f"Found repeated section names in {section_names}" # nosec # fmt: on return schema
class TeamReleaseStatusValidator(TrafaretValidator): team_id = t.Int release_id = t.Int status = t.Enum(*[e.name for e in Status]) comment = t.String(250) username = t.String password = t.String time_delay = t.Int
def find_objects_in_radius(): ''' Function for implementation API endpoint 'api/objects/find/in_radius'. > Request: - body: :param nodes: list<str>, Ids nodes :param metrics: str Type direction: 'to', 'from', 'to-from' :param max_dist: float > Response: (Success) - body: <id_node>: list<<id_object>> ... (Failed) - body: :param detail: str status: int ''' logger.setLevel(logging.INFO) logger.info("Request on API Gateway 'api/objects/find/in_radius'") # Validation of body request validator = trafaret.Dict({ trafaret.Key('nodes'): trafaret.List(trafaret.String), trafaret.Key('metrics'): trafaret.Enum("to", "from", "to-from"), trafaret.Key('max_dist'): trafaret.Float }) try: validated_data = validator.check(request.json) except trafaret.DataError: return jsonify({'details': f"Error of body request: {trafaret.extract_error(validator, request.json)}"}), 400 # Getting info for graph id_objects = _graph__get_all_id_objects() id_nodes = _graph__get_id_nodes(validated_data['nodes']) type_dir = {"to": 1, "from": 2, "to-from": 3}[validated_data['metrics']] max_dist = validated_data['max_dist'] # Results: list<(int, list<int>)>. Array of pair (id_node, array id_object) results = algorithmsWrapper.task_1_1_b(id_objects, id_nodes, type_dir, max_dist) # Converting results from graph to real with open(os.path.join(PATH_DATA, 'matching_from_graph.json'), 'r') as file: matching_from_graph = ast.literal_eval(file.read()) response_data = {} for result in results: response_data[matching_from_graph[result[0]]] = list(matching_from_graph[id_object] for id_object in result[1]) return jsonify(response_data), 200
class ElasticacheValidator(BaseValidator): _schema = tr.Dict({ tr.Key('region'): tr.Enum(*amazon_regions), tr.Key('snapshot_id'): tr.String, }) _schema_db = _schema + tr.Dict({tr.Key('database_id'): tr.String}) def create_validate(self, params): self._schema_db(params) def restore_validate(self, params): self._schema_db(params) def delete_validate(self, params): self._schema(params)
def action_validate(self, choices, **kwargs): action_schema = self.tr.Dict({ self.tr.Key('type'): t.String, self.tr.Key('action'): t.Enum(*choices), self.tr.Key('description', optional=True): t.String(max_length=200), self.tr.Key('parameters'): t.Dict().allow_extra("*"), self.tr.Key('filters', optional=True): t.List(t.Dict().allow_extra("*")) }) action_schema(kwargs)
def build_trafaret(sa_type, **kwargs): if isinstance(sa_type, sa.sql.sqltypes.Enum): trafaret = t.Enum(*sa_type.enums, **kwargs) # check for Text should be before String elif isinstance(sa_type, sa.sql.sqltypes.Text): trafaret = t.String(**kwargs) elif isinstance(sa_type, sa.sql.sqltypes.String): trafaret = t.String(max_length=sa_type.length, **kwargs) elif isinstance(sa_type, sa.sql.sqltypes.Integer): trafaret = t.Int(**kwargs) elif isinstance(sa_type, sa.sql.sqltypes.Float): trafaret = t.Float(**kwargs) elif isinstance(sa_type, sa.sql.sqltypes.DateTime): trafaret = DateTime(**kwargs) # RFC3339 elif isinstance(sa_type, sa.sql.sqltypes.Date): trafaret = DateTime(**kwargs) # RFC3339 elif isinstance(sa_type, sa.sql.sqltypes.Boolean): trafaret = t.StrBool(**kwargs) # Add PG related JSON and ARRAY elif isinstance(sa_type, postgresql.JSON): trafaret = AnyDict | t.List(AnyDict) # Add PG related JSON and ARRAY elif isinstance(sa_type, postgresql.ARRAY): item_trafaret = build_trafaret(sa_type.item_type) trafaret = t.List(item_trafaret) elif isinstance(sa_type, postgresql.INET): # TODO: depend on "ipaddress" module? trafaret = t.String(**kwargs) else: type_ = str(sa_type) msg = 'Validator for type {} not implemented'.format(type_) raise NotImplementedError(msg) return trafaret
def create_schema(): """ Build schema for the configuration's file by aggregating all the subsystem configurations """ schema = T.Dict({ "version": T.String(), "main": T.Dict({ "host": T.IP, "port": T.Int(), "client_outdir": T.String(), "log_level": T.Enum(*logging._nameToLevel.keys()), # pylint: disable=protected-access "testing": T.Bool(), }), db_config.CONFIG_SECTION_NAME: db_config.schema, director_config.CONFIG_SECTION_NAME: director_config.schema, rest_config.CONFIG_SECTION_NAME: rest_config.schema, projects_config.CONFIG_SECTION_NAME: projects_config.schema, email_config.CONFIG_SECTION_NAME: email_config.schema, computation_config.CONFIG_SECTION_NAME: computation_config.schema, storage_config.CONFIG_SECTION_NAME: storage_config.schema, T.Key(login_config.CONFIG_SECTION_NAME, optional=True): login_config.schema #s3_config.CONFIG_SECTION_NAME: s3_config.schema #TODO: enable when sockets are refactored }) section_names = [k.name for k in schema.keys] assert len(section_names) == len(set( section_names)), "Found repeated section names in %s" % section_names return schema
def filters_validate(self, **kwargs): regex_filter_schema = t.Dict({ t.Key('pattern'): t.String, t.Key('type'): t.String }) age_filter_schema = t.Dict({ t.Key('term'): t.String, t.Key('type'): t.String, t.Key('unit'): t.Enum(*['hours', 'days', 'weeks', 'months']), t.Key('count'): t.Int }) checks = {'regex': regex_filter_schema, 'age': age_filter_schema} for filter in kwargs['filters']: checks[filter['type']](filter)
def test_repr(self): trafaret = t.Enum("foo", "bar", 1) assert repr(trafaret), "<Enum('foo', 'bar', 1)>"
from .types import VolumeInfo _max_cpu_count = os.cpu_count() local_config_iv = t.Dict({ t.Key('storage-proxy'): t.Dict({ t.Key('node-id'): t.String, t.Key('num-proc', default=_max_cpu_count): t.Int[1:_max_cpu_count], t.Key('pid-file', default=os.devnull): tx.Path(type='file', allow_nonexisting=True, allow_devnull=True), t.Key('event-loop', default='asyncio'): t.Enum('asyncio', 'uvloop'), t.Key('scandir-limit', default=1000): t.Int[0:], t.Key('max-upload-size', default="100g"): tx.BinarySize, t.Key('secret'): t.String, # used to generate JWT tokens t.Key('session-expire'): tx.TimeDuration, }), t.Key('logging'): logging_config_iv, t.Key('api'): t.Dict({ t.Key('client'): t.Dict({
def test_enum(self): trafaret = t.Enum("foo", "bar", 1) trafaret.check("foo") trafaret.check(1) res = extract_error(trafaret, 2) assert res == "value doesn't match any variant"
] }) @server_status_required(ALL_ALLOWED) @admin_required @check_api_params( t.Dict({ t.Key('src'): t.String, t.Key('target'): t.String, t.Key('launchOptions', default={}): t.Dict({ t.Key('scalingGroup', default='default'): t.String, t.Key('group', default='default'): t.String, }).allow_extra('*'), t.Key('brand'): t.String, t.Key('baseDistro'): t.Enum('ubuntu', 'centos'), t.Key('minCPU', default=1): t.Int[1:], t.Key('minMemory', default='64m'): tx.BinarySize, t.Key('preferredSharedMemory', default='64m'): tx.BinarySize, t.Key('supportedAccelerators'): t.List(t.String), t.Key('runtimeType'): t.Enum('python'), t.Key('runtimePath'): tx.Path(type='file', allow_nonexisting=True, resolve=False), t.Key('CPUCountEnvs'): t.List(t.String), t.Key('servicePorts', default=[]): t.List(t.Dict({ t.Key('name'): t.String, t.Key('protocol'): t.Enum('http', 'tcp', 'pty'), t.Key('ports'): t.List(t.Int[1:65535], min_length=1), })), }).allow_extra('*')) async def import_image(request: web.Request, params: Any) -> web.Response: '''
class ImageFeaturizer: """ This object can load images, rescale, crop, and vectorize them into a uniform batch, and then featurize the images for use with custom classifiers. Methods ------------------ __init__(depth, auto_sample, downsample_size): -------------------------------- Initialize the ImageFeaturizer. Build the featurizer model with the depth and feature downsampling specified by the inputs. load_and_featurize_data(image_column_headers, image_path, csv_path, new_csv_name, scaled_size, grayscale): -------------------------------- Loads image directory and/or csv into the model, and featurizes the images load_data(image_column_headers, image_path, csv_path, new_csv_name, scaled_size, grayscale): -------------------------------- Loads image directory and/or csv into the model, and vectorize the images for input into the featurizer featurize(): -------------------------------- Featurize the loaded data, append the features to the csv, and return the full dataframe """ @t.guard(depth=t.Int(gte=1, lte=4), auto_sample=t.Bool, downsample_size=t.Int(gte=0), model=t.Enum(*supported_model_types.keys())) def __init__(self, depth=1, auto_sample=False, downsample_size=0, model='squeezenet' ): """ Initializer. Loads an initial InceptionV3 pretrained network, decapitates it and downsamples according to user specifications. Parameters: ---------- depth : int How deep to decapitate the model. Deeper means less specific but also less complex auto_sample : bool If True, feature layer is automatically downsampled to the right size. downsample_size: int The number of features to downsample the featurizer to Returns: -------- None. Initializes and saves the featurizer object attributes. """ # BUILDING THE MODEL # logging.info("Building the featurizer.") featurizer = build_featurizer(depth, auto_sample, downsample_size, model_str=model.lower()) # Saving initializations of model self.depth = depth self.auto_sample = auto_sample self.downsample_size = downsample_size self.num_features = featurizer.layers[-1].output_shape[-1] # Save the model self.model_name = model.lower() self.featurizer = featurizer self.visualize = featurizer.summary # Initializing preprocessing variables for after we load and featurize the images self.data = np.zeros((1)) self.features = np.zeros((1)) self.full_dataframe = pd.DataFrame() self.csv_path = '' self.image_list = '' self.image_column_headers = '' self.image_path = '' # Image scaling and cropping self.scaled_size = (0, 0) self.crop_size = (0, 0) self.number_crops = 0 self.isotropic_scaling = False def load_and_featurize_data(self, image_column_headers, image_path='', csv_path='', new_csv_name='featurizer_csv/generated_images_csv', grayscale=False, save_features=False, omit_time=False, omit_model=False, omit_depth=False, omit_output=False # crop_size = (299, 299), # number_crops = 0, # random_crop = False, # isotropic_scaling = True ): """ Load image directory and/or csv, and vectorize the images for input into the featurizer. Then, featurize the data. Parameters: ---------- image_column_headers : str the name of the column holding the image data, if a csv exists, or what the name of the column will be, if generating the csv from a directory image_path : str the path to the folder containing the images. If using URLs, leave blank csv_path : str the path to the csv. If just using a directory, leave blank, and specify the path for the generated csv in new_csv_name. If csv exists, this is the path where the featurized csv will be generated. new_csv_name : str the path to the new csv, if one is being generated from a directory. If no csv exists, this is the path where the featurized csv will be generated grayscale : bool Decides if image is grayscale or not. May get deprecated. Don't think it works on the InceptionV3 model due to input size. ### These features haven't been implemented yet. # isotropic_scaling : bool # if True, images are scaled keeping proportions and then cropped # # crop_size: tuple # if the image gets cropped, decides the size of the crop # # random_crop: bool # If False, only take the center crop. If True, take random crop # Returns: -------- full_dataframe : Dataframe containing the features appended to the original csv. Also writes csvs containing the features only and the full dataframe to the same path as the csv containing the list of names """ self.load_data(image_column_headers, image_path, csv_path, new_csv_name, grayscale) return self.featurize(save_features=save_features, omit_time=omit_time, omit_model=omit_model, omit_depth=omit_depth, omit_output=omit_output) def load_data(self, image_column_headers, image_path='', csv_path='', new_csv_name='featurizer_csv/generated_images_csv', batch_size=1000, grayscale=False # crop_size = (299, 299), # number_crops = 0, # random_crop = False, # isotropic_scaling = True ): """ Load image directory and/or csv, and vectorize the images for input into the featurizer. Parameters: ---------- image_column_headers : str the name of the column holding the image data, if a csv exists, or what the name of the column will be, if generating the csv from a directory image_path : str the path to the folder containing the images. If using URLs, leave blank csv_path : str the path to the csv. If just using a directory, leave blank, and specify the path for the generated csv in new_csv_name. If csv exists, this is the path where the featurized csv will be generated. new_csv_name : str the path to the new csv, if one is being generated from a directory. If no csv exists, this is the path where the featurized csv will be generated grayscale : bool Decides if image is grayscale or not. May get deprecated. Don't think it works on the InceptionV3 model due to input size. ### These features haven't been implemented yet. # isotropic_scaling : bool # if True, images are scaled keeping proportions and then cropped # # crop_size: tuple # if the image gets cropped, decides the size of the crop # # random_crop: bool # If False, only take the center crop. If True, take random crop # """ size_dict = {'squeezenet': (227, 227), 'vgg16': (224, 224), 'vgg19': (224, 224), 'resnet50': (224, 224), 'inceptionv3': (299, 299), 'xception': (299, 299)} scaled_size = size_dict[self.model_name] # Convert column header to list if it's passed a single string if isinstance(image_column_headers, str): image_column_headers = [image_column_headers] # If new csv_path is being generated, make sure the folder exists. if (csv_path == ''): # Raise error if multiple image columns are passed in without a csv if len(image_column_headers) > 1: raise ValueError('If building the csv from a directory, featurizer can only ' 'create a single image column. If two image columns are needed, ' 'please create a csv to pass in.') # Create the filepath to the new csv path_to_new_csv = os.path.dirname(new_csv_name) if not os.path.isdir(path_to_new_csv) and path_to_new_csv != '': os.makedirs(os.path.dirname(new_csv_name)) # Add backslash to end of image path if it is not there if image_path != '' and image_path[-1] != "/": image_path = '{}/'.format(image_path) # Save the full image tensor, the path to the csv, and the list of image paths (image_data, csv_path, list_of_image_paths) = \ preprocess_data(image_column_headers[0], self.model_name, image_path, csv_path, new_csv_name, scaled_size, grayscale) full_image_list = [list_of_image_paths] full_image_data = np.expand_dims(image_data, axis=0) if len(image_column_headers) > 1: for column in image_column_headers[1:]: (image_data, csv_path, list_of_image_paths) = \ preprocess_data(column, self.model_name, image_path, csv_path, new_csv_name, scaled_size, grayscale) full_image_data = np.concatenate((full_image_data, np.expand_dims(image_data, axis=0))) full_image_list.append(list_of_image_paths) # Save all of the necessary data to the featurizer self.data = full_image_data self.csv_path = csv_path self.image_list = full_image_list self.image_column_headers = image_column_headers self.scaled_size = scaled_size self.image_path = image_path @t.guard(save_features=t.Bool, omit_time=t.Bool, omit_model=t.Bool, omit_depth=t.Bool, omit_output=t.Bool) def featurize(self, save_features=False, omit_time=False, omit_model=False, omit_depth=False, omit_output=False): """ Featurize the loaded data, returning the dataframe and writing the features and the full combined data to csv Parameters ---------- None, just operates on the loaded data Returns ------- full_dataframe : pandas.DataFrame Dataframe containing the features appended to the original csv. Also writes csvs containing the features only and the full dataframe to the same path as the csv containing the list of names """ # Check data has been loaded, and that the data was vectorized correctly if np.array_equal(self.data, np.zeros((1))): raise IOError('Must load data into the model first. Call load_data.') assert len(self.image_column_headers) == self.data.shape[0] logging.info("Trying to featurize data.") # Initialize featurized data vector with appropriate size self.features = np.zeros((self.data.shape[1], self.num_features * len(self.image_column_headers))) # Save csv_names csv_name, ext = os.path.splitext(self.csv_path) # For each image column, perform the full featurization and add the features to the csv for column in range(self.data.shape[0]): # Create the correct csv path if we have multiple image columns if column == 0: csv_path = "{}{}".format(csv_name, ext) else: named_path = _named_path_finder(csv_name, self.model_name, self.depth, self.num_features, omit_model, omit_depth, omit_output, omit_time) # Save the name and extension separately, for robust naming csv_path = '{}_full{}'.format(named_path, ext) # Featurize the data, and save it to the appropriate columns self.features[:, self.num_features * column:self.num_features * column + self.num_features] \ = partial_features = featurize_data(self.featurizer, self.data[column]) # Save the full dataframe to the csv full_dataframe = _features_to_csv(self.data[column], partial_features, csv_path, self.image_column_headers[column], self.image_list, model_str=self.model_name, model_depth=self.depth, model_output=self.num_features, omit_model=omit_model, omit_time=omit_time, omit_depth=omit_depth, omit_output=omit_output, save_features=save_features, continued_column=column) self.full_dataframe = full_dataframe return full_dataframe
OptKey('le'): SimpleType, OptKey('ne'): SimpleType, OptKey('eq'): SimpleType, OptKey('like'): SimpleType, }) ASC = 'ASC' DESC = 'DESC' ListQuery = t.Dict({ OptKey('_page', default=1): t.Int[1:], OptKey('_perPage', default=30): t.Int[1:], OptKey('_sortField'): t.String, OptKey('_sortDir', default=DESC): t.Enum(DESC, ASC), OptKey('_filters'): t.Mapping(t.String, Filter | SimpleType) }) LoginForm = t.Dict({ "username": t.String, "password": t.String, }) def validate_query_structure(query): """Validate query arguments in list request. :param query: mapping with pagination and filtering information """
class AccountValidator(TrafaretValidator): name = t.String(max_length=50) nickname = t.String(max_length=20) password = t.String(max_length=20) role = t.Enum(*[e.name for e in Role])
""" rest subsystem's configuration - constants - config-file schema """ import trafaret as T from servicelib.application_keys import APP_OPENAPI_SPECS_KEY APP_OPENAPI_SPECS_KEY = APP_OPENAPI_SPECS_KEY # pylint: disable=self-assigning-variable,bad-option-value CONFIG_SECTION_NAME = 'rest' schema = T.Dict({ "version": T.Enum("{{cookiecutter.openapi_specs_version}}"), "location": T.Or(T.String, T.URL), # either path or url should contain version in it })
import trafaret as T from .rest_config import schema as rest_schema app_schema = T.Dict({ T.Key("host", default="0.0.0.0"): T.IP, "port": T.Int(), "log_level": T.Enum("DEBUG", "WARNING", "INFO", "ERROR", "CRITICAL", "FATAL", "NOTSET"), "watched_git_repositories": T.List( T.Dict({ "id": T.String(), "url": T.URL, T.Key("username", optional=True, default=""): T.Or(T.String(allow_blank=True), T.Null), T.Key("password", optional=True, default=""): T.Or(T.String(allow_blank=True), T.Null), T.Key("branch", default="master", optional=True): T.String(allow_blank=True), T.Key("tags", default="", optional=True): T.String(allow_blank=True), "pull_only_files": T.Bool(), "paths": T.List(T.String()), }),
@server_status_required(READ_ALLOWED) @auth_required @check_api_params( t.Dict({ tx.AliasedKey(['name', 'sessionName'], default='*') >> 'session_name': t.String, t.Key('ownerAccessKey', default=None) >> 'owner_access_key': t.Null | t.String, t.Key('sessionId', default=None) >> 'session_id': t.Null | tx.UUID, # NOTE: if set, sessionId overrides sessionName and ownerAccessKey parameters. tx.AliasedKey(['group', 'groupName'], default='*') >> 'group_name': t.String, t.Key('scope', default='*'): t.Enum('*', 'session', 'kernel'), })) @adefer async def push_session_events( defer, request: web.Request, params: Mapping[str, Any], ) -> web.StreamResponse: app = request.app session_name = params['session_name'] session_id = params['session_id'] scope = params['scope'] user_role = request['user']['role'] user_uuid = request['user']['uuid'] access_key = params['owner_access_key'] if access_key is None:
EVENT_SUGGESTION_TRAFARET = t.Dict({ 'title': t.String, 'agenda': t.String, 'social': t.String(allow_blank=True) | t.Null, 'place': t.String(allow_blank=True), 'registration_url': t.URL(allow_blank=True) | t.Null, 'image_url': t.URL(allow_blank=True) | t.String(max_length=0, allow_blank=True), 'level': t.Enum('NONE', 'TRAINEE', 'JUNIOR', 'MIDDLE', 'SENIOR'), 'when_start': t.String, 'when_end': (t.String(allow_blank=True) >> (lambda x: None if not x else x)) | t.Null, 'only_date': t.StrBool(), 'team': t.String() >> get_team_by_name, 'submitter_email': t.Email(), 'secret': t.String }).make_optional('secret', 'social')
import trafaret as t STORE_LOCATION_SCHEMA = t.Enum('S3', 'gcs', 'azure', 'rackspace', 'dropbox') def validate_upload_tags(d): t.List(t.String, max_length=10).check(list(d.keys())) t.Mapping(t.String(max_length=128), t.String(max_length=256)).check(d) return d STORE_SCHEMA = t.Dict( t.Key('filename', optional=True, trafaret=t.String), t.Key('mimetype', optional=True, trafaret=t.String), t.Key('location', optional=True, trafaret=t.String), t.Key('path', optional=True, trafaret=t.String), t.Key('container', optional=True, trafaret=t.String), t.Key('region', optional=True, trafaret=t.String), t.Key('access', optional=True, trafaret=t.String), t.Key('base64decode', optional=True, trafaret=t.Bool), t.Key('workflows', optional=True, trafaret=t.List(t.String)), t.Key('upload_tags', optional=True, trafaret=validate_upload_tags), )
}, 'xception': { 'label': 'Xception', 'class': Xception, 'kwargs': {}, 'depth': { 1: 1, 2: 8, 3: 18, 4: 28 } } } @t.guard(model_str=t.Enum(*supported_model_types.keys()), loaded_weights=t.String(allow_blank=True)) def _initialize_model(model_str, loaded_weights=''): """ Initialize the InceptionV3 model with the saved weights, or if the weight file can't be found, load them automatically through Keras. Parameters: ---------- model_str : str String deciding which model to use for the featurizer Returns: ------- model : keras.models.Model The initialized model loaded with pre-trained weights """ logging.info('Loading/downloading {model_label} model weights. '
def main(cli_ctx, config_path, debug): volume_config_iv = t.Dict({ t.Key('etcd'): t.Dict({ t.Key('namespace'): t.String, t.Key('addr'): tx.HostPortPair(allow_blank_host=False) }).allow_extra('*'), t.Key('logging'): t.Any, # checked in ai.backend.common.logging t.Key('agent'): t.Dict({ t.Key('mode'): t.Enum('scratch', 'vfolder'), t.Key('rpc-listen-addr'): tx.HostPortPair(allow_blank_host=True), t.Key('user-uid'): t.Int, t.Key('user-gid'): t.Int }), t.Key('storage'): t.Dict({ t.Key('mode'): t.Enum('xfs', 'btrfs'), t.Key('path'): t.String }) }).allow_extra('*') # Determine where to read configuration. raw_cfg, cfg_src_path = config.read_from_file(config_path, 'agent') config.override_with_env(raw_cfg, ('etcd', 'namespace'), 'BACKEND_NAMESPACE') config.override_with_env(raw_cfg, ('etcd', 'addr'), 'BACKEND_ETCD_ADDR') config.override_with_env(raw_cfg, ('etcd', 'user'), 'BACKEND_ETCD_USER') config.override_with_env(raw_cfg, ('etcd', 'password'), 'BACKEND_ETCD_PASSWORD') config.override_with_env(raw_cfg, ('agent', 'rpc-listen-addr', 'host'), 'BACKEND_AGENT_HOST_OVERRIDE') config.override_with_env(raw_cfg, ('agent', 'rpc-listen-addr', 'port'), 'BACKEND_AGENT_PORT') if debug: config.override_key(raw_cfg, ('debug', 'enabled'), True) config.override_key(raw_cfg, ('logging', 'level'), 'DEBUG') config.override_key(raw_cfg, ('logging', 'pkg-ns', 'ai.backend'), 'DEBUG') try: cfg = config.check(raw_cfg, volume_config_iv) cfg['_src'] = cfg_src_path except config.ConfigurationError as e: print( 'ConfigurationError: Validation of agent configuration has failed:', file=sys.stderr) print(pformat(e.invalid_data), file=sys.stderr) raise click.Abort() rpc_host = cfg['agent']['rpc-listen-addr'].host if (isinstance(rpc_host, BaseIPAddress) and (rpc_host.is_unspecified or rpc_host.is_link_local)): print( 'ConfigurationError: ' 'Cannot use link-local or unspecified IP address as the RPC listening host.', file=sys.stderr) raise click.Abort() if os.getuid() != 0: print('Storage agent can only be run as root', file=sys.stderr) raise click.Abort() if cli_ctx.invoked_subcommand is None: setproctitle('Backend.AI: Storage Agent') logger = Logger(cfg['logging']) with logger: log.info('Backend.AI Storage Agent', VERSION) log_config = logging.getLogger('ai.backend.agent.config') if debug: log_config.debug('debug mode enabled.') if 'debug' in cfg and cfg['debug']['enabled']: print('== Agent configuration ==') pprint(cfg) aiotools.start_server(server_main, num_workers=1, use_threading=True, args=(cfg, )) log.info('exit.') return 0
class ProductId(TrafaretPoweredAttribute): trafaret = trafaret.Enum('F171', 'F172', 'F173')
if row is None: raise GenericNotFound('No such user group or ' 'you are not the member of the group.') group_role = 'user' resp_data = { 'global_role': 'superadmin' if request['is_superadmin'] else 'user', 'domain_role': 'admin' if request['is_admin'] else 'user', 'group_role': group_role, } return web.json_response(resp_data) @atomic @check_api_params( t.Dict({ t.Key('type'): t.Enum('keypair', 'jwt'), t.Key('domain'): t.String, t.Key('username'): t.String, t.Key('password'): t.String, })) async def authorize(request: web.Request, params: Any) -> web.Response: if params['type'] != 'keypair': # other types are not implemented yet. raise InvalidAPIParameters('Unsupported authorization type') log.info( 'AUTH.AUTHORIZE(d:{0[domain]}, u:{0[username]}, passwd:****, type:{0[type]})', params) dbpool = request.app['dbpool'] # [Hooking point for AUTHORIZE with the FIRST_COMPLETED requirement] # The hook handlers should accept the whole ``params`` dict, and optional