def get_pool_config():
posint = schema.And(int, lambda a: a >=0)
pool_conf = schema_builder.opt_up({
opt('instance_size'): schema.And(int, lambda n: 0 < n <= 64),
opt('instances'): int,
opt('instance_group'): schema.Or('Backend', 'Common'),
opt('one_per_host', default=True): bool,
opt('use_canary', default=True): bool,
opt('hostname'): str,
opt('vips'): schema_builder.none_or(schema.Use(_parse_vip)),
opt('health_check'): {
opt('grace_period_seconds'): posint,
opt('interval_seconds'): posint,
opt('timeout_seconds'): posint,
opt('max_consecutive_failures'): posint,
opt('command', default=DEFAULT_HEALTH_CMD): str,
},
opt('haproxy_group', default='internal'):
schema.Or('internal', 'external'),
})
return {
schema.Optional('default'): pool_conf,
schema.Or(*ENVIRONMENTS): {
#instance-id e.g. /backends/urlshortener
str: schema_builder.nullable_dict(pool_conf),
}
}
def DATASCHEMA(self):
import schema
return schema.Schema({
'plane': Plane.DATASCHEMA.fget(None),
'major': schema.And(float, lambda x: x > 0),
'minor': schema.And(float, lambda x: x > 0),
})
def DATASCHEMA(self):
import schema
return schema.Schema({
'plane': Plane.DATASCHEMA.fget(None),
'radius_axis': schema.And(float, lambda x: x > 0),
'radius_pipe': schema.And(float, lambda x: x > 0)
})
def DATASCHEMA(self):
""":class:`schema.Schema` : Schema of the data representation."""
import schema
return schema.Schema({
'plane': Plane.DATASCHEMA.fget(None),
'major': schema.And(float, lambda x: x > 0),
'minor': schema.And(float, lambda x: x > 0),
})
def DATASCHEMA(self):
import schema
return schema.Schema({
'frame': Frame.DATASCHEMA.fget(None),
'xsize': schema.And(float, lambda x: x > 0),
'ysize': schema.And(float, lambda x: x > 0),
'zsize': schema.And(float, lambda x: x > 0)
})
def DATASCHEMA(self):
""":class:`schema.Schema` : Schema of the data representation."""
import schema
return schema.Schema({
'frame': Frame.DATASCHEMA.fget(None),
'xsize': schema.And(float, lambda x: x > 0),
'ysize': schema.And(float, lambda x: x > 0),
'zsize': schema.And(float, lambda x: x > 0)
})
def DATASCHEMA(self):
import schema
return schema.Schema({
'circle': {
'plane': Plane.DATASCHEMA.fget(None),
'radius': schema.And(float, lambda x: x > 0)
},
'height': schema.And(float, lambda x: x > 0)
})
def validate(data, set_dir):
def is_file(path):
return path.is_file()
def to_fixture(data):
return Fixture(**data)
def to_step(data):
return Step(**data)
def to_test(data):
return Test(**data)
def absolute_path(path):
absolute = Path(os.path.expanduser(path))
if not absolute.is_absolute():
absolute = (set_dir / path).resolve()
return absolute
def to_command(raw_command):
return shlex.split(raw_command.replace('@.', str(set_dir)))
fixture = schema.Schema(
schema.And(
{
'enter': schema.And(str, len),
'exit': schema.And(str, len)
}, schema.Use(to_fixture)))
fixtures = schema.Schema({schema.And(str, len): fixture})
step = schema.Schema(
schema.And(
{
'command':
schema.And(schema.Const(schema.And(str, len)),
schema.Use(to_command)),
schema.Optional('input', default=None):
schema.And(schema.Use(absolute_path), is_file)
}, schema.Use(to_step)))
test = schema.Schema(
schema.And(
{
schema.Optional('tags', default=None): [str],
schema.Optional('condition', default=None): str,
schema.Optional('fixture', default=None): str,
'steps': [step]
}, schema.Use(to_test)))
tests = schema.Schema({schema.And(str, len): test})
sch = schema.Schema({
schema.Optional('config-file', default=None):
schema.Use(absolute_path),
schema.Optional('fixtures', default=None):
fixtures,
'tests':
tests
})
return sch.validate(data)
def DATASCHEMA(self):
""":class:`schema.Schema` : Schema of the data representation."""
import schema
return schema.Schema({
'circle': {
'plane': Plane.DATASCHEMA.fget(None),
'radius': schema.And(float, lambda x: x > 0)
},
'height': schema.And(float, lambda x: x > 0)
})
def get_token(self, password):
response = requests.post(
self.endpoint + '/authorizations',
auth=(self.username, password),
headers=self.default_headers,
data=json.dumps({
'scopes': ['repo'],
'note': 'git-wat',
})
)
if response.status_code < 200 or response.status_code >= 300:
raise errors.RequestFailed(response)
try:
result = response.json()
s.Schema({
'hashed_token': s.And(
basestring,
lambda t: re.match(r'^[0-9a-fA-F]+$', t)
),
s.Optional(basestring): object,
}).validate(result)
self._set_token(result['hashed_token'])
except ValueError:
raise errors.UnexpectedResponse(response, "Body is not JSON")
except s.SchemaError as e:
raise errors.UnexpectedResponse(response, str(e))
class Apt(Whiteprint):
cfg_schema = {
"packages": [schema.And(str, schema.Use(str.lower))],
}
def _execute(self, mode: str) -> None:
if mode == "install":
self.exec(
"DEBIAN_FRONTEND=noninteractive "
"apt install -y %s" % " ".join(self.cfg["packages"])
)
def _validate(self, mode: str) -> Optional[str]:
if mode == "install":
res = self.exec("apt -qq list %s" % " ".join(self.cfg["packages"]))
installed_packages = set()
for line in res.stdout.decode("utf-8").splitlines():
installed_package, _ = line.split("/", 1)
installed_packages.add(installed_package.lower())
for package in self.cfg["packages"]:
if package not in installed_packages:
return "Apt package %r missing." % package
return None
else:
return None
def serve(port):
"""Run a development api server. Don't use this in production."""
try:
port = schema.And(
schema.Use(int),
lambda n: MIN_PORT_NUMBER <= n <= MAX_PORT_NUMBER).validate(port)
except schema.SchemaError:
raise InvalidAPIArgumentsException(
'Error: Invaid port. Must be in the range 1-65535.')
except Exception as e:
sys.exit('Unxpected Error!!! \n %s' % e)
"""Start the HIL API server"""
config.setup()
if cfg.has_option('devel', 'debug'):
debug = cfg.getboolean('devel', 'debug')
else:
debug = False
# We need to import api here so that the functions within it get registered
# (via `rest_call`), though we don't use it directly:
# pylint: disable=unused-variable
from hil import api, rest
server.init()
migrations.check_db_schema()
server.stop_orphan_consoles()
rest.serve(port, debug=debug)
def validate_int_option(int_option, msg, min_val):
msg = "{msg}: '{val}'".format(msg=msg, val=int_option)
validator = schema.Use(int)
if min_val is not None:
validator = schema.And(validator, lambda x: x >= min_val)
return schema.Schema(validator, error=msg).validate(int_option)
def DATASCHEMA(self):
import schema
from compas.data import is_float3
return schema.Schema({
'point': is_float3,
'radius': schema.And(float, lambda x: x > 0)
})
class LineInFile(Whiteprint):
cfg_schema = {
"path": str,
"line": schema.And(
str, lambda line: "\n" not in line, lambda line: "\r" not in line
),
}
def _execute(self, mode: str) -> None:
quoted_line = shlex.quote(self.cfg["line"])
if mode == "install":
res = self.exec(
"grep -q {} {}".format(quoted_line, self.cfg["path"]), error_ok=True
)
if res.exit_status == 0:
return None
elif res.exit_status == 1 or res.exit_status == 2:
# 1: Line not found in file
# 2: File does not exist
self.exec("echo {} >> {}".format(quoted_line, self.cfg["path"]))
else:
assert False, "Unknown grep exit status: %d" % res.exit_status
elif mode == "clean":
quoted_pattern = shlex.quote("/^{}$/d".format(self.cfg["line"]))
self.exec(
'sed --in-place="" {} {}'.format(quoted_pattern, self.cfg["path"])
)
def _validate(self, mode: str) -> Optional[str]:
quoted_line = shlex.quote(self.cfg["line"])
if mode == "install":
res = self.exec(
"grep -q {} {}".format(quoted_line, self.cfg["path"]), error_ok=True
)
if res.exit_status == 0:
return None
else:
return "Line {!r}{} not found in {!r}".format(
quoted_line[:10],
"" if len(quoted_line) < 10 else "...",
self.cfg["path"],
)
elif mode == "clean":
res = self.exec(
"grep -q {} {}".format(quoted_line, self.cfg["path"]), error_ok=True
)
if res.exit_status == 1 or res.exit_status == 2:
return None
elif res.exit_status == 0:
return "Found line {!r}{} in {!r}".format(
quoted_line[:10],
"" if len(quoted_line) < 10 else "...",
self.cfg["path"],
)
else:
return "Unknown exit status from grep: {}".format(res.exit_status)
else:
return None
def DATASCHEMA(self):
""":class:`schema.Schema` : Schema of the data representation."""
import schema
from compas.data import is_float3
return schema.Schema({
'point': is_float3,
'radius': schema.And(float, lambda x: x > 0)
})
def test_inner_schema(self):
s = ObjectSchema(
b=schema.And(int, lambda x: x > 98, lambda x: x < 100))
output = s.validate(self.data)
# validation preserves identity
self.assertIs(output, self.data)
# this schema does not change objects:
self.assertEqual(output.a, "hello")
self.assertEqual(output.b, 99)
class LstripTransformation(BaseTransformation):
schema = schema.Schema({
"field": str,
"char": schema.And(str, lambda char: len(char) == 1)
})
def apply(self, row: Dict, **kwargs):
return str(row.get(self.args["field"], None)).lstrip(self.args["char"])
class RecipeBundleFeature(RecipeFeature):
"""Recipe feature for bundling (*e.g.* making a tar archive or a squashfs
image from any previous result of another recipe) whole or parts of the
final recipe target."""
NAME = "bundle"
FEATURED_ATTRIBUTES = {"bundle"}
SCHEMA = schema.Schema({
"bundle": {
"sdk": schema.Regex(RECIPE_IDENTIFIER_RE.pattern),
schema.Optional("env", default={}): {
schema.Regex(ENVVAR_FORMAT_RE.pattern,
error="Bad environment variable name"):
str
},
"steps": schema.And([str], len),
},
str: object, # do not consider other keys
})
def bundle(self) -> None:
# using getattr to avoid static analyzers from complaining about
# missing attr (but brought by a recipe feature):
sdk = getattr(recipe.Recipe(self.recipe.config["bundle"]["sdk"]),
"sdk")
action_out_subpath = os.path.join(self.recipe.out_subpath, "bundle")
action_out_path = os.path.join(repo_root_path(), action_out_subpath)
# Always clear out the previous image result:
if os.path.exists(action_out_path):
debug("clearing {!r}...".format(action_out_path))
with ElevatedPrivileges():
shutil.rmtree(action_out_path)
# prepare output directory
os.makedirs(action_out_path)
# hook the SDK container process to the current TTY (if existent):
terminal = is_tty_attached()
with sdk.session(action_name="bundle",
action_targeted_recipe=self.recipe,
env={
key: self.replace_placeholders(value,
sdk_context=True)
for key, value in self.recipe.config["bundle"]
["env"].items()
},
terminal=terminal,
shared_host_netns=False) as sess:
for cmd in self.recipe.config["bundle"]["steps"]:
info("{!r} bundles recipe {!r}, runs:\n {}".format(
sdk.recipe.identifier, self.recipe.identifier, cmd))
sess.run(self.replace_placeholders(cmd, sdk_context=True))
def validate(kwargs):
schema.Schema({
'username': basestring,
'hostname': basestring,
'password': basestring,
'dummy_vlan': schema.And(schema.Use(int),
lambda v: 0 <= v and v <= 4096,
schema.Use(str)),
}).validate(kwargs)
def DATASCHEMA(self):
import schema
return schema.Schema({
"attributes": dict,
"dna": dict,
"dea": dict,
"node": dict,
"edge": dict,
"adjacency": dict,
"max_node": schema.And(int, lambda x: x >= -1)
})
def verify_config(config: Box) -> bool:
"""Return (True|False) result if the config matches the schema."""
config_schema = schema.Schema({
"discovergy_account": {
"email":
schema.And(str, len),
"password":
schema.Optional(str),
"save_password":
schema.And(schema.Use(str.lower), lambda x: x in
("true", "false")),
},
"file_location": {
"data_dir": str,
"log_dir": str,
},
"poll": {
"default": schema.Use(int),
"try_sleep": schema.Use(int),
},
schema.Optional("open_weather_map"): {
"id": str
},
schema.Optional("oauth1_token"): {
"key": str,
"client_secret": str,
"token": str,
"token_secret": str,
},
# meters stores the meters to read if configured. Otherwise, read all. The key is a nice name,
# value is the meter id.
schema.Optional("meters"): {
str: str
},
})
try:
config_schema.validate(config)
except schema.SchemaError as e:
log.error(f"Caught a config schema violation error: {e}")
return False
return True
def _check_appinfo(appinfo):
"""Check whether the parameters are legitimate
Args:
appinfo:(dict)
Returns:
bool
"""
rules = {
"action": schema.And(lambda n: n in ["deploy"],
error="action ind deploy")
}
return schema.Schema(rules).is_valid(appinfo)
def _check_format(schema_dic):
"""
check if a mspass.yaml file user provides is valid or not
:param schema_dic: the dictionary that a yaml file is dumped
:type schema_dic: dict
:return: `True` if the schema is valid, else return `False`
:rtype: bool
"""
# Make sure Database and Metadata exist
schema.Schema({'Database': dict, 'Metadata': dict}).validate(schema_dic)
collection_name_list = schema_dic['Database'].keys()
database_collection_schema = schema.Schema({
schema.Optional('default'):str,
schema.Optional('data_type'):str,
schema.Optional('base'):schema.And(str, lambda s: s in collection_name_list),
'schema':schema.And(dict, lambda dic: _is_valid_database_schema_definition(dic, collection_name_list))
}, ignore_extra_keys=True)
metadata_collection_schema = schema.Schema({
'schema':schema.And(dict, lambda dic: _is_valid_metadata_schema_definition(dic, collection_name_list))
}, ignore_extra_keys=True)
yaml_schema = schema.Schema({
'Database': schema.And(dict, schema.And(lambda dic: _is_valid_schema(dic, database_collection_schema),
lambda dic: _check_min_default_key(dic))),
'Metadata': schema.And(dict, lambda dic: _is_valid_schema(dic, metadata_collection_schema)),
}, ignore_extra_keys=True)
yaml_schema.validate(schema_dic)
def caliban_config(conf_path: str = CALIBAN_CONFIG) -> CalibanConfig:
"""Returns a dict that represents a `.calibanconfig.json` file if present,
empty dictionary otherwise.
If the supplied conf_path is present, but doesn't pass the supplied schema,
errors and kills the program.
"""
if not os.path.isfile(conf_path):
return {}
with us.error_schema(conf_path):
return s.And(us.Json, CalibanConfig).validate(conf_path)
def get_options(cls):
return {
"suites": schema.Use(iterable_suites),
config.ConfigOption("thread_pool_size", default=0): int,
config.ConfigOption("max_thread_pool_size", default=10): int,
config.ConfigOption("stop_on_error", default=True): bool,
config.ConfigOption("part", default=None): schema.Or(
None,
schema.And(
(int,),
lambda tp: len(tp) == 2
and 0 <= tp[0] < tp[1]
and tp[1] > 1,
),
),
config.ConfigOption(
"result", default=result.Result
): validation.is_subclass(result.Result),
config.ConfigOption("fix_spec_path", default=None): schema.Or(
None, schema.And(str, os.path.exists)
),
}
def _build_schema(*args, **kwargs):
# Helper to build schema form the arguments of input_schema & output_schema
args = list(args)
for i, a in enumerate(args):
# Dictionaries are "softened"
if isinstance(a, dict):
args[i] = soft_schema(**a)
if kwargs:
for k, a in kwargs.items():
if isinstance(a, dict):
args[k] = soft_schema(**a)
# if there are kwargs, add an objectschema to the condition
args.append(ObjectSchema(**kwargs))
return schema.Schema(schema.And(*args))
def validate(self, data):
"""Validate command arguments.
Parameters:
data: A dictionary with command arguments and options.
Returns:
A dictionary containing the validated command arguments.
Raises:
SchemaError: If any argument value is invalid.
"""
validator = schema.Schema({
"FILE":
schema.And(str, len, error="Invalid input file"),
"TARGET":
schema.And(str, len, error="Invalid destination directory"),
schema.Optional("--buffer-length"):
schema.Or(
None,
schema.And(
schema.Use(int),
lambda n: n > 0,
error=("Buffer length must be "
"an integer greater than 0"),
),
),
schema.Optional("--help"):
schema.Use(bool),
schema.Optional("-h"):
schema.Use(bool),
schema.Optional("--version"):
schema.Use(bool),
})
return validator.validate(data)
def DATASCHEMA(self):
import schema
version = LooseVersion(compas.__version__)
meta = {"compas": str, "datatype": str, "data": None}
data = {
"attributes": dict,
"node_attributes": dict,
"edge_attributes": dict,
"node": dict,
"edge": dict,
"adjacency": dict,
"max_int_key": schema.And(int, lambda x: x >= -1)
}
if version < LooseVersion('0.16.5'):
return schema.Schema(data)
meta["data"] = data
return schema.Schema(meta)
class Pip3(Whiteprint):
cfg_schema = {
# Pin version using '==X.Y.Z' notation.
"packages": [schema.And(str, schema.Use(str.lower))],
}
@staticmethod
def _mk_package_map(packages: List[str]) -> Dict[str, Optional[str]]:
"""Returns dict mapping package to version."""
package_map: Dict[str, Optional[str]] = {}
for package in packages:
package_name, *version = package.split("==", 1)
package_map[package_name] = version[0] if version else None
return package_map
def _execute(self, mode: str) -> None:
if mode == "install":
self.exec("pip3 install %s" % " ".join(self.cfg["packages"]))
def _validate(self, mode: str) -> Optional[str]:
package_map = Pip3._mk_package_map(self.cfg["packages"])
if mode == "install":
res = self.exec("pip3 show %s" % " ".join(package_map.keys()))
installed_packages: Dict[str, str] = {}
for line in res.stdout.decode("utf-8").splitlines():
# Assumes consistency in presence and order: Version must
# always exist after Name.
if line.startswith("Name: "):
installed_package = line.split(maxsplit=1)[1]
elif line.startswith("Version: "):
installed_version = line.split(maxsplit=1)[1]
installed_packages[installed_package] = installed_version
for req_package, req_version in package_map.items():
if req_package not in installed_packages:
return "Pip3 package %r missing." % req_package
elif req_version and req_version != installed_packages[req_package]:
return "Pip3 package %r wrong version: %r != %r" % (
req_package,
req_version,
installed_packages[req_package],
)
return None
else:
return None