def schema():
base = {
Required("interactive"): [{
Required("name"):
Match(r"^[-_a-zA-Z0-9.]+$"),
Required("prompts"): [All(str, Length(min=1))],
Optional("echo"):
"discard",
Required("script"): [{
Required("command"):
Any(str, None),
Optional("name"):
Match(r"^[-_a-zA-Z0-9.]+$"),
Optional("wait_for_prompt"):
bool,
Optional("successes"): [{
Required("message"):
All(str, Length(min=1))
}],
Optional("failures"): [{
Required("message"):
All(str, Length(min=1)),
Optional("exception"):
Any("InfrastructureError", "JobError", "TestError"),
Optional("error"):
All(str, Length(min=1)),
}],
}],
}]
}
return {**test.schema(), **base}
def test_SomeOf_max_validation():
validator = SomeOf(
max_valid=2,
validators=[Match(r'.*[A-Z]', 'no uppercase letters'),
Match(r'.*[a-z]', 'no lowercase letters'),
Match(r'.*[0-9]', 'no numbers')],
msg='max validation test failed')
validator('Aa')
with raises(TooManyValid, 'max validation test failed'):
validator('Aa1')
def testdef():
return {
Required("metadata"): {
Required("format"): "Lava-Test Test Definition 1.0",
Required("name"): str,
Optional("description"): str,
Optional("environment"): [str],
Optional("devices"): [str],
Optional("maintainer"): [str],
Optional("scope"): [str],
Optional("os"): [str],
},
Optional("install"): {
Optional("deps"): [str],
Optional(Match(r"deps-.+")): [str],
Optional("git-repos"): [str],
Optional("keys"): [str],
Optional("steps"): [str],
Optional("sources"): [str],
},
Optional("parameters"): dict,
Optional("params"): dict,
Optional("parse"): {
Optional("fixupdict"): {
str: Any("pass", "fail", "skip", "unknown")
}
},
Optional("run"): {
Required("steps"): [str]
},
}
def _interactive_script_schema():
return Schema(
[
{
Optional("name"): Match(
r"^[a-zA-Z0-9-_]+$", msg=INVALID_CHARACTER_ERROR_MSG
),
Optional("command"): Any(None, str),
Optional("delay"): int,
Optional("lava-send"): str,
Optional("lava-sync"): str,
Optional("lava-wait"): str,
Optional("lava-wait-all"): str,
Optional("failures"): [
{
Required("message"): str,
Optional("exception"): Any(
"InfrastructureError", "JobError", "TestError"
),
Optional("error"): str,
}
],
Optional("successes"): [{Required("message"): str}],
}
]
)
def _schema_1():
"""Returns Voluptuous Schema object."""
return Schema({
Required('schema'): 1,
Required('bugzilla'): {
Required('product'): All(str, Length(min=1)),
Required('component'): All(str, Length(min=1)),
},
'origin': {
Required('name'): All(str, Length(min=1)),
Required('description'): All(str, Length(min=1)),
Required('url'): FqdnUrl(),
Required('license'): Msg(License(), msg='Unsupported License'),
Required('release'): All(str, Length(min=1)),
},
'vendoring': {
Required('url'): FqdnUrl(),
Required('revision'): Match(r'^[a-fA-F0-9]{12,40}$'),
'patches': Unique([str]),
'keep': Unique([str]),
'exclude': Unique([str]),
'include': Unique([str]),
'run_after': Unique([str]),
},
})
def test_encryption(self):
self.task_schema = Schema({
'task': {
'payload': {
'encryptedEnv': All(Length(2), [Match(r'^wcB')]) # Must have 2 elements, starting with wcB
}
}
}, required=True, extra=True)
test_kwargs = create_firefox_test_args({
'updates_enabled': True,
'repo_path': 'foo/bar',
'branch': 'mozilla-beta',
'signing_class': 'dep-signing',
'release_channels': ['beta'],
'final_verify_channels': ['beta'],
'signing_pvt_key': PVT_KEY_FILE,
'accepted_mar_channel_id': 'firefox-mozilla-beta',
'signing_cert': 'dep',
'moz_disable_mar_cert_verification': True,
'en_US_config': {
"platforms": {
"macosx64": {"unsigned_task_id": "xyz", "signed_task_id": "xyz"},
"win32": {"unsigned_task_id": "xyz", "signed_task_id": "xyz"},
}
},
})
graph = make_task_graph(**test_kwargs)
do_common_assertions(graph)
for p in ("win32", "macosx64"):
for v in ("38.0build1", "37.0build2"):
balrog = get_task_by_name(graph, "{}_en-US_{}_funsize_balrog_task".format(p, v))
verify(balrog, self.task_schema)
def test_SomeOf_min_validation():
validator = All(Length(min=8), SomeOf(
min_valid=3,
validators=[Match(r'.*[A-Z]', 'no uppercase letters'),
Match(r'.*[a-z]', 'no lowercase letters'),
Match(r'.*[0-9]', 'no numbers'),
Match(r'.*[$@$!%*#?&^:;/<,>|{}()\-\'._+=]', 'no symbols')]))
validator('ffe532A1!')
with raises(MultipleInvalid, 'length of value must be at least 8'):
validator('a')
with raises(MultipleInvalid, 'no uppercase letters, no lowercase letters'):
validator('wqs2!#s111')
with raises(MultipleInvalid, 'no lowercase letters, no symbols'):
validator('3A34SDEF5')
def _interactive_def_schema():
return Schema([{
Required("name"):
Match(r"^[a-zA-Z0-9-_]+$", msg=INVALID_CHARACTER_ERROR_MSG),
Required("prompts"):
list,
Required("script"):
_interactive_script_schema(),
}])
def schema():
image_name = Match("^[a-z0-9._:/-]+$", msg="Invalid docker image name")
base = {
Required("to"): "docker",
Required("image"): Any(
image_name, {Required("name"): image_name, "local": bool}
),
Optional("repeat"): Range(min=1),
}
return {**deploy.schema(), **base}
def test_match_error_has_path():
"""https://github.com/alecthomas/voluptuous/issues/347"""
s = Schema({
Required('q2'): Match("a"),
})
try:
s({'q2': 12})
except MultipleInvalid as exc:
assert exc.errors[0].path == ['q2']
else:
assert False, "Did not raise MatchInvalid"
def _monitor_def_schema():
return Schema([
{
Required('name'): Match(r'^[a-zA-Z0-9-_]+$',
msg=INVALID_CHARACTER_ERROR_MSG),
Required('start'): str,
Required('end'): str,
Required('pattern'): str,
Optional('fixupdict'): dict
}
])
def register():
schema = {
Required('name'):
All(Match(r'^[a-zA-Z0-9_.-]+$', msg='username has invalid symbol'),
Length(min=1, max=30, msg='username too long')),
Required('nickname'):
All(
str,
Length(min=1, max=30, msg='nickname too long'),
),
Required('email'):
All(Email(), msg='bad email format'),
Required('password'):
All(
str,
Length(min=6, msg='password less then 6 letters'),
),
}
payload = retrieve_payload(schema)
user = User.query.filter_by(name=payload['name']).first()
if user:
raise ConflictException(desc='username already used')
user = User.query.filter_by(email=payload['email']).first()
if user:
raise ConflictException(desc='email already used')
avatar = pagan.Avatar(payload['name'], pagan.SHA512)
upload_folder = current_app.config['UPLOAD_FOLDER']
filepath = os.path.join('avatar', uuid4().hex[:8])
avatar.save(os.path.join(upload_folder, filepath), '_.png')
avatar_path = '/' + os.path.join('upload', filepath, '_.png')
user = User(
role=User.ROLE_USER,
name=payload['name'],
nickname=payload['nickname'],
email=payload['email'],
avatar=avatar_path,
)
user.password = payload['password']
with db.auto_commit():
db.session.add(user)
confirm_url = 'https://morio.cc/confirm?token={}'.format(
user.gen_email_token())
mailgun.send_mail.delay(
user.email,
'Welcome to Morio',
html=render_template('email/confirm.html', url=confirm_url),
)
resp = jsonify(user)
resp.headers['Authorization'] = user.gen_auth_token()
return resp
def setUp(self):
self.graph_schema = Schema(
{
'scopes':
generate_scope_validator(scopes={'queue:task-priority:high'})
},
extra=True,
required=True)
self.task_schema = Schema(
{
'task': {
'provisionerId': 'aws-provisioner-v1',
'workerType': 'b2gtest',
'payload': {
'image': Match(r'^rail/python-test-runner'),
'command': [str],
'env': dict,
}
}
},
extra=True,
required=True)
test_kwargs = create_firefox_test_args({
'push_to_releases_enabled':
True,
'release_channels': ['beta', 'release'],
'final_verify_channels': ['beta', 'release'],
'final_verify_platforms':
["macosx64", "win32", "win64", "linux", "linux64"],
'signing_pvt_key':
PVT_KEY_FILE,
'en_US_config': {
"platforms": {
"macosx64": {},
"win32": {},
"win64": {},
"linux": {},
"linux64": {},
}
},
})
self.graph = make_task_graph(**test_kwargs)
self.tasks = [
get_task_by_name(self.graph,
"{chan}_final_verify".format(chan=chan))
for chan in (
'beta',
'release',
)
]
def schema(live=False):
common = {
Required("path"):
str,
Required("name"):
str,
Optional("skip_install"):
[Any("keys", "sources", "deps", "steps", "git-repos", "all")],
# Optional("parameters"):
}
base = {
Required("definitions"): [
Any(
{
Required("repository"): str,
Required("from"): Any("git", "bzr"),
Optional("branch"): str,
Optional("lava-signal"): Any("kmsg", "stdout"),
Optional("history"): bool,
Optional("revision"): str,
Optional("parameters"): {},
Optional("params"): {},
**common,
},
{
Required("repository"): {
Required("metadata"): dict, # TODO: what's required?
Optional("install"): {
Optional("deps"): [str],
Optional(Match(r"deps-.+")): [str],
Optional("git-repos"): [str],
Optional("keys"): [str],
Optional("steps"): [str],
Optional("sources"): [str],
},
Optional("parse"): {
Optional("fixupdict"): {
str: Any("pass", "fail", "skip", "unknown")
}
},
Optional("run"): {
Required("steps"): [str]
},
},
Required("from"): "inline",
**common,
},
)
]
}
return {**test.schema(live), **base}
def schema():
base = {
Required("monitors"): [{
Required("name"): Match(r"^[-_a-zA-Z0-9.]+$"),
Required("start"): str,
Required("end"): str,
Required("pattern"): str,
Optional("fixupdict"): {
str: Any("pass", "fail", "skip", "unknown")
},
}]
}
return {**test.schema(), **base}
def _monitor_def_schema():
return Schema([{
Required("name"):
Match(r"^[a-zA-Z0-9-_]+$", msg=INVALID_CHARACTER_ERROR_MSG),
Required("start"):
str,
Required("end"):
str,
Required("pattern"):
str,
Optional("fixupdict"):
dict,
}])
def sortable(response_class=None):
"""Decorator for enabling sort.
This decorator looks into the request for one or more `_sort` parameters
and maps them into a dictionary in which keys are column names and the
values are the ordering (either `asc` for ascending or `desc` descending).
The parameter values are expected to be strings with the column name
prefixed with either `+` or `-` to set the ordering (being `+` the default
in absence of any prefix).
Once the request parameters have been transformed into the dictionary
object it's passed as the `sort` parameter to the decorated function.
A `voluptuous.error.Invalid` exception will be raised if any of the request
parameters has an invalid value.
"""
fields = response_class.resource_fields if response_class else {}
schema = Schema(
[
Match(
# `@` allowed for compatibility with elasticsearch fields
r'[+-]?[\w@]+',
msg=(
'`_sort` parameter should be a column name '
'optionally prefixed with +/-'
),
),
],
extra=REMOVE_EXTRA,
)
def sortable_dec(func):
@wraps(func)
def create_sort_params(*args, **kw):
"""Validate sort parameters and pass them to the wrapped function.
"""
# maintain order of sort fields
sort_params = OrderedDict([
(
param.lstrip('+-'),
'desc' if param[0] == '-' else 'asc',
)
for param in schema(request.args.getlist('_sort'))
])
if fields:
_validate_fields(fields, sort_params, SORT)
return func(sort=sort_params, *args, **kw)
return create_sort_params
return sortable_dec
def _test_definition_schema():
return Schema([{
Required('repository'):
Any(_inline_schema(), str),
Required('from'):
str,
Required('name'):
Match(r'^[a-zA-Z0-9-_]+$', msg=INVALID_CHARACTER_ERROR_MSG),
Required('path'):
str,
Optional('parameters'):
dict,
}],
extra=True)
class SearchView(FhirDataView):
# Base class for FHIR resource search views
# BB2-149 note, check authenticated first, then app active etc.
permission_classes = [
permissions.IsAuthenticated,
ApplicationActivePermission,
ResourcePermission,
SearchCrosswalkPermission,
DataAccessGrantPermission,
TokenHasProtectedCapability,
]
# Regex to match a valid _lastUpdated value that can begin with lt, le, gt and ge operators
REGEX_LASTUPDATED_VALUE = r'^((lt)|(le)|(gt)|(ge)).+'
QUERY_TRANSFORMS = {
'count': '_count',
}
QUERY_SCHEMA = {
Required('startIndex', default=0):
Coerce(int),
Required('_count', default=DEFAULT_PAGE_SIZE):
All(Coerce(int), Range(min=0, max=MAX_PAGE_SIZE)),
'_lastUpdated': [
Match(REGEX_LASTUPDATED_VALUE,
msg="the _lastUpdated operator is not valid")
]
}
def __init__(self, version=1):
self.resource_type = None
super().__init__(version)
def initial(self, request, *args, **kwargs):
return super().initial(request, self.resource_type, *args, **kwargs)
def get(self, request, *args, **kwargs):
return super().get(request, self.resource_type, *args, **kwargs)
def build_url(self, resource_router, resource_type, *args, **kwargs):
if resource_router.fhir_url.endswith('v1/fhir/'):
# only if called by tests
return "{}{}/".format(resource_router.fhir_url, resource_type)
else:
return "{}/{}/fhir/{}/".format(resource_router.fhir_url,
'v2' if self.version == 2 else 'v1',
resource_type)
def setUp(self):
self.task_schema = Schema(
{
'task': {
'provisionerId': 'aws-provisioner-v1',
'workerType': 'b2gtest',
'payload': {
'command': [str],
'env': dict,
'image': Match(r'^rail/python-test-runner'),
}
}
},
extra=True,
required=True)
self.graph_schema = Schema(
{
'scopes':
generate_scope_validator(scopes={'queue:task-priority:high'}),
},
extra=True,
required=True)
test_args = create_firefox_test_args({
'push_to_releases_enabled':
True,
'uptake_monitoring_enabled':
True,
'release_channels': ['foo'],
'final_verify_channels': ['foo'],
'final_verify_platforms':
["macosx64", "win32", "win64", "linux", "linux64"],
'enUS_platforms':
["linux", "linux64", "win64", "win32", "macosx64"],
'signing_pvt_key':
PVT_KEY_FILE,
'accepted_mar_channel_id':
'firefox-mozilla-beta',
'signing_cert':
'dep',
'moz_disable_mar_cert_verification':
True,
'en_US_config':
EN_US_CONFIG,
})
self.graph = make_task_graph(**test_args)
self.task = get_task_by_name(self.graph, "foo_final_verify")
def schema():
base = {
Required("to"):
"mps",
Required("images"):
All(
{
Optional("recovery_image"):
deploy.url(),
Optional(Match("test_binary(_\\w+)?$")):
deploy.url({Optional("rename"): str}),
},
Length(min=1),
),
}
return {**deploy.schema(), **base}
def test_repr():
"""Verify that __repr__ returns valid Python expressions"""
match = Match('a pattern', msg='message')
replace = Replace('you', 'I', msg='you and I')
range_ = Range(min=0, max=42, min_included=False,
max_included=False, msg='number not in range')
coerce_ = Coerce(int, msg="moo")
all_ = All('10', Coerce(int), msg='all msg')
assert_equal(repr(match), "Match('a pattern', msg='message')")
assert_equal(repr(replace), "Replace('you', 'I', msg='you and I')")
assert_equal(
repr(range_),
"Range(min=0, max=42, min_included=False, max_included=False, msg='number not in range')"
)
assert_equal(repr(coerce_), "Coerce(int, msg='moo')")
assert_equal(repr(all_), "All('10', Coerce(int, msg=None), msg='all msg')")
def create_repo():
schema = {
Required('name'): All(
Match(r'^[a-zA-Z0-9_.-]+$',
msg='repository name has invalid symbol'),
Length(min=1, max=30, msg='repository name too long')
),
**common_repo_schema,
}
payload = retrieve_payload(schema)
src = Repository.query \
.filter_by(user_id=g.user.id, name=payload['name']) \
.first()
if src:
raise ConflictException(desc='repo already exist')
repo = Repository(user_id=g.user.id, **payload)
with db.auto_commit():
db.session.add(repo)
return jsonify(repo)
class SearchViewExplanationOfBenefit(SearchView):
# Class used for ExplanationOfBenefit resource search view
# Regex to match a valid type value
REGEX_TYPE_VALUE = r"(carrier)|" + \
r"(pde)|" + \
r"(dme)|" + \
r"(hha)|" + \
r"(hospice)|" + \
r"(inpatient)|" + \
r"(outpatient)|" + \
r"(snf)|" + \
r"(https://bluebutton.cms.gov/resources/codesystem/eob-type\|)|" + \
r"(https://bluebutton.cms.gov/resources/codesystem/eob-type\|carrier)|" + \
r"(https://bluebutton.cms.gov/resources/codesystem/eob-type\|pde)|" + \
r"(https://bluebutton.cms.gov/resources/codesystem/eob-type\|dme)|" + \
r"(https://bluebutton.cms.gov/resources/codesystem/eob-type\|hha)|" + \
r"(https://bluebutton.cms.gov/resources/codesystem/eob-type\|hospice)|" + \
r"(https://bluebutton.cms.gov/resources/codesystem/eob-type\|inpatient)|" + \
r"(https://bluebutton.cms.gov/resources/codesystem/eob-type\|outpatient)|" + \
r"(https://bluebutton.cms.gov/resources/codesystem/eob-type\|snf)"
# Regex to match a list of comma separated type values with IGNORECASE
REGEX_TYPE_VALUES_LIST = r'(?i)^((' + REGEX_TYPE_VALUE + r')\s*,*\s*)+$'
# Add type parameter to schema only for EOB
QUERY_SCHEMA = {
**SearchView.QUERY_SCHEMA, 'type':
Match(REGEX_TYPE_VALUES_LIST,
msg="the type parameter value is not valid")
}
def __init__(self, version=1):
super().__init__(version)
self.resource_type = "ExplanationOfBenefit"
def build_parameters(self, request, *args, **kwargs):
return {
'_format': 'application/json+fhir',
'patient': request.crosswalk.fhir_id,
}
def schema():
common = {
Required("path"): str,
Required("name"): Match(r"^[-_a-zA-Z0-9.]+$"),
Optional("skip_install"): [
Any("keys", "sources", "deps", "steps", "git-repos", "all")
],
# Optional("parameters"):
Optional("lava-signal"): Any("kmsg", "stdout"),
}
base = {
Required("definitions"): [
Any(
{
Required("repository"): str,
Required("from"): "git",
Optional("branch"): str,
Optional("history"): bool,
Optional("revision"): str,
Optional("parameters"): dict,
Optional("params"): dict,
**common,
},
{
Required("repository"): str,
Required("from"): "url",
Required("compression"): str,
Optional("parameters"): dict,
Optional("params"): dict,
**common,
},
{
Required("repository"): {**testdef()},
Required("from"): "inline",
**common,
},
)
]
}
return {**test.schema(), **base}
def test_encryption(self):
self.task_schema = Schema(
{
'task': {
'payload': {
'encryptedEnv': All(Length(2), [
Match(r'^wcB')
]) # Must have 2 elements, starting with wcB
}
}
},
required=True,
extra=True)
test_kwargs = create_fennec_test_args({
'updates_enabled': True,
'repo_path': 'foo/bar',
'branch': 'mozilla-beta',
'signing_class': 'dep-signing',
'release_channels': ['beta'],
'final_verify_channels': ['beta'],
'signing_pvt_key': PVT_KEY_FILE,
'en_US_config': {
"platforms": {
"android-4-0-armv7-api15": {
"unsigned_task_id": "xyz",
"signed_task_id": "xyz"
},
"android-4-2-x86": {
"unsigned_task_id": "xyz",
"signed_task_id": "xyz"
},
}
},
})
graph = make_task_graph(**test_kwargs)
do_common_assertions(graph)
PUB_KEY,
algorithms=[ALGORITHMS.RS512])['taskId']
@truth
def index_route_requirement(task_routes):
rel_routes = [r.startswith("index.releases.") for r in task_routes]
return len(rel_routes) >= 2
COMMON_TASK_SCHEMA = Schema(
All(
task_signature_test,
{ # Must pass task signature test, and the below Schema
'requires':
Any([Match(TASKCLUSTER_ID_REGEX)], None),
Required('reruns', msg="Required for releasetasks schema."):
Range(min=0, max=100),
Required('taskId', msg="Required for TaskCluster schema."):
Match(TASKCLUSTER_ID_REGEX),
Required('task', msg="Required for TaskCluster schema."):
All(
task_provisionerId_test,
Schema(
{
Required('created',
msg="Required for TaskCluster schema."):
str,
Required('deadline',
msg="Required for TaskCluster schema."):
str,
Required('DestinationPOPGroup'):
Any('Back Office', 'Lab', 'Border Router', 'Corporate Office', 'MCDN',
'Storage', 'Streaming', 'Edge', 'Internet', 'Internet Block', 'NAT',
'Staging', 'Internal Staging'),
Required('DestinationNetworkGroup'):
Any(None, str),
Required('DestinationPOP'): [Any(None, str)],
Required('DestinationPurpose'):
Any(None, str),
Required('DestinationSubPurpose'):
Any(None, str),
Required('DestinationVIP'):
Any(None, int),
Required('DestinationVLAN'): [Any(None, str)],
Required('DestinationProtocol'): [Any(str)],
Required('DestinationPort'): [Any('any', int, None, Match(r'^[0-9- ]+$'))],
Required('DestinationHostname'):
Any(None, str)
}
source = {
Required('SourceEnvironment'):
Any('EC', 'IR500', 'DLA'),
Required('SourcePOPGroup'):
Any('Back Office', 'Lab', 'Border Router', 'Corporate Office', 'MCDN',
'Storage', 'Streaming', 'Edge', 'Internet', 'Internet Block', 'NAT',
'Staging', 'Internal Staging'),
Required('SourceNetworkGroup'):
Any(None, str),
Required('SourcePOP'): [Any(None, str)],
Required('SourcePurpose'):
Any(None, str),
import os
import sys
import yaml
import gzip
import lzma
from voluptuous import Schema, Required, All, Any, Length, Range, Match, Url
from optparse import OptionParser
import multiprocessing as mp
schema_header = Schema({
Required('File'):
All(str, 'DEP-11', msg="Must be \"DEP-11\""),
Required('Origin'):
All(str, Length(min=1)),
Required('Version'):
All(str, Match(r'(\d+\.?)+$'), msg="Must be a valid version number"),
Required('MediaBaseUrl'):
All(str, Url()),
'Time':
All(str),
'Priority':
All(int),
})
schema_translated = Schema(
{
Required('C'):
All(str, Length(min=1), msg="Must have an unlocalized 'C' key"),
dict:
All(str, Length(min=1)),
},
Match,
message,
Optional,
Range,
Schema,
)
from .misc import (
make_ts,
parse_timedelta,
)
key = All(
str,
Length(min=1),
Match("^[a-zA-Z0-9-_@]+$"),
)
dotted_key = All(
str,
Length(min=1),
Match("^[a-zA-Z0-9-_@.]+$"),
)
seasonality = Schema({
Optional('daytime', default=False): Boolean(),
Optional('weekday', default=False): Boolean(),
})
score = Any(All(Any(int, float), Range(min=0, max=100)), None)
