def test_and():
assert And(int, lambda n: 0 < n < 5).validate(3) == 3
with SE: And(int, lambda n: 0 < n < 5).validate(3.33)
assert And(Use(int), lambda n: 0 < n < 5).validate(3.33) == 3
with SE: And(Use(int), lambda n: 0 < n < 5).validate('3.33')
class Config(SkeleYaml):
"""
Root Config Class for Skelebot YAML File
Built on top of the SkeleYaml parent Object in order to enherit and extend the functionality
of yaml file generation and parsing
"""
schema = Schema(
{
'name':
And(str, error='\'name\' must be a String'),
Optional('env'):
And(str, error='\'env\' must be a String'),
Optional('description'):
And(str, error='\'description\' must be a String'),
Optional('maintainer'):
And(str, error='\'maintainer\' must be a String'),
Optional('contact'):
And(str, error='\'contact\' must be a String'),
Optional('host'):
And(str, error='\'host\' must be a String'),
'language':
And(str, error='\'language\' must be a String'),
Optional('baseImage'):
And(str, error='\'baseImage\' must be a String'),
Optional('timezone'):
And(str, error='\'timezone\' must be a String'),
Optional('primaryJob'):
And(str, error='\'primaryJob\' must be a String'),
Optional('primaryExe'):
And(str,
Use(str.upper),
lambda s: s in ('CMD', 'ENTRYPOINT'),
error='\'primaryExe\' must be CMD or ENTRYPOINT'),
Optional('ephemeral'):
And(bool, error='\'ephemeral\' must be a Boolean'),
Optional('dependencies'):
Or(dict, list, error='\'dependencies\' must be a Dict or List'),
Optional('ignores'):
And(list, error='\'ignores\' must be a List'),
Optional('jobs'):
And(list, error='\'jobs\' must be a List'),
Optional('ports'):
And(list, error='\'ports\' must be a List'),
Optional('components'):
And(dict, error='\'components\' must be a Dictionary'),
Optional('params'):
And(list, error='\'params\' must be a List'),
Optional('commands'):
And(list, error='\'commands\' must be a List'),
Optional('pythonVersion'):
And(str,
Or(*PYTHON_VERSIONS),
error='\'pythonVersion\' must be one of:' +
', '.join(PYTHON_VERSIONS))
},
ignore_extra_keys=True)
name = None
env = None
description = None
version = None
maintainer = None
contact = None
host = None
language = None
baseImage = None
timezone = None
primaryJob = None
primaryExe = None
ephemeral = None
dependencies = None
ignores = None
jobs = None
ports = None
components = None
params = None
commands = None
pythonVersion = '3.6'
def __init__(self,
name=None,
env=None,
description=None,
version=None,
maintainer=None,
contact=None,
host=None,
language=None,
baseImage=None,
timezone=None,
primaryJob=None,
primaryExe=None,
ephemeral=None,
dependencies=None,
ignores=None,
jobs=None,
ports=None,
components=None,
params=None,
commands=None,
pythonVersion='3.6'):
"""Initialize the config object with all provided optional attributes"""
self.name = name
self.env = env
self.description = description
self.version = version
self.maintainer = maintainer
self.contact = contact
self.host = host
self.language = language
self.baseImage = baseImage
self.timezone = timezone
self.primaryJob = primaryJob
self.primaryExe = primaryExe.upper(
) if primaryExe is not None else "CMD"
self.ephemeral = ephemeral
self.dependencies = dependencies if dependencies is not None else []
self.ignores = ignores if ignores is not None else []
self.jobs = jobs if jobs is not None else []
self.ports = ports if ports is not None else []
self.components = components if components is not None else []
self.params = params if params is not None else []
self.commands = commands if commands is not None else []
self.pythonVersion = pythonVersion
def toDict(self):
"""
Extends the parent function in order to logic for handling the conversion of
components since the Class structure and yaml sctructures do not match
"""
components_dict = {}
for component in self.components:
component_dict = component.toDict()
if component_dict != {}:
components_dict[
component.__class__.__name__.lower()] = component_dict
dct = super().toDict()
dct["components"] = components_dict
return dct
def getBaseImage(self):
"""
Returns the proper base image based on the values for language and kerberos,
or returns the user defined base image if provided in the config yaml
"""
if self.baseImage:
image = self.baseImage
else:
language = self.language if self.language is not None else "NA"
image = LANGUAGE_IMAGE[language]
if language == 'Python':
image['base'] = image['base'].format(
pythonVersion=self.pythonVersion)
variant = "base"
for component in self.components:
if component.__class__.__name__.lower() == "kerberos":
variant = "krb"
image = image[variant]
return image
def getImageName(self):
"""Construct and return the name for the docker image based on the project name"""
image_name = self.name.lower().replace(" ", "-")
if self.env:
image_name += "-{env}".format(env=self.env)
return image_name
def getHost(self, job=None, args=None):
host = self.host if self.host is not None else None
if job is not None:
host = job.host if job.host is not None else host
if args is not None:
host = args.host if getattr(args, 'host',
None) is not None else host
return host
def loadComponents(self, config):
"""
Parses the components section of skelebot.yaml config to generate the complete list of
components for the project based on the active component list and each components'
Activation attribute
"""
componentFactory = ComponentFactory()
if (config is None):
# Build the default components for a non-skelebot project
defaultActivations = [Activation.EMPTY, Activation.ALWAYS]
self.components = componentFactory.buildComponents(
defaultActivations)
else:
# Build the components that are defined in the config yaml data
compNames = []
components = []
if ("components" in config):
configComps = config["components"]
for compName in configComps:
# Ensures that artifactory component is not loaded if repository is present
if ("repository"
not in configComps) or (compName != "artifactory"):
component = componentFactory.buildComponent(
compName, configComps[compName])
if (component is not None):
components.append(component)
compNames.append(component.__class__.__name__)
# Build the additonal components that are active without configuration data
activations = [Activation.PROJECT, Activation.ALWAYS]
components.extend(
componentFactory.buildComponents(activations,
ignores=compNames))
self.components = components
@classmethod
def load(cls, config):
"""Load the config Dict from the yaml file into the Config object"""
cfg = cls()
if config is not None:
cls.validate(config)
values = {}
for attr, value in config.items():
if (attr in vars(Config)) and (attr != "components") and (
attr != "version"):
if (attr == "jobs"):
values[attr] = Job.loadList(value)
elif (attr == "params"):
values[attr] = Param.loadList(value)
else:
values[attr] = value
cfg = cls(**values)
cfg.loadComponents(config)
return cfg
def get(self):
params = self.validate_argument(
{Optional("page", default=None): Use(int)})
posts = PostService.get_posts(**params)
self.success(posts)
print('Sent %s messages in %.3fs' % (count, time.time() - start))
# Close all of those connections.
for client in clients:
client.close()
if __name__ == '__main__':
options = docopt(main.__doc__)
try:
options = Schema({
'--client-id':
And(str, error='--client-id is required'),
'--token-file':
Or(None, Use(str, error='--token-file should be path')),
'--host':
str,
'--port':
Use(int),
'--count':
Use(int),
'--sleep':
Use(float),
'--connections':
Use(int),
'--word-count':
Use(int),
object:
object,
}).validate(options)
ActionsApi = Blueprint(
'ActionsApi',
__name__,
description='Endpoints for API methods not associated with `arn`.')
class NextflowConfigParseArgs(m.Schema):
class Meta:
unknown = m.INCLUDE
nextflow_config = m.fields.String(location="json")
valid_schema = Schema({
'workDir': And(Use(str), lambda s: s.startswith("s3://")),
'process': {
'scratch': '/docker_scratch',
'queue': And(Use(str)),
'executor': 'awsbatch',
Optional('container'): And(Use(str))
},
'aws': {
'region': 'us-west-2',
'batch': {
'volumes': '/docker_scratch',
'cliPath': '/home/ec2-user/miniconda/bin/aws'
}
}
})
def __init__(self, name, unvalidated_tree, sections, config):
"""Fix up settings that depend on the [DXR] section or have
inter-setting dependencies. (schema can't do multi-setting validation
yet, and configobj can't do cross-section interpolation.)
Add a ``config`` attr to trees as a shortcut back to the [DXR] section
and a ``name`` attr to save cumbersome tuple unpacks in callers.
"""
self.config = config
self.name = name
schema = Schema({
Optional('build_command', default='make -j {workers}'): basestring,
Optional('clean_command', default='make clean'): basestring,
Optional('description', default=''): basestring,
Optional('disabled_plugins', default=plugin_list('')): Plugins,
Optional('enabled_plugins', default=plugin_list('*')): Plugins,
Optional('es_index', default=config.es_index): basestring,
Optional('es_shards', default=5):
Use(int, error='"es_shards" must be an integer.'),
Optional('ignore_patterns',
default=['.hg', '.git', 'CVS', '.svn', '.bzr',
'.deps', '.libs', '.DS_Store', '.nfs*', '*~',
'._*']): WhitespaceList,
Optional('object_folder', default=None): AbsPath,
'source_folder': AbsPath,
Optional('source_encoding', default='utf-8'): basestring,
Optional('temp_folder', default=None): AbsPath,
Optional('p4web_url', default='http://p4web/'): basestring,
Optional('workers', default=None): WORKERS_VALIDATOR,
Optional(basestring): dict})
tree = schema.validate(unvalidated_tree)
if tree['temp_folder'] is None:
tree['temp_folder'] = config.temp_folder
if tree['object_folder'] is None:
tree['object_folder'] = tree['source_folder']
if tree['workers'] is None:
tree['workers'] = config.workers
# Convert enabled_plugins to a list of plugins:
if tree['disabled_plugins'].is_all:
# * doesn't really mean "all" in a tree. It means "everything the
# [DXR] section enabled".
tree['disabled_plugins'] = config.enabled_plugins
else:
# Add anything globally disabled to our local disabled list:
tree['disabled_plugins'].extend(p for p in config.disabled_plugins
if p not in
tree['disabled_plugins'])
if tree['enabled_plugins'].is_all:
tree['enabled_plugins'] = [p for p in config.enabled_plugins
if p not in tree['disabled_plugins']]
tree['enabled_plugins'].insert(0, core_plugin())
# Split ignores into paths and filenames:
tree['ignore_paths'] = [i for i in tree['ignore_patterns']
if i.startswith('/')]
tree['ignore_filenames'] = [i for i in tree['ignore_patterns']
if not i.startswith('/')]
# Delete misleading, useless, or raw values people shouldn't use:
del tree['ignore_patterns']
del tree['disabled_plugins']
# Validate plugin config:
enableds_with_all_optional_config = set(
p for p in tree['enabled_plugins']
if all(isinstance(k, Optional) for k in p.config_schema.iterkeys()))
plugin_schema = Schema(merge(
dict((Optional(name) if plugin in enableds_with_all_optional_config
or plugin not in tree['enabled_plugins']
else name,
plugin.config_schema)
for name, plugin in all_plugins_but_core().iteritems()),
# And whatever isn't a plugin section, that we don't care about:
{object: object}))
# Insert empty missing sections for enabled plugins with entirely
# optional config so their defaults get filled in. (Don't insert them
# if the plugin has any required options; then we wouldn't produce the
# proper error message about the section being absent.)
for plugin in enableds_with_all_optional_config:
tree.setdefault(plugin.name, {})
tree = plugin_schema.validate(tree)
super(TreeConfig, self).__init__(tree)
def main(argv, session=None):
args = docopt(__doc__, argv=argv)
# Validate args.
s = Schema({
six.text_type:
Use(bool),
'<query>':
Use(lambda x: ' '.join(x)),
'--parameters':
Use(lambda x: get_args_dict(x, query_string=True)),
'--header':
Or(None,
And(Use(get_args_dict), dict),
error='--header must be formatted as --header="key:value"'),
'--sort':
list,
'--field':
list,
'--timeout':
Use(lambda x: float(x[0]), error='--timeout must be integer or float.')
})
try:
args = s.validate(args)
except SchemaError as exc:
print('{0}\n{1}'.format(str(exc), printable_usage(__doc__)),
file=sys.stderr)
sys.exit(1)
# Support comma separated values.
fields = list(chain.from_iterable([x.split(',') for x in args['--field']]))
sorts = list(chain.from_iterable([x.split(',') for x in args['--sort']]))
r_kwargs = dict(
headers=args['--header'],
timeout=args['--timeout'],
)
search = session.search_items(args['<query>'],
fields=fields,
sorts=sorts,
params=args['--parameters'],
request_kwargs=r_kwargs)
try:
if args['--num-found']:
print('{0}'.format(search.num_found))
sys.exit(0)
for result in search:
if args['--itemlist']:
print(result.get('identifier', ''))
else:
j = json.dumps(result)
print(j)
except ValueError as e:
print('error: {0}'.format(e), file=sys.stderr)
except ConnectTimeout as exc:
print(
'error: Request timed out. Increase the --timeout and try again.',
file=sys.stderr)
sys.exit(1)
except AuthenticationError as exc:
print('error: {}'.format(exc), file=sys.stderr)
sys.exit(1)
def validation_setup():
# There's a mix of GET, PUT and POST in these; mixing it up may give us
# better coverage. The particulars of which PUT and POST calls get which
# methods are arbitrary, though GET calls have some specificity.
# We have several kinds of calls we want to validate here:
# 1. No arguments in the URL or body (no_args)
@rest.rest_call(['GET', 'POST'], '/no/args', Schema({}))
# pylint: disable=unused-variable
def no_args():
pass
# 2. Argument in the URL and not in the body (url_args)
@rest.rest_call(['GET', 'POST'], '/url/args/<arg1>/<arg2>',
Schema({
'arg1': basestring,
'arg2': basestring,
}))
# pylint: disable=unused-variable
def url_args(arg1, arg2):
return json.dumps([arg1, arg2])
# 3. Arguments in both the URL and body (mixed_args)
@rest.rest_call(['GET', 'PUT'], '/mixed/args/<arg1>',
Schema({
'arg1': basestring,
'arg2': basestring,
}))
# pylint: disable=unused-variable
def mixed_args(arg1, arg2):
return json.dumps([arg1, arg2])
# 4. Arguments in body (query parameters for GET) and not in the URL.
@rest.rest_call(['GET', 'POST'], '/just/bodyargs',
Schema({
'arg1': basestring,
'arg2': basestring,
}))
# pylint: disable=unused-variable
def just_bodyargs(arg1, arg2):
return json.dumps([arg1, arg2])
# 5. One optional argument.
@rest.rest_call(['GET', 'POST'], '/just/bodyargs/optional_arg2_int',
Schema({
'arg1': basestring,
Optional('arg2'): Use(int),
}))
# pylint: disable=unused-variable
def bodyargs_optional_arg2_int(arg1, arg2=-42):
return json.dumps([arg1, arg2])
# Let's also make sure we're testing something with a schema that isn't
# just basestring:
@rest.rest_call('PUT',
'/put-int-body-arg',
schema=Schema({"the_value": int}))
# pylint: disable=unused-variable
def put_int_body_arg(the_value):
return json.dumps(the_value)
@rest.rest_call('GET',
'/get-int-body-arg-with-use',
schema=Schema({"the_value": Use(int)}))
# pylint: disable=unused-variable
def get_int_body_arg_with_use(the_value):
return json.dumps(the_value)
# coding=utf-8
# __author__ = 'Mio'
import re
from schema import And, Use, Or
from tornado.escape import utf8
def power_split(value, separator=',', schema=str):
assert callable(schema)
value = utf8(value)
value = value.strip()
l = re.split("\s*" + separator + "\s*", value) # 这个slip直接去除逗号左右的空格
return [v for v in l if v != '']
schema_utf8 = And(Or(Use(utf8), Use(str)), len) # 非空
schema_utf8_none = Or(Use(utf8), Use(str))
schema_int = Use(int)
schema_float = Use(float)
# MongoDB object_id
schema_object_id = And(schema_utf8, lambda x: len(x) == 24)
def play_quizzes():
body = request.get_json()
if not body:
abort(400, 'no json body was found')
schema = Schema({
'previous_questions': [
And(Use(int),
lambda question: Question.query.get(question) is not None)
],
Optional('quiz_category'):
And(Use(int),
lambda category: Category.query.get(category) is not None)
})
# validate quiz input
quiz_data = {}
try:
quiz_data = schema.validate(body)
except:
abort(400, 'quiz input was bad or not formatted correctly')
prev_questions = [
Question.query.get(question)
for question in quiz_data['previous_questions']
]
if 'quiz_category' in quiz_data:
category = Category.query.get(quiz_data['quiz_category'])
for question in prev_questions:
if question.category_id != category.id:
abort(400, 'a question does not belong to category')
total_questions = len(category.questions) - len(prev_questions)
if total_questions == 0:
return jsonify({
'success': True,
'total_questions': total_questions,
'categoy': category.id
})
question = list(
filter(lambda question: question not in prev_questions,
category.questions))[random.randint(
0, total_questions - 1)]
return jsonify({
'success': True,
'question': question.format(),
'total_questions': total_questions,
'categoy': category.id
})
else:
questions = Question.query.all()
total_questions = len(questions) - len(prev_questions)
if total_questions == 0:
return jsonify({
'success': True,
'total_questions': total_questions,
})
question = list(
filter(lambda question: question not in prev_questions,
questions))[random.randint(0, total_questions - 1)]
return jsonify({
'success': True,
'question': question.format(),
'total_questions': total_questions,
})
__ver__ = "0.4.0"
if __name__ == "__main__":
arguments = docopt(__doc__, version=__ver__)
schema = Schema({
'<script>':
And(
And(path.isfile, error='Script should exist!'),
And(lambda f: f.endswith('.py'),
error='Script should be a python file')),
'-o':
str,
'--props':
Or(None, Use(json.loads, error='Invalid JSON')),
'--verbose':
bool,
'--help':
False,
'--version':
False
})
try:
arguments = schema.validate(arguments)
except SchemaError as e:
sys.exit(e)
if arguments['--verbose']:
logging.basicConfig(level=logging.DEBUG)
def main(argv, session):
args = docopt(__doc__, argv=argv)
# Validation error messages.
destdir_msg = '--destdir must be a valid path to a directory.'
itemlist_msg = '--itemlist must be a valid path to an existing file.'
# Validate args.
s = Schema({
str: Use(bool),
'--destdir': Or([], And(Use(lambda d: d[0]), dir_exists), error=destdir_msg),
'--format': list,
'--glob': Use(lambda l: l[0] if l else None),
'<file>': list,
'--search': Or(str, None),
'--itemlist': Or(None, And(lambda f: os.path.isfile(f)), error=itemlist_msg),
'<identifier>': Or(str, None),
'--retries': Use(lambda x: x[0]),
'--search-parameters': Use(lambda x: get_args_dict(x, query_string=True)),
'--on-the-fly': Use(bool),
'--no-change-timestamp': Use(bool),
'--download-history': Use(bool),
'--parameters': Use(lambda x: get_args_dict(x, query_string=True)),
})
# Filenames should be unicode literals. Support PY2 and PY3.
if six.PY2:
args['<file>'] = [f.decode('utf-8') for f in args['<file>']]
try:
args = s.validate(args)
if args['--glob']:
if args['--format']:
raise(SchemaError(None, '--glob and --format cannot be used together.'))
except SchemaError as exc:
print('{0}\n{1}'.format(
str(exc), printable_usage(__doc__)), file=sys.stderr)
sys.exit(1)
retries = int(args['--retries'])
if args['--itemlist']:
with open(args['--itemlist']) as fp:
ids = [x.strip() for x in fp]
total_ids = len(ids)
elif args['--search']:
try:
_search = session.search_items(args['--search'],
params=args['--search-parameters'])
total_ids = _search.num_found
if total_ids == 0:
print('error: the query "{0}" returned no results'.format(
args['--search']), file=sys.stderr)
sys.exit(1)
ids = _search
except ValueError as e:
print('error: {0}'.format(e), file=sys.stderr)
sys.exit(1)
# Download specific files.
if args['<identifier>'] and args['<identifier>'] != '-':
if '/' in args['<identifier>']:
identifier = args['<identifier>'].split('/')[0]
files = ['/'.join(args['<identifier>'].split('/')[1:])]
else:
identifier = args['<identifier>']
files = args['<file>']
total_ids = 1
ids = [identifier]
elif args['<identifier>'] == '-':
total_ids = 1
ids = sys.stdin
files = None
else:
files = None
errors = list()
for i, identifier in enumerate(ids):
if args['--stdout']:
item = session.get_item(identifier)
f = list(item.get_files(args['<file>']))
try:
assert len(f) == 1
except AssertionError:
print('error: {0}/{1} does not exist!'.format(
identifier, args['<file>'][0]), file=sys.stderr)
sys.exit(1)
if six.PY2:
stdout_buf = sys.stdout
else:
stdout_buf = sys.stdout.buffer
f[0].download(retries=args['--retries'],
fileobj=stdout_buf,
params=args['--parameters'])
sys.exit(0)
try:
identifier = identifier.strip()
except AttributeError:
identifier = identifier.get('identifier')
if total_ids > 1:
item_index = '{0}/{1}'.format((i + 1), total_ids)
else:
item_index = None
try:
item = session.get_item(identifier)
except Exception as exc:
print('{0}: failed to retrieve item metadata - errors'.format(identifier),
file=sys.stderr)
if 'You are attempting to make an HTTPS' in str(exc):
print('\n{0}'.format(exc), file=sys.stderr)
sys.exit(1)
else:
continue
# Otherwise, download the entire item.
ignore_history_dir = True if not args['--download-history'] else False
_errors = item.download(
files=files,
formats=args['--format'],
glob_pattern=args['--glob'],
dry_run=args['--dry-run'],
verbose=args['--verbose'],
silent=args['--silent'],
ignore_existing=args['--ignore-existing'],
checksum=args['--checksum'],
destdir=args['--destdir'],
no_directory=args['--no-directories'],
retries=retries,
item_index=item_index,
ignore_errors=True,
on_the_fly=args['--on-the-fly'],
no_change_timestamp=args['--no-change-timestamp'],
params=args['--parameters'],
ignore_history_dir=ignore_history_dir,
)
if _errors:
errors.append(_errors)
if errors:
# TODO: add option for a summary/report.
sys.exit(1)
else:
sys.exit(0)
def validate_args(args):
"""Validate the input parameters."""
schema = Schema({
'--ip': And(str, Regex(r'^(?:[0-9]{1,3}\.){3}[0-9]{1,3}$'), error="Invalid input for `--ip`. Please enter a valid IP address. Example: `127.0.0.1`."),
'--port': And(Use(int), lambda x: 1024 <= x <= 65535, error="Invalid input for `--port`. Please enter a number between 1024 and 65535."),
'--date': Or(None, And(str, Regex(r'^\s*(3[01]|[12][0-9]|0?[1-9])\.(1[012]|0?[1-9])\.((?:19|20)\d{2})\s*$')), error="Invalid input for `--date`. Please enter a valid date. Example: `01.01.2020`."),
'--order-date': Or(None, And(str, Regex(r'^\s*(3[01]|[12][0-9]|0?[1-9])\.(1[012]|0?[1-9])\.((?:19|20)\d{2})\s*$')), error="Invalid input for `--order-date`. Please enter a valid date. Example: `01.01.2020`."),
'--event-date': Or(None, And(str, Regex(r'^\s*(3[01]|[12][0-9]|0?[1-9])\.(1[012]|0?[1-9])\.((?:19|20)\d{2})\s*$')), error="Invalid input for `--event-date`. Please enter a valid date. Example: `01.01.2020`."),
'--quantity': Or(None, And(Use(int)), error="Invalid input for `--quantity`. Please enter an integer number. Example: `3`."),
'--name': Or(None, And(Use(str))),
'--location': Or(None, And(str, Regex(r'^([^0-9]+) ([0-9]+.*?), ([0-9]{5}) (.*)$')), error="Invalid input for `--location`. Please enter a valid address. Example: `Friedrich Ebert Straße 30, 78054 Villingen-Schwenningen`."),
'--address': Or(None, And(str, Regex(r'^([^0-9]+) ([0-9]+.*?), ([0-9]{5}) (.*)$')), error="Invalid input for `--address`. Please enter a valid address. Example: `Friedrich Ebert Straße 30, 78054 Villingen-Schwenningen`."),
'--ticket-price': Or(None, And(Use(str), Regex(r'^\s*(?=.*[1-9])\d*(?:\.\d{1,2})?\s*$')), error="Invalid input for `--ticket-price`. Please enter a valid amount of money. Example: `5.60`."),
'--max-tickets': Or(None, And(Use(int)), error="Invalid input for `--max-tickets`. Please enter an integer number. Example: `3`."),
'--max-tickets-per-customer': Or(None, And(Use(int)), error="Invalid input for `--max-tickets-per-customer`. Please enter an integer number. Example: `3`."),
'--sale-start-date': Or(None, And(str, Regex(r'^\s*(3[01]|[12][0-9]|0?[1-9])\.(1[012]|0?[1-9])\.((?:19|20)\d{2})\s*$')), error="Invalid input for `--sale-start-date`. Please enter a valid date. Example: `01.01.2020`."),
'--sale-period': Or(None, And(Use(int)), error="Invalid input for `--sale-period`. Please enter the number of days as an integer number. Example: `3`."),
'--budget': Or(None, And(Use(str), Regex(r'^\s*(?=.*[1-9])\d*(?:\.\d{1,2})?\s*$')), error="Invalid input for `--budget`. Please enter a valid amount of money. Example: `5.60`."),
'<event-id>': Or(None, And(Use(int)), error="Invalid input for `<event-id>`. The event ID can only be an integer number. Example: `3`."),
'<customer-id>': Or(None, And(Use(int)), error="Invalid input for `<customer-id>`. The customer ID can only be an integer number. Example: `3`."),
'--year': Or(None, And(Use(int)), error="Invalid input for `--year`. Please enter an integer number. Example: `2020`.")
})
if args.get('--date') is not None and args.get('--sale-start-date') is not None:
event_date = DateHelper.date_to_timestamp(args.get('--date'))
sale_start_date = DateHelper.date_to_timestamp(
args.get('--sale-start-date'))
if sale_start_date > event_date:
raise SchemaError(
None, errors="Invalid input for `--date` or `--sale-start-date`. The event date cannot be earlier than the start date of the sale. Please check your input.")
try:
schema.validate(args)
except SchemaWrongKeyError as ex:
pass
except SchemaError as ex:
sys.exit(ex)
def main(argv, session):
args = docopt(__doc__, argv=argv)
# Validate args.
s = Schema({
six.text_type: bool,
'<identifier>': list,
'--modify': list,
'--header': Or(None, And(Use(get_args_header_dict), dict),
error='--header must be formatted as --header="key:value"'),
'--append': list,
'--append-list': list,
'--remove': list,
'--spreadsheet': Or(None, And(lambda f: os.path.exists(f),
error='<file> should be a readable file or directory.')),
'--target': Or(None, str),
'--priority': Or(None, Use(int, error='<priority> should be an integer.')),
})
try:
args = s.validate(args)
except SchemaError as exc:
print('{0}\n{1}'.format(str(exc), printable_usage(__doc__)), file=sys.stderr)
sys.exit(1)
formats = set()
responses = []
for i, identifier in enumerate(args['<identifier>']):
item = session.get_item(identifier)
# Check existence of item.
if args['--exists']:
if item.exists:
responses.append(True)
print('{0} exists'.format(identifier))
else:
responses.append(False)
print('{0} does not exist'.format(identifier), file=sys.stderr)
if (i + 1) == len(args['<identifier>']):
if all(r is True for r in responses):
sys.exit(0)
else:
sys.exit(1)
# Modify metadata.
elif (args['--modify'] or args['--append'] or args['--append-list']
or args['--remove']):
if args['--modify']:
metadata_args = args['--modify']
elif args['--append']:
metadata_args = args['--append']
elif args['--append-list']:
metadata_args = args['--append-list']
if args['--remove']:
metadata_args = args['--remove']
try:
metadata = get_args_dict(metadata_args)
if any('/' in k for k in metadata):
metadata = get_args_dict_many_write(metadata)
except ValueError:
print("error: The value of --modify, --remove, --append or --append-list "
"is invalid. It must be formatted as: --modify=key:value",
file=sys.stderr)
sys.exit(1)
if args['--remove']:
responses.append(remove_metadata(item, metadata, args))
else:
responses.append(modify_metadata(item, metadata, args))
if (i + 1) == len(args['<identifier>']):
if all(r.status_code == 200 for r in responses):
sys.exit(0)
else:
for r in responses:
if r.status_code == 200:
continue
# We still want to exit 0 if the non-200 is a
# "no changes to xml" error.
elif 'no changes' in r.content.decode('utf-8'):
continue
else:
sys.exit(1)
# Get metadata.
elif args['--formats']:
for f in item.get_files():
formats.add(f.format)
if (i + 1) == len(args['<identifier>']):
print('\n'.join(formats))
# Dump JSON to stdout.
else:
metadata = json.dumps(item.item_metadata)
print(metadata)
# Edit metadata for items in bulk, using a spreadsheet as input.
if args['--spreadsheet']:
if not args['--priority']:
args['--priority'] = -5
with io.open(args['--spreadsheet'], 'rU', newline='', encoding='utf-8') as csvfp:
spreadsheet = csv.DictReader(csvfp)
responses = []
for row in spreadsheet:
if not row['identifier']:
continue
item = session.get_item(row['identifier'])
if row.get('file'):
del row['file']
metadata = dict((k.lower(), v) for (k, v) in row.items() if v)
responses.append(modify_metadata(item, metadata, args))
if all(r.status_code == 200 for r in responses):
sys.exit(0)
else:
for r in responses:
if r.status_code == 200:
continue
# We still want to exit 0 if the non-200 is a
# "no changes to xml" error.
elif 'no changes' in r.content.decode('utf-8'):
continue
else:
sys.exit(1)
Optional("accessRights"): Or(str, [str]),
Optional("accessRightsComments"): Or(str, [str]),
Optional("bucket"): str,
Optional("byteSize"): Or(str, int),
Optional("category"): str,
Optional("creator"): Or(str, {
"name": str,
"email": str,
}),
Optional("contactpoint"): [],
Optional("code"): str,
Optional("description"): str,
Optional("distribution"): [],
Optional("format"): str,
Optional("identifier"): str,
Optional("issued"): And(Use(is_datetime_string)),
Optional("keyword"): Or(str, [str]),
Optional("keywords"): Or(str, [str]),
Optional("landingPage"): str,
Optional("language"): str,
Optional("license"): Or(str, {
"name": str,
"url": str,
}),
Optional("modified"): And(Use(is_datetime_string)),
Optional("notebook"): str,
Optional("provenance"): Or(str, object),
Optional("publisher"): str,
Optional("repo"): str,
Optional("rights"): str,
Optional("sample"): [str],
from functools import partial
from os.path import abspath
from schema import Optional, Use, And
from dxr.config import AbsPath
from dxr.plugins import Plugin, AdHocTreeToIndex, filters_from_namespace, refs_from_namespace
from dxr.plugins.xpidl import filters, refs
from dxr.plugins.xpidl.mappings import mappings
from dxr.plugins.xpidl.indexers import FileToIndex
def split_on_space_into_abspaths(value):
return map(abspath, value.strip().split())
ColonPathList = And(basestring,
Use(split_on_space_into_abspaths),
error='This should be a space-separated list of paths.')
plugin = Plugin(tree_to_index=partial(AdHocTreeToIndex,
file_to_index_class=FileToIndex),
refs=refs_from_namespace(refs.__dict__),
filters=filters_from_namespace(filters.__dict__),
badge_colors={'xpidl': '#DAF6B9'},
mappings=mappings,
config_schema={
'header_path': AbsPath,
Optional('include_folders', default=[]): ColonPathList
})
from funcy import merge
from hashlib import sha1
from more_itertools import first
from pkg_resources import resource_string
from schema import Optional, Use, And, Schema, SchemaError
from dxr.exceptions import ConfigError
from dxr.plugins import all_plugins_but_core, core_plugin
from dxr.utils import cd, if_raises
# Format version, signifying the instance format this web frontend code is
# able to serve. Must match exactly; deploy will do nothing until it does.
FORMAT = resource_string('dxr', 'format').strip()
WORKERS_VALIDATOR = And(Use(int),
lambda v: v >= 0,
error='"workers" must be a non-negative integer.')
class DotSection(object):
"""In the absense of an actual attribute, let attr lookup fall through to
``self._section[attr]``."""
def __getattr__(self, attr):
if not hasattr(self, '_section'): # Happens during unpickling
raise AttributeError(attr)
try:
val = self._section[attr]
except KeyError:
raise AttributeError(attr)
from schema import Schema, And, Use, Optional, Or, Regex
import re
import sys
inp_spec_schema = Schema({'format': And(str, Use(str.lower), lambda s: s in ('mxf', 'ts', 'mov', 'mp4', 'mpg')),
'vcodec': And(str, Use(str.lower), lambda s: s in ('h264', 'mpeg2video', 'copy')),
'acodec': And(str, Use(str.lower), lambda s: s in ('mp2', 'aac', 'pcm_s24le', 'ac3', 'copy')),
'n_out_aud_tracks': And(Use(int), lambda n: 1 <= n <= 16),
'aud_ch': And(Use(int), lambda n: 1 <= n <= 2),
'gop_length': And(Use(int)),
'vid_out_resolution': And(str, Use(str.lower), lambda s: s in ('1920x1080', '1280x720', '720x576',\
'720x480', '640x480')),
'vid_out_bitrate': Or(And(Use(str)), And(Regex('[0-9\.]*[kK]'), Use(str)),\
And(Regex('[0-9\.]*[M]'), Use(str))),
'vid_out_fps': And(str, Use(float), lambda f: f in [25.0, 29.97, 30.0]),
#Optional(Regex('aud_map')): Or(((Use(int),Use(int), Use(int))), Use(tuple(Use(int), Use(int)))),
Optional(Regex('aud_map')): Or(And(str, Use(eval), lambda a: 1<=a[0]<16 and 1<=a[1]<=16),\
And(str, Use(eval), lambda a: 1<=a[0][0]<16 and 1<=a[0][1]<=16 and 1<=a[1]<=16)),
'vid_inp_scan_type': And(str, Use(str.lower), lambda s: s in ('progressive', 'interlaced')),
'n_inp_aud_tracks': And(Use(int), lambda n: 1 <= n <=16),
'streamid_vid': And(Use(int)),
'streamid_aud_start': And(Use(int))
})
def _get_vid_bitrate_(s, d):
if s[-1] in 'kK':
o_bitrate = int(s[:-1]) * 1000
elif d['vid_out_bitrate'][-1] in 'M':
def __init__(self, input, relative_to=None):
"""Pull in and validate a config file.
:arg input: A string or dict from which to populate the config
:arg relative_to: The dir relative to which to interpret relative paths
Raise ConfigError if the configuration is invalid.
"""
if not relative_to:
relative_to = getcwd()
self.path = relative_to
with cd(relative_to):
schema = Schema({
'DXR': {
Optional('temp_folder', default=abspath('dxr-temp-{tree}')):
AbsPath,
Optional('default_tree', default=None): basestring,
Optional('disabled_plugins', default=plugin_list('')): Plugins,
Optional('enabled_plugins', default=plugin_list('*')): Plugins,
Optional('generated_date',
default=datetime.utcnow()
.strftime("%a, %d %b %Y %H:%M:%S +0000")):
basestring,
Optional('log_folder', default=abspath('dxr-logs-{tree}')):
AbsPath,
Optional('workers', default=if_raises(NotImplementedError,
cpu_count,
1)):
WORKERS_VALIDATOR,
Optional('skip_stages', default=[]): WhitespaceList,
Optional('www_root', default=''): Use(lambda v: v.rstrip('/')),
Optional('google_analytics_key', default=''): basestring,
Optional('es_hosts', default='http://127.0.0.1:9200/'):
WhitespaceList,
# A semi-random name, having the tree name and format version in it.
Optional('es_index', default='dxr_{format}_{tree}_{unique}'):
basestring,
Optional('es_alias', default='dxr_{format}_{tree}'):
basestring,
Optional('es_catalog_index', default='dxr_catalog'):
basestring,
Optional('es_catalog_replicas', default=1):
Use(int, error='"es_catalog_replicas" must be an integer.'),
Optional('max_thumbnail_size', default=20000):
And(Use(int),
lambda v: v >= 0,
error='"max_thumbnail_size" must be a non-negative '
'integer.'),
Optional('es_indexing_timeout', default=60):
And(Use(int),
lambda v: v >= 0,
error='"es_indexing_timeout" must be a non-negative '
'integer.'),
Optional('es_indexing_retries', default=0):
And(Use(int),
lambda v: v >= 0,
error='"es_indexing_retries" must be a non-negative '
'integer.'),
Optional('es_refresh_interval', default=60):
Use(int, error='"es_refresh_interval" must be an integer.')
},
basestring: dict
})
# Parse the ini into nested dicts:
config_obj = ConfigObj(input.splitlines() if isinstance(input,
basestring)
else input,
list_values=False)
try:
config = schema.validate(config_obj.dict())
except SchemaError as exc:
raise ConfigError(exc.code, ['DXR'])
self._section = config['DXR']
# Normalize enabled_plugins:
if self.enabled_plugins.is_all:
# Then explicitly enable anything that isn't explicitly
# disabled:
self._section['enabled_plugins'] = [
p for p in all_plugins_but_core().values()
if p not in self.disabled_plugins]
# Now that enabled_plugins and the other keys that TreeConfig
# depends on are filled out, make some TreeConfigs:
self.trees = OrderedDict() # name -> TreeConfig
for section in config_obj.sections:
if section != 'DXR':
try:
self.trees[section] = TreeConfig(section,
config[section],
config_obj[section].sections,
self)
except SchemaError as exc:
raise ConfigError(exc.code, [section])
self._section['es_catalog_index'] = self._section['es_catalog_index'].format(
config_path_hash=self.path_hash())
# Make sure default_tree is defined:
if not self.default_tree:
self._section['default_tree'] = first(self.trees.iterkeys())
# These aren't intended for actual use; they're just to influence
# enabled_plugins of trees, and now we're done with them:
del self._section['enabled_plugins']
del self._section['disabled_plugins']
import asyncio
import time
import var as V
from knlAgentIntr import KnlAgentIntr
from knlAgentWithoutIpq import KnlAgentWithoutIpq
from devAgent import DevAgent
from appAgent import AppAgent
from sndAgent import SndAgent
if __name__ == "__main__":
args = docopt(__doc__)
schema = Schema({
'--kernel-type':
And(Use(int),
lambda n: 0 <= n <= 1,
error='--kernel-type should be 0 or 1'),
'--infering-type':
Or(None,
And(Use(int), lambda n: 0 <= n <= 1),
error='--infering-type should be 0 or 1'),
'--sender-type':
And(Use(int),
lambda n: 0 <= n <= 4,
error='--sender-type should between 0 and 4'),
'--sender-period':
And(Use(float),
lambda n: 0.1 <= n <= 1.0,
error='--sender-period should be 0 or 1'),
'--senders':
def validate_method(method, is_challenge):
'''Validate method configuration passed as a JSON file.'''
# Define a dictionary schema
# TODO would be nice to not copy-paste for multiple datasets
schema = Schema({
Optional('metadata'): {
'publish_anonymously':
bool,
'authors':
str,
'contact_email':
str,
'method_name':
str,
'method_description':
str,
# 'descriptor_type': str,
# 'descriptor_size': And(int, lambda v: v >= 1),
Optional('link_to_website'):
str,
Optional('link_to_pdf'):
str,
Optional('under_review'):
bool,
Optional('under_review_override'):
str,
},
'config_common': {
'json_label': str,
'keypoint': And(Use(str), lambda v: '_' not in v),
'descriptor': And(Use(str), lambda v: '_' not in v),
'num_keypoints': And(int, lambda v: v > 1),
},
Optional('config_phototourism_stereo'): {
Optional('use_custom_matches'): bool,
Optional('custom_matches_name'): str,
Optional('matcher'): {
'method':
And(str, lambda v: v in ['nn']),
'distance':
And(str, lambda v: v.lower() in ['l1', 'l2', 'hamming']),
'flann':
bool,
'num_nn':
And(int, lambda v: v >= 1),
'filtering': {
'type':
And(
str, lambda v: v.lower() in [
'none', 'snn_ratio_pairwise', 'snn_ratio_vs_last',
'fginn_ratio_pairwise'
]),
Optional('threshold'):
And(Use(float), lambda v: 0 < v <= 1),
Optional('fginn_radius'):
And(Use(float), lambda v: 0 < v < 100.),
},
Optional('descriptor_distance_filter'): {
'threshold': And(Use(float), lambda v: v > 0),
},
'symmetric': {
'enabled':
And(bool),
Optional('reduce'):
And(str, lambda v: v.lower() in ['both', 'any']),
},
},
Optional('outlier_filter'): {
'method':
And(Use(str), lambda v: v.lower() in ['none', 'cne-bp-nd']),
},
Optional('geom'): {
'method':
And(
str, lambda v: v.lower() in [
'cv2-ransac-f', 'cv2-ransac-e', 'cv2-lmeds-f',
'cv2-lmeds-e', 'cv2-7pt', 'cv2-8pt',
'cv2-patched-ransac-f', 'cmp-degensac-f',
'cmp-degensac-f-laf', 'cmp-gc-ransac-f',
'cmp-degensac-f-laf', 'cmp-gc-ransac-f',
'cmp-magsac-f', 'cmp-gc-ransac-e', 'skimage-ransac-f',
'intel-dfe-f'
]),
Optional('threshold'):
And(Use(float), lambda v: v > 0),
Optional('confidence'):
And(Use(float), lambda v: v > 0),
Optional('max_iter'):
And(Use(int), lambda v: v > 0),
Optional('postprocess'):
And(Use(bool), lambda v: v is not None),
Optional('error_type'):
And(Use(str),
lambda v: v.lower() in ['sampson', 'symm_epipolar']),
Optional('degeneracy_check'):
bool,
}
},
Optional('config_phototourism_multiview'): {
Optional('use_custom_matches'): bool,
Optional('custom_matches_name'): str,
Optional('matcher'): {
'method':
And(str, lambda v: v in ['nn']),
'distance':
And(str, lambda v: v.lower() in ['l1', 'l2', 'hamming']),
'flann':
bool,
'num_nn':
And(int, lambda v: v >= 1),
'filtering': {
'type':
And(
str, lambda v: v.lower() in [
'none', 'snn_ratio_pairwise', 'snn_ratio_vs_last',
'fginn_ratio_pairwise'
]),
Optional('threshold'):
And(Use(float), lambda v: 0 < v <= 1),
Optional('fginn_radius'):
And(Use(float), lambda v: 0 < v < 100.),
},
Optional('descriptor_distance_filter'): {
'threshold': And(Use(float), lambda v: v > 0),
},
'symmetric': {
'enabled':
And(bool),
Optional('reduce'):
And(str, lambda v: v.lower() in ['both', 'any']),
},
},
Optional('outlier_filter'): {
'method':
And(Use(str), lambda v: v.lower() in ['none', 'cne-bp-nd']),
},
Optional('colmap'): {},
},
Optional('config_phototourism_relocalization'): {},
})
schema.validate(method)
# Check for metadata for challenge entries
if is_challenge and not method['metadata']:
raise ValueError('Must specify metadata')
# Check what we are running
do_pt_stereo = False if 'config_phototourism_stereo' not in method \
else bool(method['config_phototourism_stereo'])
do_pt_multiview = False if 'config_phototourism_multiview' not in method \
else bool(method['config_phototourism_multiview'])
do_pt_relocalization = False if 'config_phototourism_relocalization' not \
in method else bool(method['config_phototourism_relocalization'])
if do_pt_stereo:
print('Running: Phototourism, stereo track')
if do_pt_multiview:
print('Running: Phototourism, multiview track')
if do_pt_relocalization:
print('Running: Phototourism, relocalization track')
if not any([do_pt_stereo, do_pt_multiview, do_pt_relocalization]):
raise ValueError('No tasks were specified')
# Check for incorrect, missing, or redundant options
for dataset in ['phototourism']:
for task in ['stereo', 'multiview', 'relocalization']:
cur_key = 'config_{}_{}'.format(dataset, task)
if cur_key not in method:
print('Key "{}" is empty -> skipping check'.format(cur_key))
continue
else:
print('Validating key "{}"'.format(cur_key))
# If dict is not empty, use_custom_matches should exist
if method[cur_key] and ('use_custom_matches'
not in method[cur_key]):
raise ValueError('Must specify use_custom_matches')
# If using custom matches, we cannot specify further options
if method[cur_key] and ('use_custom_matches' in method[cur_key]) \
and method[cur_key]['use_custom_matches']:
if 'matcher' in method[cur_key] or 'outlier_filter' in method[
cur_key]:
raise ValueError(
'Cannot specify a matcher or outlier filter with '
'use_custom_matches=True')
# Matcher and filter
if 'matcher' in method[cur_key]:
matcher = method[cur_key]['matcher']
if matcher['symmetric']['enabled'] and 'reduce' not in \
matcher['symmetric']:
raise ValueError(
'[{}/{}] Must specify "reduce" if "symmetric" is enabled'
.format(dataset, task))
# Check for redundant settings with custom matches
if 'config_{}_stereo'.format(dataset) in method:
cur_config = method['config_{}_stereo'.format(dataset)]
if cur_config['use_custom_matches']:
if 'matcher' in cur_config or 'outlier_filter' in cur_config \
or 'geom' in cur_config:
raise ValueError(
'[{}/stereo] Found redundant settings with use_custom_matches=True'
.format(dataset))
else:
if 'matcher' not in cur_config or 'outlier_filter' not in \
cur_config or 'geom' not in cur_config:
raise ValueError(
'[{}/stereo] Missing required settings with use_custom_matches=False'
.format(dataset))
if cur_config['use_custom_matches']:
if 'matcher' in cur_config or 'outlier_filter' in cur_config \
or 'geom' in cur_config:
raise ValueError(
'[{}/stereo] Found redundant settings with use_custom_matches=True'
)
# For stereo, check also geom
if task == 'stereo' and \
'config_{}_stereo'.format(dataset) in method and \
'geom' in method['config_{}_stereo'.format(dataset)]:
geom = method['config_{}_stereo'.format(dataset)]['geom']
# Threshold for RANSAC
if geom['method'].lower() in [
'cv2-ransac-f', 'cv2-ransac-e', 'cv2-patched-ransac-f',
'cmp-degensac-f', 'cmp-gc-ransac-f', 'cmp-gc-ransac-e',
'cmp-degensac-f-laf', 'cmp-magsac-f',
'skimage-ransac-f', 'intel-dfe-f'
]:
if 'threshold' not in geom:
raise ValueError(
'[{}] Must specify a threshold for this method'.
format(dataset))
else:
if 'threshold' in geom:
raise ValueError(
'[{}] Cannot specify a threshold for this method'.
format(dataset))
# Degeneracy check for RANSAC
if geom['method'].lower() in [
'cmp-degensac-f', 'cmp-degensac-f-laf'
]:
if 'degeneracy_check' not in geom:
raise ValueError(
'[{}] Must indicate degeneracy check for this method'
.format(dataset))
if 'error_type' not in geom:
raise ValueError(
'[{}] Must indicate error type for this method'.
format(dataset))
else:
if 'degeneracy_check' in geom:
raise ValueError(
'[{}] Cannot apply degeneracy check to this method'
.format(dataset))
if 'error_type' in geom:
raise ValueError(
'[{}] Cannot indicate error type for this method'.
format(dataset))
# Confidence for RANSAC/LMEDS
if geom['method'].lower() in [
'cv2-ransac-f',
'cv2-patched-ransac-f',
'cv2-ransac-e',
'cv2-lmeds-f',
'cv2-lmeds-e',
'cmp-degensac-f',
'cmp-degensac-f-laf',
'cmp-gc-ransac-f',
'cmp-gc-ransac-e',
'skimage-ransac-f',
'cmp-magsac-f',
]:
if 'confidence' not in geom:
raise ValueError(
'[{}] Must specify a confidence value for OpenCV or DEGENSAC'
.format(dataset))
else:
if 'confidence' in geom:
raise ValueError(
'[{}] Cannot specify a confidence value for this method'
.format(dataset))
# Maximum number of RANSAC iterations
if geom['method'].lower() in [
'cv2-patched-ransac-f',
'cmp-degensac-f',
'cmp-degensac-f-laf',
'cmp-gc-ransac-f',
'cmp-gc-ransac-e',
'skimage-ransac-f',
'cmp-magsac-f',
]:
if 'max_iter' not in geom:
raise ValueError(
'[{}] Must indicate max_iter for this method'.
format(dataset))
else:
if 'max_iter' in geom:
raise ValueError(
'[{}] Cannot indicate max_iter for this method'.
format(dataset))
# DFE-specific
if geom['method'].lower() in ['intel-dfe-f']:
if 'postprocess' not in geom:
raise ValueError(
'[{}] Must specify a postprocess flag for DFE'.
format(dataset))
else:
if 'postprocess' in geom:
raise ValueError(
'[{}] Cannot specify a postprocess flag for this method'
.format(dataset))
user = User('username', 'password', 'access')
integrity_error = IntegrityError('Mock', 'mock', Exception('mock', 'mock'))
aleatory_errors = [
DataError('Mock', 'mock', Exception('mock', 'mock')),
OperationalError('Mock', 'mock', Exception('mock', 'mock')),
InternalError('Mock', 'mock', Exception('mock', 'mock')),
ProgrammingError('Mock', 'mock', Exception('mock', 'mock')),
NotSupportedError('Mock', 'mock', Exception('mock', 'mock'))
]
user_validate = Schema({
'id': Or(int, None),
'updated': Or(str, None),
'created': Or(str, None),
'access': Or(str, None),
'username': Use(str),
'removed': Or(str, None)
})
user_list_validate = Schema([{
'id': Or(int, None),
'updated': Or(str, None),
'created': Or(str, None),
'access': Or(str, None),
'username': Use(str),
'removed': Or(str, None)
}])
class TestFind(unittest.TestCase):
@patch('app.views.user.m_User')
commandRenameRoomSchema = dictSchema({
"command": str,
"server": str,
"data": {
str: {
"roomUID": str
}
}
})
commandShareRoomSchema = dictSchema(
{
"command": str,
"server": str,
"data": {
Use(str, error='specify roomUID'): {
Use(is_email, error="please specify a valid mail address"):
Optional(str, error='specify the name of the user')
}
}
},
error="please specify a room name and at least one valid mail address")
commandCreateRoomSchema = dictSchema(
{
"command": str,
"server": str,
"data": {
Use(str, error='specify room name'): {
"email":
Use(is_email, error="please specify a valid mail address"),
request_json_must_have_one,
request_json_must_have, token_admin_required,
token_required, auth)
import unittest
class User:
def __init__(self, access):
self.id = 1
self.access = access
self.username = '******'
self.password = '******'
user_dict_validate = Schema({
'id': Use(int),
'access': Use(str),
'username': Use(str)
})
user_dict = {'id': 1, 'access': 'access', 'username': '******'}
def inject_kwargs(f):
def wrapper(*args, **kwargs):
kwargs['id'] = 1
f(*args, **kwargs)
return wrapper
def main(argv, session, cmd='copy'):
args = docopt(__doc__, argv=argv)
src_path = args['<src-identifier>/<src-file>']
dest_path = args['<dest-identifier>/<dest-file>']
# If src == dest, file get's deleted!
try:
assert src_path != dest_path
except AssertionError:
print('error: The source and destination files cannot be the same!',
file=sys.stderr)
sys.exit(1)
global SRC_ITEM
SRC_ITEM = session.get_item(src_path.split('/')[0])
# Validate args.
s = Schema({
str:
Use(bool),
'<src-identifier>/<src-file>':
And(
str,
And(And(
str,
lambda x: '/' in x,
error='Destination not formatted correctly. See usage example.'
),
assert_src_file_exists,
error=('https://archive.org/download/{} does not exist. '
'Please check the identifier and filepath and retry.'.
format(src_path)))),
'<dest-identifier>/<dest-file>':
And(str,
lambda x: '/' in x,
error='Destination not formatted correctly. See usage example.'),
'--metadata':
Or(None,
And(Use(get_args_dict), dict),
error='--metadata must be formatted as --metadata="key:value"'),
'--header':
Or(None,
And(Use(get_args_dict), dict),
error='--header must be formatted as --header="key:value"'),
})
try:
args = s.validate(args)
except SchemaError as exc:
# This module is sometimes called by other modules.
# Replace references to 'ia copy' in ___doc__ to 'ia {cmd}' for clarity.
usage = printable_usage(__doc__.replace('ia copy',
'ia {}'.format(cmd)))
print('{0}\n{1}'.format(str(exc), usage), file=sys.stderr)
sys.exit(1)
args['--header']['x-amz-copy-source'] = '/{}'.format(src_path)
args['--header']['x-amz-metadata-directive'] = 'COPY'
args['--header']
# Add keep-old-version by default.
if 'x-archive-keep-old-version' not in args['--header']:
args['--header']['x-archive-keep-old-version'] = '1'
url = '{}//s3.us.archive.org/{}'.format(session.protocol, dest_path)
req = ia.iarequest.S3Request(url=url,
method='PUT',
metadata=args['--metadata'],
headers=args['--header'],
access_key=session.access_key,
secret_key=session.secret_key)
p = req.prepare()
r = session.send(p)
if r.status_code != 200:
try:
msg = get_s3_xml_text(r.text)
except Exception as e:
msg = r.text
print('error: failed to {} "{}" to "{}" - {}'.format(
cmd, src_path, dest_path, msg))
sys.exit(1)
elif cmd == 'copy':
print('success: copied "{}" to "{}".'.format(src_path, dest_path))
else:
return (r, SRC_FILE)
def main(*argv):
# Gather command line arguments
argv = list(*argv) if len(argv) == 1 else [i for i in argv]
args = docopt(__doc__,
argv=argv,
help=True,
version=__version__,
options_first=False)
# Choose an appropriate log level
if args['--verbose']:
log_level = logging.DEBUG
elif args['--quiet']:
log_level = logging.INFO
else:
# Default to debugging (complete I/O output)
log_level = logging.DEBUG
logging.basicConfig(
level=log_level,
format='%(levelname)5s:%(asctime)s.%(msecs)03d %(message)s',
datefmt='%d/%m/%Y %H:%M:%S')
"""
Create input schemata - Options accepting user input as value are checked for plausibility
"""
input_schema = Schema(
{"IN_FILE": And(os.path.exists, error="IN_FILE should exist")},
ignore_extra_keys=True,
)
config_schema = Schema(
{"CONFIG_FILE": And(os.path.exists, error="CONFIG_FILE should exist")},
ignore_extra_keys=True,
)
gcode_schema = Schema(
{"--f": And(os.path.exists, error="G-Code file should exist")},
ignore_extra_keys=True,
)
connection_schema = Schema(
{
"--ip":
And(
validate_ip,
error=
'IPv4 needs to be in format "[0-255].[0-255].[0-255].[0-255]"',
),
"--port":
And(
Use(int),
validate_port,
error="Port needs to be unsigned short (16 bits): 0..65535)",
),
},
ignore_extra_keys=True,
)
usb_schema = Schema(
{
"--vid":
And(Use(lambda x: int(x, 16)),
validate_id,
error=
"Vendor ID needs to be unsigned short (16 bits): 0..65535)"),
"--pid":
And(Use(lambda x: int(x, 16)),
validate_id,
error=
"Product ID needs to be unsigned short (16 bits): 0..65535)")
},
ignore_extra_keys=True,
)
"""
Dispatch sub-functions - new sub-commands are called here
"""
# noinspection PyBroadException
try:
if args["--gi"]:
usb_present = args["--vid"] is not None and args[
"--pid"] is not None
tcp_present = args["--ip"] is not None and args[
"--port"] is not None
if tcp_present:
args.update(connection_schema.validate(args))
ip, port, safe = args["--ip"], args["--port"], args["--safe"]
if usb_present:
# Robot and PCB
args.update(usb_schema.validate(args))
if args["--f"]:
gcode_schema.validate(args)
execute_gcode(ip, port, (args["--vid"], args["--pid"]),
args["--f"])
else:
interactive_gcode(ip, port,
(args["--vid"], args["--pid"]))
else:
# Robot only
interactive_gcode_robot_only(ip, port, safe_return=safe)
elif usb_present:
# PCB only
args.update(usb_schema.validate(args))
interactive_gcode_printer_only((args["--vid"], args["--pid"]))
else:
# None present
raise ValueError
elif args["--validate"]:
input_schema.validate(args)
config_schema.validate(args)
check_trajectory(config_f=args["CONFIG_FILE"],
gcode_f=args["IN_FILE"])
else:
args.update(connection_schema.validate(args))
ip, port, safe = (
args["--ip"],
args["--port"],
args["--safe"],
)
if args["--mi"]:
interactive_melfa(ip, port, safe_return=safe)
elif args["--demo"]:
demo_mode(ip, port, safe_return=safe)
else:
raise ApiException(
"Unknown option passed. Type --help for more info.")
except SchemaError:
# Input validation error
logging.exception("Input data invalid.")
sys.exit(EXIT_BAD_INPUT)
except (ApiException, Singularity, OutOfReachError, TrajectoryError):
# Intentionally thrown exception by functions of this module
logging.exception("Internal error.")
sys.exit(EXIT_INTERNAL_ERROR)
except KeyError:
# Accessing the arg dictionary with different keys as specified in docstring
logging.exception(
"This might have happened due to different versions of CLI documentation and parsing."
)
sys.exit(EXIT_UNEXPECTED_ERROR)
except NotImplementedError:
# This might be used in some functions
logging.exception("Encountered not implemented feature.")
sys.exit(EXIT_UNEXPECTED_ERROR)
except Exception:
# Exception that has not been caught and rethrown as a proper ApiException (= Bug)
logging.exception("External or unexpected exception!")
sys.exit(EXIT_UNEXPECTED_ERROR)
else:
# Everything okay, no exception occurred
sys.exit(EXIT_SUCCESS)
'description':
And(
str,
lambda s: len(s) > 30,
error='Description length must be > 40 characters',
),
'doi':
And(
validators.url,
str,
error='DOI must be a URL linking to the orginal paper',
),
'variables':
list,
'remote': {
'url': Use(URL),
Optional('username'): str,
Optional(Or('service', 'password', only_one=True)): str,
Optional('port'): int,
},
'local_store':
Use(Path),
Optional('pipelines'): {
str: {
'data_path': Use(Path, error='data_path must be a valid path'),
'functions': Use(get_modules_from_list),
}
},
})
if __name__ == '__main__':
- `expression`: the <expression> passed in by the user
- `regex`: Boolean set by --regex
:return: expression if not None or '*' or '.*' depending on regex
"""
if expresion is None:
if not regex:
return '*'
return '.*'
return expresion
finder_schema_dict = {
Arguments.expression: Or(None, str),
Arguments.root: Or(None, os.path.exists),
Arguments.shallow: Use(bool),
Arguments.regex: Use(bool)
}
schema = Schema(finder_schema_dict)
def setup_finder(arguments):
"""
Chooses the find function and expression based on the arguments
:param:
- `arguments`: validated dictionary of arguments
:return: <find function> <glob-expression>
"""
from schema import And, Use, Schema
def is_json_serializable(data):
if data is None:
return True
try:
json.dumps(data)
return True
except Exception:
return False
doc_schema = Schema({
'method': And(Use(str)),
'data': And(Use(is_json_serializable)),
'url': And(Use(str)),
'url_kwargs': And(Use(is_json_serializable)),
'format': And(Use(str)),
'headers': And(Use(is_json_serializable)),
'success': And(Use(bool)),
'meta': {
'docs': And(Use(str)),
'method_name': And(Use(str)),
'class_name': And(Use(str)),
'app_name': And(Use(str))
},
'response': {
'data': And(Use(is_json_serializable)),
'content_type': And(Use(str)),
def test_issue_add_dict_key_in_error():
try:
Schema({'attr1': Use(int)}).validate({'attr1': 'str'})
except SchemaError as e:
assert 'attr1' in e.autos[-1]
assert e.errors == [None, None]