def get_yaql_parser(cls):
if cls._parser is None:
global_options = {
'yaql.limitIterators': cfg.CONF.yaql.limit_iterators,
'yaql.memoryQuota': cfg.CONF.yaql.memory_quota
}
cls._parser = yaql.YaqlFactory().create(global_options)
return cls._parser
def create_engine():
global _global_engine
engine_options = {
'yaql.limitIterators': settings.YAQL_LIMIT_ITERATORS or 10000,
'yaql.memoryQuota': settings.YAQL_MEMORY_QUOTA or 100 * 1024 * 1024,
'yaql.convertTuplesToLists': True,
'yaql.convertSetsToLists': True
}
if _global_engine is None:
_global_engine = yaql.YaqlFactory().create(engine_options)
return _global_engine
def create_engine(limit_iterators=None, memory_quota=None):
"""Creates a new yaql engine instance."""
if not limit_iterators:
limit_iterators = settings.YAQL_LIMIT_ITERATORS or 10000
if not memory_quota:
memory_quota = settings.YAQL_MEMORY_QUOTA or 100 * 1024 * 1024
engine_options = {
'yaql.limitIterators': limit_iterators,
'yaql.memoryQuota': memory_quota,
'yaql.convertTuplesToLists': True,
'yaql.convertSetsToLists': True
}
return yaql.YaqlFactory().create(engine_options)
def init_parser(self):
"""
Initializes yaql parser engine. Builds parsing rules.
There is usually one engine for the whole application. Heavyweight object.
:return:
"""
factory = yaql.YaqlFactory()
versions_yaql.register_factory(factory)
engine_options = {
'yaql.limitIterators': 100000,
'yaql.convertSetsToLists': True,
'yaql.memoryQuota': 1000000
}
self.engine = factory.create(options=engine_options)
def _evaluate(yaql_expression, yaml_data, legacy=False):
engine_options = {
'yaql.limitIterators': 100,
'yaql.convertSetsToLists': True,
'yaql.memoryQuota': 10000
}
if legacy:
factory = yaql.legacy.YaqlFactory()
context = yaql.legacy.create_context()
context['legacy'] = True
else:
factory = yaql.YaqlFactory()
context = yaql.create_context()
parser = factory.create(options=engine_options)
return parser(yaql_expression).evaluate(yaml_data, context)
def test_limit_iterables(self):
engine = yaql.YaqlFactory().create({
'yaql.limitIterators': 1,
'yaql.convertTuplesToLists': True,
'yaql.convertSetsToLists': True
})
functions = ['added', 'deleted', 'changed']
expressions = ['$.nodes', '$.configs.nova']
for exp in expressions:
for func in functions:
with self.assertRaises(exceptions.CollectionTooLargeException):
self.evaluate('{0}({1})'.format(func, exp), engine=engine)
expressions = ['$.configs.nova.value', '$.cluster.status']
for exp in expressions:
for func in functions:
self.evaluate('{0}({1})'.format(func, exp), engine=engine)
def __init__(self):
self._taskflow = _TaskFlow()
self._yaql_engine = \
_yaql.YaqlFactory(allow_delegates=True) \
.create(
options={
# 'yaql.convertInputData': False
'yaql.convertInputData': True
})
self._yaql_ctx = _yaql.create_context(
convention=_yaql.language.conventions.PythonConvention(),
delegates=True)
self._yaql_expr = _Obj(
dict((k, self._yaql_engine(v)) for k, v in [
("merge_yaml_consts",
"call(let, [none], $.aggregate($1.merge_with($2.yaml.get(const, {})), {}))"
),
("merge_dict_list", "$.aggregate($1.merge_with($2), {})"),
]))
_std_function_registry.fill_context(self._yaql_ctx)
for i in _YAQL_PREPARE_CTX:
self._yaql_ctx = self._yaql_engine(i).evaluate(
context=self._yaql_ctx, data=None)
self._taskflow.add_tasks([
('init', _init_task_init),
('load_yamls', _init_task_load_yamls),
('merge_yaml_consts', _init_task_merge_yaml_consts),
('py_deps', _init_task_py_deps),
('load_plugins', _init_task_load_plugins),
('load_py', _init_task_load_py),
('setup_yaql_extras', _init_task_setup_yaql_extras),
('setup_jinja', _init_task_setup_jinja),
('fragments', _init_task_fragments),
('prepare', _init_task_prepare),
('produce', _init_task_produce),
('result', _init_task_result),
])
def main():
p = optparse.OptionParser()
p.add_option('--data', '-d')
p.add_option('-t', action='store_true', dest='tokens')
p.add_option('--legacy', action='store_true', dest='legacy')
options, arguments = p.parse_args()
if options.data:
try:
with open(options.data) as f:
data = json.load(f)
except Exception:
print('Unable to load data from ' + options.data)
return
else:
data = None
engine_options = {
'yaql.limitIterators': 100,
'yaql.convertSetsToLists': True,
'yaql.memoryQuota': 10000
}
if options.legacy:
factory = yaql.legacy.YaqlFactory()
context = yaql.legacy.create_context()
context['legacy'] = True
else:
factory = yaql.YaqlFactory()
context = yaql.create_context()
parser = factory.create(options=engine_options)
cli_functions.register_in_context(context, parser)
if options.tokens:
parser('__main(true)').evaluate(data, context)
else:
parser('__main(false)').evaluate(data, context)
import yaql
engine = yaql.YaqlFactory().create()
def get_component_section(data, id):
for component in data['lightweight_components']:
if component['execution_id'] is int(id):
return component
def evaluate(data, query):
expression = engine(query)
return expression.evaluate(data)
def get_voms_config(data, component_section):
default_voms_config = []
try:
supported_vos = component_section['config']['vos']
except KeyError:
supported_vos = evaluate(data, "$.supported_virtual_organizations")
try:
voms_config = evaluate(data, "$.voms_config")
except KeyError:
for vo in supported_vos:
default_pool_accounts = []
default_pool_account_key = "default_pool_account_" + vo['name']
default_pool_account__sgm_key = "default_pool_account_" + vo[
'name'] + "sgm"
if default_pool_account_key in data:
def _set_up_yaql():
legacy_engine_options = {
'yaql.limitIterators': 100,
'yaql.memoryQuota': 20000
}
return yaql.YaqlFactory().create(options=legacy_engine_options)
def main():
p = optparse.OptionParser()
p.add_option('--data', '-d', help="input file")
p.add_option('--string',
'-s',
action='store_true',
help="input is a string")
p.add_option('--native',
'-n',
action='store_true',
help="output data in Python native format")
p.add_option('--array',
'-a',
action='store_true',
help="read input line by line")
p.add_option('--tokens',
'-t',
action='store_true',
dest='tokens',
help="print lexical tokens info")
p.add_option('--legacy',
'-l',
action='store_true',
dest='legacy',
help="enable legacy v0.2 compatibility mode")
p.add_option('--limit',
'--limit-iterators',
'-L',
type=int,
dest='limit_iterators',
default=1000,
help="limit iterators by the given number of "
"elements (-1 means infinity)")
p.add_option('--sets-to-lists',
'-S',
action="store_true",
dest='sets_to_lists',
default=True,
help="convert all sets in results to lists")
p.add_option('--tuples-to-lists',
'-T',
action="store_true",
dest='tuples_to_lists',
default=False,
help="convert all tuples in results to lists")
p.add_option('--iterable-dicts',
'-D',
action="store_true",
dest='iterable_dicts',
default=False,
help="consider dictionaries to be iterable over their keys")
p.add_option('--memory-quota',
'--mem',
'-m',
type=int,
dest='memory',
default=100000,
help="change memory usage quota (in bytes) "
"for all data produced by expressions "
"(-1 means infinity)")
p.add_option('--keyword-operator',
'-k',
type=str,
dest='keyword_operator',
default="=>",
help="configure keyword/mapping symbol "
"(empty string means disabled)")
p.add_option('--allow-delegates',
'-A',
action="store_true",
dest='allow_delegates',
default=False,
help="enable delegate expression parsing")
options, arguments = p.parse_args()
if options.data:
try:
if options.data == '-':
data = read_data(sys.stdin, options)
else:
with open(options.data) as f:
data = read_data(f, options)
except Exception:
print('Unable to load data from ' + options.data, file=sys.stderr)
exit(1)
else:
data = None
engine_options = {
'yaql.limitIterators': options.limit_iterators,
'yaql.convertSetsToLists': options.sets_to_lists,
'yaql.convertTuplesToLists': options.tuples_to_lists,
'yaql.iterableDicts': options.iterable_dicts,
'yaql.memoryQuota': options.memory
}
if options.legacy:
factory = yaql.legacy.YaqlFactory(
allow_delegates=options.allow_delegates)
context = yaql.legacy.create_context()
context['legacy'] = True
else:
factory = yaql.YaqlFactory(allow_delegates=options.allow_delegates,
keyword_operator=options.keyword_operator)
context = yaql.create_context()
if options.native:
context['#nativeOutput'] = True
parser = factory.create(options=engine_options)
cli_functions.register_in_context(context, parser)
if len(arguments) > 0:
for arg in arguments:
cli_functions.evaluate(arg, parser, data, context)
elif options.tokens:
parser('__main(true)').evaluate(data, context)
else:
parser('__main(false)').evaluate(data, context)
def main():
p = optparse.OptionParser()
p.add_option('--data', '-d', help="input file")
p.add_option('--string',
'-s',
action='store_true',
help="input is a string")
p.add_option('--native',
'-n',
action='store_true',
help="output data in Python native format")
p.add_option('--array',
'-a',
action='store_true',
help="read input line by line")
p.add_option('--tokens',
'-t',
action='store_true',
dest='tokens',
help="print lexical tokens info")
p.add_option('--legacy',
action='store_true',
dest='legacy',
help="enable legacy v0.2 compatibility mode")
options, arguments = p.parse_args()
if options.data:
try:
if options.data == '-':
data = read_data(sys.stdin, options)
else:
with open(options.data) as f:
data = read_data(f, options)
except Exception:
print('Unable to load data from ' + options.data, file=sys.stderr)
exit(1)
else:
data = None
engine_options = {
'yaql.limitIterators': 1000,
'yaql.convertSetsToLists': True,
'yaql.memoryQuota': 100000
}
if options.legacy:
factory = yaql.legacy.YaqlFactory()
context = yaql.legacy.create_context()
context['legacy'] = True
else:
factory = yaql.YaqlFactory()
context = yaql.create_context()
if options.native:
context['#nativeOutput'] = True
parser = factory.create(options=engine_options)
cli_functions.register_in_context(context, parser)
if len(arguments) > 0:
for arg in arguments:
cli_functions.evaluate(arg, parser, data, context)
elif options.tokens:
parser('__main(true)').evaluate(data, context)
else:
parser('__main(false)').evaluate(data, context)
