def PredicateSql(self, name, allocator=None, external_vocabulary=None):
"""Producing SQL for a predicate."""
# Load proto if necessary.
rules = list(self.GetPredicateRules(name))
if len(rules) == 1:
[rule] = rules
return (self.SingleRuleSql(rule, allocator, external_vocabulary) +
self.annotations.OrderByClause(name) +
self.annotations.LimitClause(name))
elif len(rules) > 1:
rules_sql = []
for rule in rules:
if 'distinct_denoted' in rule:
raise rule_translate.RuleCompileException(
color.Format(
'For distinct denoted predicates multiple rules are not '
'currently supported. Consider taking '
'{warning}union of bodies manually{end}, if that was what '
'you intended.'), rule['full_text'])
rules_sql.append('\n%s\n' % Indent2(
self.SingleRuleSql(rule, allocator, external_vocabulary)))
rules_sql = [
'\n'.join(' ' + l for l in r.split('\n')) for r in rules_sql
]
return 'SELECT * FROM (\n%s\n) AS UNUSED_TABLE_NAME %s %s' % (
' UNION ALL\n'.join(rules_sql),
self.annotations.OrderByClause(name),
self.annotations.LimitClause(name))
else:
raise rule_translate.RuleCompileException(
color.Format(
'No rules are defining {warning}{name}{end}, but compilation '
'was requested.', dict(name=name)), r' ¯\_(ツ)_/¯')
def SingleRuleSql(self,
rule,
allocator=None,
external_vocabulary=None,
is_combine=False):
"""Producing SQL for a given rule in the program."""
allocator = allocator or self.NewNamesAllocator()
r = rule
if (is_combine):
r = self.execution.dialect.DecorateCombineRule(
r, allocator.AllocateVar())
s = rule_translate.ExtractRuleStructure(r, allocator,
external_vocabulary)
s.ElliminateInternalVariables(assert_full_ellimination=False)
self.RunInjections(s, allocator)
s.ElliminateInternalVariables(assert_full_ellimination=True)
s.UnificationsToConstraints()
try:
sql = s.AsSql(self.MakeSubqueryTranslator(allocator),
self.flag_values)
except RuntimeError as runtime_error:
if (str(runtime_error).startswith('maximum recursion')):
raise rule_translate.RuleCompileException(
RecursionError(), s.full_rule_text)
else:
raise runtime_error
if 'nil' in s.tables.values():
# Mark rule for deletion.
sql = '/* nil */' + sql
return sql
def __contains__(self, key):
if rule_predicate == '@Make':
ThrowException()
return
raise rule_translate.RuleCompileException(
'Annotation may not use variables, but this one uses '
'variable %s.' % (color.Warn(key)), rule_text)
def ThrowException(*args, **xargs):
_ = args
_ = xargs
if rule_predicate == '@Make':
# pylint: disable=raising-format-tuple
raise rule_translate.RuleCompileException(
'Incorrect syntax for functor call. '
'Functor call to be made as\n'
' R := F(A: V, ...)\n'
'or\n'
' @Make(R, F, {A: V, ...})\n'
'Where R, F, A\'s and V\'s are all '
'predicate names.', rule_text)
else:
raise rule_translate.RuleCompileException(
'Can not understand annotation.', rule_text)
def CompileAsUdf(self, predicate_name):
result = predicate_name in self.annotations['@CompileAsUdf']
if result and self.TvfSignature(predicate_name):
raise rule_translate.RuleCompileException(
'A predicate can not be UDF and TVF at the '
'same time %s.' % predicate_name,
'Predicate: ' + predicate_name)
return result
def __init__(self, rules, table_aliases=None, user_flags=None):
"""Initializes the program.
Args:
rules: A list of dictionary representations of parsed Logica rules.
table_aliases: A map from an undefined Logica predicate name to a
BigQuery table name. This table will be used in place of predicate.
user_flags: Dictionary of user specified flags.
"""
rules = self.UnfoldRecursion(rules)
# TODO: Should allocator be a member of Logica?
self.preparsed_rules = rules
self.rules = []
self.defined_predicates = set()
self.dollar_params = list(self.ExtractDollarParams(rules))
self.table_aliases = table_aliases or {}
self.execution = None
self.user_flags = user_flags or {}
self.annotations = Annotations(rules, self.user_flags)
self.flag_values = self.annotations.flag_values
# Dictionary custom_udfs maps function name to a format string to use
# in queries.
self.custom_udfs = collections.OrderedDict()
# Dictionary custom_udf_definitions maps function name to SQL defining the
# function.
self.custom_udf_definitions = collections.OrderedDict()
if not set(self.dollar_params) <= set(self.flag_values):
raise rule_translate.RuleCompileException(
'Parameters %s are undefined.' %
(list(set(self.dollar_params) - set(self.flag_values))),
str(list(set(self.dollar_params) - set(self.flag_values))))
self.functors = None
# Extending rules with functors.
extended_rules = self.RunMakes(rules) # Populates self.functors.
# Extending rules with the library of the dialect.
library_rules = parse.ParseFile(
dialects.Get(self.annotations.Engine()).LibraryProgram())['rule']
extended_rules.extend(library_rules)
for rule in extended_rules:
predicate_name = rule['head']['predicate_name']
self.defined_predicates.add(predicate_name)
self.rules.append((predicate_name, rule))
# We need to recompute annotations, because 'Make' created more rules and
# annotations.
self.annotations = Annotations(extended_rules, self.user_flags)
# Build udfs, populating custom_udfs and custom_udf_definitions.
self.BuildUdfs()
# Function compilation may have added irrelevant defines:
self.execution = None
if False:
self.RunTypechecker()
def ExtractSingleton(self, annotation_name, default_value):
if not self.annotations[annotation_name]:
return default_value
results = list(self.annotations[annotation_name].keys())
if len(results) > 1:
raise rule_translate.RuleCompileException(
'Single %s must be provided. Provided: %s' %
(annotation_name, results),
self.annotations[annotation_name][results[0]]['__rule_text'])
return results[0]
def LimitOf(self, predicate_name):
"""Limit of the query corresponding to the predicate as per annotation."""
if predicate_name not in self.annotations['@Limit']:
return None
annotation = FieldValuesAsList(
self.annotations['@Limit'][predicate_name])
if (len(annotation) != 1 or not isinstance(annotation[0], int)):
raise rule_translate.RuleCompileException(
'Bad limit specification for predicate %s.' % predicate_name,
'Predicate: ' + predicate_name)
return annotation[0]
def UseFlagsAsParameters(self, sql):
"""Running flag substitution in a loop to the fixed point."""
# We do it in a loop to deal with flags that refer to other flags.
prev_sql = ''
num_subs = 0
while sql != prev_sql:
num_subs += 1
prev_sql = sql
if num_subs > 100:
raise rule_translate.RuleCompileException(
'You seem to have recursive flags. It is disallowed.',
'Flags:\n' + '\n'.join('--{0}={1}'.format(*i)
for i in self.flag_values.items()))
# Do the substitution!
for flag, value in self.flag_values.items():
sql = sql.replace('${%s}' % flag, value)
return sql
def With(self, predicate_name):
"""Return whether this predicate should be compiled to a WITH-table.
This only applies if the predicate is not inlined earlier in the flow.
"""
is_with = self.ForceWith(predicate_name)
is_nowith = self.ForceNoWith(predicate_name)
if is_with and is_nowith:
raise rule_translate.RuleCompileException(
color.Format('Predicate is annotated both with @With and @NoWith.'),
'Predicate: %s' % predicate_name)
if is_with:
return True
if is_nowith or self.Ground(predicate_name):
return False
# TODO: return false for predicates that will be injected.
return True
def BuildFlagValues(self):
"""Building values by overriding defaults with user flags."""
default_values = {}
for flag, a in self.annotations['@DefineFlag'].items():
default_values[flag] = a.get('1', '${%s}' % flag)
programmatic_flag_values = {}
for flag, a in self.annotations['@ResetFlagValue'].items():
programmatic_flag_values[flag] = a.get('1', '${%s}' % flag)
if not set(self.user_flags) <= set(default_values):
raise rule_translate.RuleCompileException(
'Undefined flags used: %s' %
list(set(self.user_flags) - set(default_values)),
str(set(self.user_flags) - set(default_values)))
flag_values = default_values
flag_values.update(**programmatic_flag_values)
flag_values.update(**self.user_flags)
return flag_values
def RunInjections(self, s, allocator):
iterations = 0
while True:
iterations += 1
if iterations > sys.getrecursionlimit():
raise rule_translate.RuleCompileException(
RecursionError(), s.full_rule_text)
new_tables = collections.OrderedDict()
for table_name_rsql, table_predicate_rsql in s.tables.items():
rules = list(self.GetPredicateRules(table_predicate_rsql))
if (len(rules) == 1 and ('distinct_denoted' not in rules[0])
and
self.annotations.OkInjection(table_predicate_rsql)):
[r] = rules
rs = rule_translate.ExtractRuleStructure(
r, allocator, None)
rs.ElliminateInternalVariables(
assert_full_ellimination=False)
new_tables.update(rs.tables)
InjectStructure(s, rs)
new_vars_map = {}
new_inv_vars_map = {}
for (table_name,
table_var), clause_var in s.vars_map.items():
if table_name != table_name_rsql:
new_vars_map[table_name, table_var] = clause_var
new_inv_vars_map[clause_var] = (table_name,
table_var)
else:
if table_var not in rs.select:
if '*' in rs.select:
subscript = {
'literal': {
'the_symbol': {
'symbol': table_var
}
}
}
s.vars_unification.append({
'left': {
'variable': {
'var_name': clause_var
}
},
'right': {
'subscript': {
'subscript': subscript,
'record': rs.select['*']
}
}
})
else:
extra_hint = '' if table_var != '*' else (
' Are you using ..<rest of> for injectible predicate? '
'Please list the fields that you extract explicitly. '
'Tracking bug: b/131759583.')
raise rule_translate.RuleCompileException(
color.Format(
'Predicate {warning}{table_predicate_rsql}{end} '
'does not have an argument '
'{warning}{table_var}{end}, but '
'this rule tries to access it. {extra_hint}',
dict(table_predicate_rsql=
table_predicate_rsql,
table_var=table_var,
extra_hint=extra_hint)),
s.full_rule_text)
else:
s.vars_unification.append({
'left': {
'variable': {
'var_name': clause_var
}
},
'right':
rs.select[table_var]
})
s.vars_map = new_vars_map
s.inv_vars_map = new_inv_vars_map
else:
new_tables[table_name_rsql] = table_predicate_rsql
if s.tables == new_tables:
break
s.tables = new_tables
def FunctionSql(self, name, allocator=None, internal_mode=False):
"""Print formatted SQL function creation statement."""
# TODO: Refactor this into FunctionSqlInternal and FunctionSql.
if not allocator:
allocator = self.NewNamesAllocator()
rules = list(self.GetPredicateRules(name))
# Check that the predicate is defined via a single rule.
if not rules:
raise rule_translate.RuleCompileException(
color.Format(
'No rules are defining {warning}{name}{end}, but compilation '
'was requested.', dict(name=name)), r' ¯\_(ツ)_/¯')
elif len(rules) > 1:
raise rule_translate.RuleCompileException(
color.Format(
'Predicate {warning}{name}{end} is defined by more than 1 rule '
'and can not be compiled into a function.',
dict(name=name)),
'\n\n'.join(r['full_text'] for r in rules))
[rule] = rules
# Extract structure and assert that it is isomorphic to a function.
s = rule_translate.ExtractRuleStructure(rule,
external_vocabulary=None,
names_allocator=allocator)
udf_variables = [
v if isinstance(v, str) else 'col%d' % v for v in s.select
if v != 'logica_value'
]
s.select = self.TurnPositionalIntoNamed(s.select)
variables = [v for v in s.select if v != 'logica_value']
if 0 in variables:
raise rule_translate.RuleCompileException(
color.Format(
'Predicate {warning}{name}{end} must have all aruments named for '
'compilation as a function.', dict(name=name)),
rule['full_text'])
for v in variables:
if ('variable' not in s.select[v]
or s.select[v]['variable']['var_name'] != v):
raise rule_translate.RuleCompileException(
color.Format(
'Predicate {warning}{name}{end} must not rename arguments '
'for compilation as a function.', dict(name=name)),
rule['full_text'])
vocabulary = {v: v for v in variables}
s.external_vocabulary = vocabulary
self.RunInjections(s, allocator)
s.ElliminateInternalVariables(assert_full_ellimination=True)
s.UnificationsToConstraints()
sql = s.AsSql(subquery_encoder=self.MakeSubqueryTranslator(allocator))
if s.constraints or s.unnestings or s.tables:
raise rule_translate.RuleCompileException(
color.Format(
'Predicate {warning}{name}{end} is not a simple function, but '
'compilation as function was requested. Full SQL:\n{sql}',
dict(name=name, sql=sql)), rule['full_text'])
if 'logica_value' not in s.select:
raise rule_translate.RuleCompileException(
color.Format(
'Predicate {warning}{name}{end} does not have a value, but '
'compilation as function was requested. Full SQL:\n%s' %
sql), rule['full_text'])
# pylint: disable=g-long-lambda
# Compile the function!
ql = expr_translate.QL(
vocabulary,
self.MakeSubqueryTranslator(allocator),
lambda message: rule_translate.RuleCompileException(
message, rule['full_text']),
self.flag_values,
custom_udfs=self.custom_udfs,
dialect=self.execution.dialect)
value_sql = ql.ConvertToSql(s.select['logica_value'])
sql = 'CREATE TEMP FUNCTION {name}({signature}) AS ({value})'.format(
name=name,
signature=', '.join('%s ANY TYPE' % v for v in variables),
value=value_sql)
sql = FormatSql(sql)
if internal_mode:
return ('%s(%s)' % (name, ', '.join('{%s}' % v
for v in udf_variables)), sql)
return sql
def ExtractAnnotations(cls, rules, restrict_to=None, flag_values=None):
"""Extracting annotations from the rules."""
result = {
p: collections.OrderedDict()
for p in cls.ANNOTATING_PREDICATES
}
for rule in rules:
rule_predicate = rule['head']['predicate_name']
if restrict_to and rule_predicate not in restrict_to:
continue
if (rule_predicate[0] == '@'
and rule_predicate not in cls.ANNOTATING_PREDICATES):
raise rule_translate.RuleCompileException(
'Only {0} and {1} special predicates are allowed.'.format(
', '.join(cls.ANNOTATING_PREDICATES[:-1]),
cls.ANNOTATING_PREDICATES[-1]), rule['full_text'])
if rule_predicate in cls.ANNOTATING_PREDICATES:
rule_text = rule['full_text']
# pylint: disable=cell-var-from-loop
def ThrowException(*args, **xargs):
_ = args
_ = xargs
if rule_predicate == '@Make':
# pylint: disable=raising-format-tuple
raise rule_translate.RuleCompileException(
'Incorrect syntax for functor call. '
'Functor call to be made as\n'
' R := F(A: V, ...)\n'
'or\n'
' @Make(R, F, {A: V, ...})\n'
'Where R, F, A\'s and V\'s are all '
'predicate names.', rule_text)
else:
raise rule_translate.RuleCompileException(
'Can not understand annotation.', rule_text)
class Thrower(object):
def __contains__(self, key):
if rule_predicate == '@Make':
ThrowException()
return
raise rule_translate.RuleCompileException(
'Annotation may not use variables, but this one uses '
'variable %s.' % (color.Warn(key)), rule_text)
flag_values = flag_values or Thrower()
ql = expr_translate.QL(Thrower(), ThrowException,
ThrowException, flag_values)
ql.convert_to_json = True
annotation = rule['head']['predicate_name']
field_values_json_str = ql.ConvertToSql(
{'record': rule['head']['record']})
try:
field_values = json.loads(field_values_json_str)
except:
raise rule_translate.RuleCompileException(
'Could not understand arguments of annotation.',
rule['full_text'])
if ('0' in field_values and isinstance(field_values['0'], dict)
and 'predicate_name' in field_values['0']):
subject = field_values['0']['predicate_name']
else:
subject = field_values['0']
del field_values['0']
if rule_predicate in ['@OrderBy', '@Limit', '@NoInject']:
field_values_list = FieldValuesAsList(field_values)
if field_values_list is None:
raise rule_translate.RuleCompileException(
'@OrderBy and @Limit may only have positional '
'arguments.', rule['full_text'])
if rule_predicate == '@Limit' and len(
field_values_list) != 1:
raise rule_translate.RuleCompileException(
'Annotation @Limit must have exactly two arguments: '
'predicate and limit.', rule['full_text'])
updated_annotation = result.get(annotation, {})
field_values['__rule_text'] = rule['full_text']
if subject in updated_annotation:
raise rule_translate.RuleCompileException(
color.Format(
'{annotation} annotates {warning}{subject}{end} more '
'than once: {before}, {after}',
dict(annotation=annotation,
subject=subject,
before=updated_annotation[subject]
['__rule_text'],
after=field_values['__rule_text'])),
rule['full_text'])
updated_annotation[subject] = field_values
result[annotation] = updated_annotation
return result
def AnnotationError(message, annotation_value):
raise rule_translate.RuleCompileException(message,
annotation_value['__rule_text'])
def RaiseCompilerError(message, context):
raise rule_translate.RuleCompileException(message, context)
