def PredicateSql(self, name, allocator=None, external_vocabulary=None):
"""Producing SQL for a predicate."""
# Load proto if necessary.
rules = list(self.GetPredicateRules(name))
if len(rules) == 1:
[rule] = rules
return (self.SingleRuleSql(rule, allocator, external_vocabulary) +
self.annotations.OrderByClause(name) +
self.annotations.LimitClause(name))
elif len(rules) > 1:
rules_sql = []
for rule in rules:
if 'distinct_denoted' in rule:
raise rule_translate.RuleCompileException(
color.Format(
'For distinct denoted predicates multiple rules are not '
'currently supported. Consider taking '
'{warning}union of bodies manually{end}, if that was what '
'you intended.'), rule['full_text'])
rules_sql.append('\n%s\n' % Indent2(
self.SingleRuleSql(rule, allocator, external_vocabulary)))
rules_sql = [
'\n'.join(' ' + l for l in r.split('\n')) for r in rules_sql
]
return 'SELECT * FROM (\n%s\n) AS UNUSED_TABLE_NAME %s %s' % (
' UNION ALL\n'.join(rules_sql),
self.annotations.OrderByClause(name),
self.annotations.LimitClause(name))
else:
raise rule_translate.RuleCompileException(
color.Format(
'No rules are defining {warning}{name}{end}, but compilation '
'was requested.', dict(name=name)), r' ¯\_(ツ)_/¯')
def RunTest(name,
src,
predicate,
golden,
user_flags=None,
overwrite=False,
announce=False):
"""Run one test."""
if announce:
print('Running test:', name)
test_result = '{warning}RUNNING{end}'
print(color.Format('% 50s %s' % (name, test_result)))
result = logica_lib.RunPredicate(src, predicate, user_flags=user_flags)
# Hacky way to remove query that BQ prints.
if '+---' in result[200:]:
result = result[result.index('+---'):]
if overwrite:
with open(golden, 'w') as w:
w.write(result)
golden_result = open(golden).read()
if result == golden_result:
test_result = '{ok}PASSED{end}'
else:
p = subprocess.Popen(['diff', '-', golden], stdin=subprocess.PIPE)
p.communicate(result.encode())
test_result = '{error}FAILED{end}'
print('\033[F\033[K' + color.Format('% 50s %s' % (name, test_result)))
def PrintHeader():
"""Print header for all tests."""
print(
color.Format('% 64s %s' %
('{warning}TEST{end}', '{warning}RESULT{end}')))
print(
color.Format('% 64s %s' %
('{warning}----{end}', '{warning}------{end}')))
def SortUnnestings(self):
"""Sorts unnestings in dependency order."""
unnesting_of = {
u[0]['variable']['var_name']: u
for u in self.unnestings
}
unnesting_variables = set(unnesting_of)
depends_on = {
u[0]['variable']['var_name']:
set(AllMentionedVariables(u[1], dive_in_combines=True))
& unnesting_variables
for u in self.unnestings
}
unnested = set()
ordered_unnestings = []
while unnesting_of:
for v, u in sorted(unnesting_of.items()):
if depends_on[v] <= unnested:
ordered_unnestings.append(unnesting_of[v])
del unnesting_of[v]
unnested.add(v)
break
else:
raise RuleCompileException(
color.Format(
'There seem to be a circular dependency of {warning}In{end} '
'calls. '
'This error might also come from injected sub-rules.'),
self.full_rule_text)
self.unnestings = ordered_unnestings
def ShowMessage(self, stream=sys.stderr):
"""Printing error message."""
print(color.Format('{underline}Parsing{end}:'), file=stream)
before, error, after = self.location.Pieces()
if len(before) > 300:
before = before[-300:]
if len(after) > 300:
after = after[:300]
if not error:
error = '<EMPTY>'
print(color.Format('{before}{warning}{error_text}{end}{after}',
dict(before=before,
error_text=error,
after=after)), file=stream)
print(
color.Format('\n[ {error}Error{end} ] ') + str(self), file=stream)
def With(self, predicate_name):
"""Return whether this predicate should be compiled to a WITH-table.
This only applies if the predicate is not inlined earlier in the flow.
"""
is_with = self.ForceWith(predicate_name)
is_nowith = self.ForceNoWith(predicate_name)
if is_with and is_nowith:
raise rule_translate.RuleCompileException(
color.Format('Predicate is annotated both with @With and @NoWith.'),
'Predicate: %s' % predicate_name)
if is_with:
return True
if is_nowith or self.Ground(predicate_name):
return False
# TODO: return false for predicates that will be injected.
return True
def ExtractRuleStructure(rule, names_allocator=None, external_vocabulary=None):
"""Extracts RuleStructure from rule."""
rule = copy.deepcopy(rule)
# Not disambiguating if this rule is extracting structure of the combine
# itself, as variables of this combine were already disambiguated from
# parent rule.
if rule['head']['predicate_name'] != 'Combine':
DisambiguateCombineVariables(rule, names_allocator)
s = RuleStructure(names_allocator, external_vocabulary)
InlinePredicateValues(rule, names_allocator)
s.full_rule_text = rule['full_text']
s.this_predicate_name = rule['head']['predicate_name']
(s.select, aggregated_vars) = HeadToSelect(rule['head'])
# Adding internal variable unification with select arguments to avoid
# confusion of user variables between injected predicates.
for k, expr in s.select.items():
if 'variable' in expr:
s.vars_unification.append({
'left': expr,
'right': {
'variable': {
'var_name':
names_allocator.AllocateVar('extract_%s_%s' %
(s.this_predicate_name, k))
}
}
})
if 'body' in rule:
ExtractConjunctiveStructure(rule['body']['conjunction']['conjunct'], s)
distinct_denoted = 'distinct_denoted' in rule
s.distinct_denoted = distinct_denoted
if aggregated_vars and not distinct_denoted:
raise RuleCompileException(
color.Format(
'Aggregating predicate must be {warning}distinct{end} denoted.'
), s.full_rule_text)
if distinct_denoted:
s.distinct_vars = sorted(
list(set(s.select.keys()) - set(aggregated_vars)), key=str)
return s
def ExtractAnnotations(cls, rules, restrict_to=None, flag_values=None):
"""Extracting annotations from the rules."""
result = {
p: collections.OrderedDict()
for p in cls.ANNOTATING_PREDICATES
}
for rule in rules:
rule_predicate = rule['head']['predicate_name']
if restrict_to and rule_predicate not in restrict_to:
continue
if (rule_predicate[0] == '@'
and rule_predicate not in cls.ANNOTATING_PREDICATES):
raise rule_translate.RuleCompileException(
'Only {0} and {1} special predicates are allowed.'.format(
', '.join(cls.ANNOTATING_PREDICATES[:-1]),
cls.ANNOTATING_PREDICATES[-1]), rule['full_text'])
if rule_predicate in cls.ANNOTATING_PREDICATES:
rule_text = rule['full_text']
# pylint: disable=cell-var-from-loop
def ThrowException(*args, **xargs):
_ = args
_ = xargs
if rule_predicate == '@Make':
# pylint: disable=raising-format-tuple
raise rule_translate.RuleCompileException(
'Incorrect syntax for functor call. '
'Functor call to be made as\n'
' R := F(A: V, ...)\n'
'or\n'
' @Make(R, F, {A: V, ...})\n'
'Where R, F, A\'s and V\'s are all '
'predicate names.', rule_text)
else:
raise rule_translate.RuleCompileException(
'Can not understand annotation.', rule_text)
class Thrower(object):
def __contains__(self, key):
if rule_predicate == '@Make':
ThrowException()
return
raise rule_translate.RuleCompileException(
'Annotation may not use variables, but this one uses '
'variable %s.' % (color.Warn(key)), rule_text)
flag_values = flag_values or Thrower()
ql = expr_translate.QL(Thrower(), ThrowException,
ThrowException, flag_values)
ql.convert_to_json = True
annotation = rule['head']['predicate_name']
field_values_json_str = ql.ConvertToSql(
{'record': rule['head']['record']})
try:
field_values = json.loads(field_values_json_str)
except:
raise rule_translate.RuleCompileException(
'Could not understand arguments of annotation.',
rule['full_text'])
if ('0' in field_values and isinstance(field_values['0'], dict)
and 'predicate_name' in field_values['0']):
subject = field_values['0']['predicate_name']
else:
subject = field_values['0']
del field_values['0']
if rule_predicate in ['@OrderBy', '@Limit', '@NoInject']:
field_values_list = FieldValuesAsList(field_values)
if field_values_list is None:
raise rule_translate.RuleCompileException(
'@OrderBy and @Limit may only have positional '
'arguments.', rule['full_text'])
if rule_predicate == '@Limit' and len(
field_values_list) != 1:
raise rule_translate.RuleCompileException(
'Annotation @Limit must have exactly two arguments: '
'predicate and limit.', rule['full_text'])
updated_annotation = result.get(annotation, {})
field_values['__rule_text'] = rule['full_text']
if subject in updated_annotation:
raise rule_translate.RuleCompileException(
color.Format(
'{annotation} annotates {warning}{subject}{end} more '
'than once: {before}, {after}',
dict(annotation=annotation,
subject=subject,
before=updated_annotation[subject]
['__rule_text'],
after=field_values['__rule_text'])),
rule['full_text'])
updated_annotation[subject] = field_values
result[annotation] = updated_annotation
return result
def RecursionError():
return color.Format('The rule appears to use recursion. '
'{warning}Recursion{end} is neither supported by '
'Logica nor by StandardSQL.')
def ConvertToSql(self, expression):
"""Converting Logica expression into SQL."""
# print('EXPR:', expression)
# Variables.
if 'variable' in expression:
return self.Variable(expression['variable'])
# Literals.
if 'literal' in expression:
literal = expression['literal']
if 'the_number' in literal:
return self.IntLiteral(literal['the_number'])
if 'the_string' in literal:
return self.StrLiteral(literal['the_string'])
if 'the_list' in literal:
return self.ListLiteral(literal['the_list'])
if 'the_bool' in literal:
return self.BoolLiteral(literal['the_bool'])
if 'the_null' in literal:
return self.NullLiteral(literal['the_null'])
if 'the_predicate' in literal:
return self.PredicateLiteral(literal['the_predicate'])
assert False, 'Logica bug: unsupported literal parsed: %s' % literal
if 'call' in expression:
call = expression['call']
arguments = self.ConvertRecord(call['record'])
if call['predicate_name'] in self.ANALYTIC_FUNCTIONS:
return self.ConvertAnalytic(call)
if call['predicate_name'] == 'SqlExpr':
return self.GenericSqlExpression(call['record'])
if call['predicate_name'] == 'Cast':
if (len(arguments) != 2 or 'literal' not in call['record']
['field_value'][1]['value']['expression']
or 'the_string' not in call['record']['field_value'][1]
['value']['expression']['literal']):
raise self.exception_maker(
'Cast must have 2 arguments and the second argument must be a '
'string literal.')
cast_to = (
call['record']['field_value'][1]['value']['expression']
['literal']['the_string']['the_string'])
return 'CAST(%s AS %s)' % (self.ConvertToSql(
call['record']['field_value'][0]['value']['expression']),
cast_to)
if call['predicate_name'] == 'FlagValue':
if (len(arguments) != 1 or 'literal' not in call['record']
['field_value'][0]['value']['expression']
or 'the_string' not in call['record']['field_value'][0]
['value']['expression']['literal']):
raise self.exception_maker(
'FlagValue argument must be a string literal.')
flag = (call['record']['field_value'][0]['value']['expression']
['literal']['the_string']['the_string'])
if flag not in self.flag_values:
raise self.exception_maker('Unspecified flag: %s' % flag)
return self.StrLiteral({'the_string': self.flag_values[flag]})
for ydg_f, sql_f in self.built_in_functions.items():
if call['predicate_name'] == ydg_f:
if not sql_f:
raise self.exception_maker(
'Function %s is not supported by %s dialect.' %
(color.Warn(ydg_f), color.Warn(
self.dialect.Name())))
if len(arguments) == 2 and ydg_f == '-':
continue # '-' is the only operator with variable arity.
arity_range = self.BuiltInFunctionArityRange(ydg_f)
if not arity_range[0] <= len(arguments) <= arity_range[1]:
raise self.exception_maker(
color.Format(
'Built-in function {warning}{ydg_f}{end} takes {a} '
'arguments, but {b} arguments were given.',
dict(ydg_f=ydg_f,
a=arity_range,
b=len(arguments))))
return self.Function(sql_f, arguments)
for udf, udf_sql in self.custom_udfs.items():
if call['predicate_name'] == udf:
# TODO: Treatment of positional arguments should be
# simplified everywhere.
arguments = dict(
(k, v) if isinstance(k, str) else ('col%d' % k, v)
for k, v in arguments.items())
try:
result = udf_sql.format(**arguments)
except KeyError:
raise self.exception_maker(
'Function %s call is inconsistent with its signature %s.'
% (color.Warn(udf), udf_sql))
return result
for ydg_op, sql_op in self.built_in_infix_operators.items():
if call['predicate_name'] == ydg_op:
result = self.Infix(sql_op, arguments)
result = '(' + result + ')'
return result
if 'subscript' in expression:
sub = expression['subscript']
subscript = sub['subscript']['literal']['the_symbol']['symbol']
# TODO: Record literal and record of subscript should have
# different keys.
# Try to opimize and return the field from a record.
if 'record' in sub['record']:
for f_v in sub['record']['record']['field_value']:
if f_v['field'] == subscript:
# Optimizing and returning the field directly.
return self.ConvertToSql(f_v['value']['expression'])
# Trying to optmize subscript of implication.
if 'implication' in sub['record']:
simplified_sub = self.SubIfStruct(sub['record']['implication'],
subscript)
if simplified_sub:
return simplified_sub
# Couldn't optimize, just return the '.' expression.
record = self.ConvertToSql(sub['record'])
return self.Subscript(record, subscript)
if 'record' in expression:
record = expression['record']
return self.Record(record)
if 'combine' in expression:
return '(%s)' % (self.subquery_translator.TranslateRule(
expression['combine'], self.vocabulary, is_combine=True))
if 'implication' in expression:
implication = expression['implication']
return self.Implication(implication)
if 'call' in expression and 'predicate_name' in expression['call']:
raise self.exception_maker(
color.Format(
'Unsupported supposedly built-in function: '
'{warning}{predicate}{end}.',
dict(predicate=expression['call']['predicate_name'])))
assert False, (
'Logica bug: expression %s failed to compile for unknown reason.' %
str(expression))
def ShowMessage(self, stream=sys.stderr):
print(color.Format('{underline}Compiling{end}:'), file=stream)
print(self.rule_str, file=stream)
print(color.Format('\n[ {error}Error{end} ] ') + str(self),
file=stream)
def AsSql(self, subquery_encoder=None, flag_values=None):
"""Outputing SQL representing this structure."""
# pylint: disable=g-long-lambda
ql = expr_translate.QL(
self.VarsVocabulary(),
subquery_encoder,
lambda message: RuleCompileException(message, self.full_rule_text),
flag_values,
custom_udfs=subquery_encoder.execution.custom_udfs,
dialect=subquery_encoder.execution.dialect)
r = 'SELECT\n'
fields = []
if not self.select:
raise RuleCompileException(
color.Format(
'Tables with {warning}no columns{end} are not allowed in '
'StandardSQL, so they are not allowed in Logica.'),
self.full_rule_text)
for k, v in self.select.items():
if k == '*':
fields.append('%s.*' % ql.ConvertToSql(v))
else:
fields.append('%s AS %s' %
(ql.ConvertToSql(v), LogicaFieldToSqlField(k)))
r += ',\n'.join(' ' + f for f in fields)
if (self.tables or self.unnestings or self.constraints
or self.distinct_denoted):
r += '\nFROM\n'
tables = []
for k, v in self.tables.items():
if subquery_encoder:
# Note that we are passing external_vocabulary, not VarsVocabulary
# here. I.e. if this is a sub-query then variables of outer tables
# can be used.
sql = subquery_encoder.TranslateTable(
v, self.external_vocabulary)
if not sql:
raise RuleCompileException(
color.Format(
'Rule uses table {warning}{table}{end}, which is not '
'defined. External tables can not be used in '
'{warning}\'testrun\'{end} mode. This error may come '
'from injected sub-rules.', dict(table=v)),
self.full_rule_text)
if sql != k:
tables.append(sql + ' AS ' + k)
else:
tables.append(sql)
self.SortUnnestings()
for element, the_list in self.unnestings:
tables.append(
'UNNEST(%s) as %s' %
(ql.ConvertToSql(the_list), ql.ConvertToSql(element)))
if not tables:
tables.append('UNNEST(ARRAY[\'UNUSED\']) as unused_unnest')
from_str = ', '.join(tables)
# Indent the from_str.
from_str = '\n'.join(' ' + l for l in from_str.split('\n'))
r += from_str
if self.constraints:
r += '\nWHERE\n'
constraints = []
for c in self.constraints:
constraints.append(ql.ConvertToSql(c))
r += ' AND\n'.join(map(Indent2, constraints))
if self.distinct_vars:
ordered_distinct_vars = [
v for v in self.select.keys() if v in self.distinct_vars
]
r += '\nGROUP BY '
r += ', '.join(
map(LogicaFieldToSqlField, ordered_distinct_vars))
return r
def FunctionSql(self, name, allocator=None, internal_mode=False):
"""Print formatted SQL function creation statement."""
# TODO: Refactor this into FunctionSqlInternal and FunctionSql.
if not allocator:
allocator = self.NewNamesAllocator()
rules = list(self.GetPredicateRules(name))
# Check that the predicate is defined via a single rule.
if not rules:
raise rule_translate.RuleCompileException(
color.Format(
'No rules are defining {warning}{name}{end}, but compilation '
'was requested.', dict(name=name)), r' ¯\_(ツ)_/¯')
elif len(rules) > 1:
raise rule_translate.RuleCompileException(
color.Format(
'Predicate {warning}{name}{end} is defined by more than 1 rule '
'and can not be compiled into a function.',
dict(name=name)),
'\n\n'.join(r['full_text'] for r in rules))
[rule] = rules
# Extract structure and assert that it is isomorphic to a function.
s = rule_translate.ExtractRuleStructure(rule,
external_vocabulary=None,
names_allocator=allocator)
udf_variables = [
v if isinstance(v, str) else 'col%d' % v for v in s.select
if v != 'logica_value'
]
s.select = self.TurnPositionalIntoNamed(s.select)
variables = [v for v in s.select if v != 'logica_value']
if 0 in variables:
raise rule_translate.RuleCompileException(
color.Format(
'Predicate {warning}{name}{end} must have all aruments named for '
'compilation as a function.', dict(name=name)),
rule['full_text'])
for v in variables:
if ('variable' not in s.select[v]
or s.select[v]['variable']['var_name'] != v):
raise rule_translate.RuleCompileException(
color.Format(
'Predicate {warning}{name}{end} must not rename arguments '
'for compilation as a function.', dict(name=name)),
rule['full_text'])
vocabulary = {v: v for v in variables}
s.external_vocabulary = vocabulary
self.RunInjections(s, allocator)
s.ElliminateInternalVariables(assert_full_ellimination=True)
s.UnificationsToConstraints()
sql = s.AsSql(subquery_encoder=self.MakeSubqueryTranslator(allocator))
if s.constraints or s.unnestings or s.tables:
raise rule_translate.RuleCompileException(
color.Format(
'Predicate {warning}{name}{end} is not a simple function, but '
'compilation as function was requested. Full SQL:\n{sql}',
dict(name=name, sql=sql)), rule['full_text'])
if 'logica_value' not in s.select:
raise rule_translate.RuleCompileException(
color.Format(
'Predicate {warning}{name}{end} does not have a value, but '
'compilation as function was requested. Full SQL:\n%s' %
sql), rule['full_text'])
# pylint: disable=g-long-lambda
# Compile the function!
ql = expr_translate.QL(
vocabulary,
self.MakeSubqueryTranslator(allocator),
lambda message: rule_translate.RuleCompileException(
message, rule['full_text']),
self.flag_values,
custom_udfs=self.custom_udfs,
dialect=self.execution.dialect)
value_sql = ql.ConvertToSql(s.select['logica_value'])
sql = 'CREATE TEMP FUNCTION {name}({signature}) AS ({value})'.format(
name=name,
signature=', '.join('%s ANY TYPE' % v for v in variables),
value=value_sql)
sql = FormatSql(sql)
if internal_mode:
return ('%s(%s)' % (name, ', '.join('{%s}' % v
for v in udf_variables)), sql)
return sql
def Error(msg):
print(color.Format('[ {error}Error{end} ] {msg}', {'msg': msg}))
sys.exit(1)
def main(argv):
if len(argv) <= 1 or argv[1] == 'help':
print('Usage:')
print(' logica <l file> <command> <predicate name> [flags]')
print(' Commands are:')
print(' print: prints the StandardSQL query for the predicate.')
print(
' run: runs the StandardSQL query on BigQuery with pretty output.'
)
print(' run_to_csv: runs the query on BigQuery with csv output.')
print('')
print('')
print('Example:')
print(' python3 logica.py - run GoodIdea <<<\' '
'GoodIdea(snack: "carrots")\'')
return 1
if len(argv) == 3 and argv[2] == 'parse':
pass # compile needs just 2 actual arguments.
else:
if len(argv) < 4:
print('Not enough arguments. Run \'logica help\' for help.',
file=sys.stderr)
return 1
if argv[1] == '-':
filename = '/dev/stdin'
else:
filename = argv[1]
command = argv[2]
commands = ['parse', 'print', 'run', 'run_to_csv']
if command not in commands:
print(
color.Format(
'Unknown command {warning}{command}{end}. '
'Available commands: {commands}.',
dict(command=command, commands=', '.join(commands))))
return 1
if not os.path.exists(filename):
print('File not found: %s' % filename, file=sys.stderr)
return 1
program_text = open(filename).read()
try:
parsed_rules = parse.ParseFile(program_text,
import_root=GetImportRoot())['rule']
except parse.ParsingException as parsing_exception:
parsing_exception.ShowMessage()
sys.exit(1)
if command == 'parse':
# No indentation to avoid file size inflation.
print(json.dumps(parsed_rules, sort_keys=True, indent=''))
return 0
predicates = argv[3]
user_flags = ReadUserFlags(parsed_rules, argv[4:])
predicates_list = predicates.split(',')
for predicate in predicates_list:
try:
logic_program = universe.LogicaProgram(parsed_rules,
user_flags=user_flags)
formatted_sql = logic_program.FormattedPredicateSql(predicate)
preamble = logic_program.execution.preamble
defines_and_exports = logic_program.execution.defines_and_exports
main_predicate_sql = logic_program.execution.main_predicate_sql
except rule_translate.RuleCompileException as rule_compilation_exception:
rule_compilation_exception.ShowMessage()
sys.exit(1)
except functors.FunctorError as functor_exception:
functor_exception.ShowMessage()
sys.exit(1)
if command == 'print':
print(formatted_sql)
engine = logic_program.annotations.Engine()
if command == 'run' or command == 'run_to_csv':
# We should split and move this logic to dialects.
if engine == 'bigquery':
output_format = 'csv' if command == 'run_to_csv' else 'pretty'
p = subprocess.Popen([
'bq', 'query', '--use_legacy_sql=false',
'--format=%s' % output_format
],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
o, _ = p.communicate(formatted_sql.encode())
elif engine == 'sqlite':
# TODO: Make multi-statement scripts work.
format = ('artistictable' if command == 'run' else 'csv')
statements_to_execute = ([preamble] + defines_and_exports +
[main_predicate_sql])
o = sqlite3_logica.RunSqlScript(statements_to_execute,
format).encode()
elif engine == 'psql':
p = subprocess.Popen(
['psql', '--quiet'] +
(['--csv'] if command == 'run_to_csv' else []),
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
commands = []
o, _ = p.communicate('\n'.join(commands +
[formatted_sql]).encode())
elif engine == 'trino':
a = logic_program.annotations.annotations['@Engine']['trino']
params = GetTrinoParameters(a)
p = subprocess.Popen(
['trino'] + params +
(['--output-format=CSV_HEADER_UNQUOTED'] if command
== 'run_to_csv' else ['--output-format=ALIGNED']),
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
o, _ = p.communicate(formatted_sql.encode())
elif engine == 'presto':
a = logic_program.annotations.annotations['@Engine']['presto']
catalog = a.get('catalog', 'memory')
server = a.get('server', 'localhost:8080')
p = subprocess.Popen([
'presto',
'--catalog=%s' % catalog,
'--server=%s' % server, '--file=/dev/stdin'
] + (['--output-format=CSV_HEADER_UNQUOTED'] if command
== 'run_to_csv' else ['--output-format=ALIGNED']),
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
o, _ = p.communicate(formatted_sql.encode())
else:
assert False, 'Unknown engine: %s' % engine
print(o.decode())
def ShowMessage(self):
print(color.Format('{underline}Making{end}:'), file=sys.stderr)
print(self.functor_name, file=sys.stderr)
print(color.Format('\n[ {error}Error{end} ] ') + self.message,
file=sys.stderr)
def main(argv):
if len(argv) <= 1 or argv[1] == 'help':
print('Usage:')
print(' logica <l file> <command> <predicate name> [flags]')
print(' Commands are:')
print(' print: prints the StandardSQL query for the predicate.')
print(' run: runs the StandardSQL query on BigQuery with pretty output.')
print(' run_to_csv: runs the query on BigQuery with csv output.')
print('')
print('')
print('Example:')
print(' python3 logica.py - run GoodIdea <<<\' '
'GoodIdea(snack: "carrots")\'')
return 1
if len(argv) == 3 and argv[2] == 'parse':
pass # compile needs just 2 actual arguments.
else:
if len(argv) < 4:
print('Not enought arguments. Run \'logica help\' for help.',
file=sys.stderr)
return 1
if argv[1] == '-':
filename = '/dev/stdin'
else:
filename = argv[1]
command = argv[2]
commands = ['parse', 'print', 'run', 'run_to_csv']
if command not in commands:
print(color.Format('Unknown command {warning}{command}{end}. '
'Available commands: {commands}.',
dict(command=command, commands=', '.join(commands))))
return 1
if not os.path.exists(filename):
print('File not found: %s' % filename, file=sys.stderr)
return 1
program_text = open(filename).read()
try:
parsed_rules = parse.ParseFile(program_text)['rule']
except parse.ParsingException as parsing_exception:
parsing_exception.ShowMessage()
sys.exit(1)
if command == 'parse':
# No indentation to avoid file size inflation.
print(json.dumps(parsed_rules, sort_keys=True, indent=''))
return 0
predicates = argv[3]
user_flags = ReadUserFlags(parsed_rules, argv[4:])
predicates_list = predicates.split(',')
for predicate in predicates_list:
try:
p = universe.LogicaProgram(parsed_rules, user_flags=user_flags)
formatted_sql = p.FormattedPredicateSql(predicate)
except rule_translate.RuleCompileException as rule_compilation_exception:
rule_compilation_exception.ShowMessage()
sys.exit(1)
except functors.FunctorError as functor_exception:
functor_exception.ShowMessage()
sys.exit(1)
if command == 'print':
print(formatted_sql)
engine = p.annotations.Engine()
if command == 'run' or command == 'run_to_csv':
if engine == 'bigquery':
output_format = 'csv' if command == 'run_to_csv' else 'pretty'
p = subprocess.Popen(['bq', 'query',
'--use_legacy_sql=false',
'--format=%s' % output_format],
stdin=subprocess.PIPE, stdout=subprocess.PIPE)
o, _ = p.communicate(formatted_sql.encode())
elif engine == 'sqlite':
p = subprocess.Popen(['sqlite3'],
stdin=subprocess.PIPE, stdout=subprocess.PIPE)
commands = []
if command == 'run_to_csv':
commands.append('.mode csv')
o, _ = p.communicate(
'\n'.join(commands + [formatted_sql]).encode())
elif engine == 'psql':
p = subprocess.Popen(['psql', '--quiet'] +
(['--csv'] if command == 'run_to_csv' else []),
stdin=subprocess.PIPE, stdout=subprocess.PIPE)
commands = []
o, _ = p.communicate(
'\n'.join(commands + [formatted_sql]).encode())
else:
assert False, 'Unknown engine: %s' % engine
print(o.decode())
def RunInjections(self, s, allocator):
iterations = 0
while True:
iterations += 1
if iterations > sys.getrecursionlimit():
raise rule_translate.RuleCompileException(
RecursionError(), s.full_rule_text)
new_tables = collections.OrderedDict()
for table_name_rsql, table_predicate_rsql in s.tables.items():
rules = list(self.GetPredicateRules(table_predicate_rsql))
if (len(rules) == 1 and ('distinct_denoted' not in rules[0])
and
self.annotations.OkInjection(table_predicate_rsql)):
[r] = rules
rs = rule_translate.ExtractRuleStructure(
r, allocator, None)
rs.ElliminateInternalVariables(
assert_full_ellimination=False)
new_tables.update(rs.tables)
InjectStructure(s, rs)
new_vars_map = {}
new_inv_vars_map = {}
for (table_name,
table_var), clause_var in s.vars_map.items():
if table_name != table_name_rsql:
new_vars_map[table_name, table_var] = clause_var
new_inv_vars_map[clause_var] = (table_name,
table_var)
else:
if table_var not in rs.select:
if '*' in rs.select:
subscript = {
'literal': {
'the_symbol': {
'symbol': table_var
}
}
}
s.vars_unification.append({
'left': {
'variable': {
'var_name': clause_var
}
},
'right': {
'subscript': {
'subscript': subscript,
'record': rs.select['*']
}
}
})
else:
extra_hint = '' if table_var != '*' else (
' Are you using ..<rest of> for injectible predicate? '
'Please list the fields that you extract explicitly. '
'Tracking bug: b/131759583.')
raise rule_translate.RuleCompileException(
color.Format(
'Predicate {warning}{table_predicate_rsql}{end} '
'does not have an argument '
'{warning}{table_var}{end}, but '
'this rule tries to access it. {extra_hint}',
dict(table_predicate_rsql=
table_predicate_rsql,
table_var=table_var,
extra_hint=extra_hint)),
s.full_rule_text)
else:
s.vars_unification.append({
'left': {
'variable': {
'var_name': clause_var
}
},
'right':
rs.select[table_var]
})
s.vars_map = new_vars_map
s.inv_vars_map = new_inv_vars_map
else:
new_tables[table_name_rsql] = table_predicate_rsql
if s.tables == new_tables:
break
s.tables = new_tables
def ElliminateInternalVariables(self, assert_full_ellimination=False):
"""Elliminates internal variables via substitution."""
variables = self.InternalVariables()
while True:
done = True
for u in self.vars_unification:
for k, r in [['left', 'right'], ['right', 'left']]:
if u[k] == u[r]:
continue
ur_variables = AllMentionedVariables(u[r])
ur_variables_incl_combines = AllMentionedVariables(
u[r], dive_in_combines=True)
if (isinstance(u[k], dict) and 'variable' in u[k]
and u[k]['variable']['var_name'] in variables
and u[k]['variable']['var_name']
not in ur_variables_incl_combines and
(ur_variables <= self.ExtractedVariables() or not str(
u[k]['variable']['var_name']).startswith('x_'))):
u_left = u[k]['variable']['var_name']
u_right = u[r]
if 'variable' in u_right:
l = self.synonym_log.get(
u_right['variable']['var_name'], [])
l.append(u_left)
l.extend(self.synonym_log.get(u_left, []))
self.synonym_log[u_right['variable']
['var_name']] = l
ReplaceVariable(u_left, u_right, self.unnestings)
ReplaceVariable(u_left, u_right, self.select)
ReplaceVariable(u_left, u_right, self.vars_unification)
ReplaceVariable(u_left, u_right, self.constraints)
done = False
if done:
variables = self.InternalVariables()
if assert_full_ellimination:
if True:
if variables:
violators = []
for v in variables:
violators.extend(self.synonym_log.get(v, []))
violators.append(v)
violators = {
v
for v in violators if not v.startswith('x_')
}
assert violators, (
'Logica needs better error messages: purely internal '
'variable was not eliminated. It looks like you have '
'not passed a required argument to some called predicate. '
'Use --add_debug_info_to_var_names flag to make this message '
'a little more informatvie. '
'Variables: %s, synonym_log: %s' %
(str(variables), str(self.synonym_log)))
# Remove disambiguation suffixes from variables not to confuse
# the user.
violators = {
v.split(' # disambiguated')[0]
for v in violators
}
raise RuleCompileException(
color.Format(
'Found no way to assign variables: '
'{warning}{violators}{end}. '
'This error might also come from injected sub-rules.',
dict(violators=', '.join(sorted(
violators)))), self.full_rule_text)
else:
assert not variables, (
'Not all internal variables were eliminated. Violators:\n'
+ ',\n'.join('%s (aka %s)' %
(v, self.synonym_log[v])
for v in variables) +
'\nRule: %s' % self)
else:
unassigned_variables = []
for v in variables:
if not v.startswith('x_'):
unassigned_variables.append(v)
# Remove disambiguation suffixes from variables not to confuse
# the user.
unassigned_variables = {
v.split(' # disambiguated')[0]
for v in unassigned_variables
}
if unassigned_variables:
raise RuleCompileException(
color.Format(
'Found no way to assign variables: '
'{warning}{violators}{end}. '
'This error might also come from injected sub-rules.',
dict(violators=', '.join(
sorted(unassigned_variables)))),
self.full_rule_text)
break
