def UnfoldRecursivePredicate(self, predicate, cover, depth, rules):
"""Unfolds recurive predicate."""
new_predicate_name = predicate + '_recursive'
new_predicate_head_name = predicate + '_recursive_head'
def ReplaceRecursivePredicate(x):
if isinstance(x, dict) and 'predicate_name' in x:
if x['predicate_name'] == predicate:
x['predicate_name'] = new_predicate_name
return []
def ReplaceRecursiveHeadPredicate(x):
if isinstance(x, dict) and 'predicate_name' in x:
if x['predicate_name'] == predicate:
x['predicate_name'] = new_predicate_head_name
return []
def ReplacerOfCoverMember(member):
def Replace(x):
if isinstance(x, dict) and 'predicate_name' in x:
if x['predicate_name'] == member:
x['predicate_name'] = member + '_recursive_head'
return []
return Replace
for r in rules:
if r['head']['predicate_name'] == predicate:
r['head']['predicate_name'] = new_predicate_head_name
Walk(r, ReplaceRecursivePredicate)
for c in cover - {predicate}:
Walk(r, ReplacerOfCoverMember(c))
elif r['head']['predicate_name'] in cover:
Walk(r, ReplaceRecursivePredicate)
for c in cover - {predicate}:
Walk(r, ReplacerOfCoverMember(c))
elif (r['head']['predicate_name'][0] == '@'
and r['head']['predicate_name'] != '@Make'):
Walk(r, ReplaceRecursiveHeadPredicate)
for c in cover - {predicate}:
Walk(r, ReplacerOfCoverMember(c))
else:
# This rule simply uses the predicate, keep the name.
pass
lib = recursion_library.GetRecursionFunctor(depth)
lib = lib.replace('P', predicate)
lib_rules = parse.ParseFile(lib)['rule']
rules.extend(lib_rules)
for c in cover - {predicate}:
rename_lib = recursion_library.GetRenamingFunctor(c, predicate)
rename_lib_rules = parse.ParseFile(rename_lib)['rule']
rules.extend(rename_lib_rules)
def __init__(self, rules, table_aliases=None, user_flags=None):
"""Initializes the program.
Args:
rules: A list of dictionary representations of parsed Logica rules.
table_aliases: A map from an undefined Logica predicate name to a
BigQuery table name. This table will be used in place of predicate.
user_flags: Dictionary of user specified flags.
"""
rules = self.UnfoldRecursion(rules)
# TODO: Should allocator be a member of Logica?
self.preparsed_rules = rules
self.rules = []
self.defined_predicates = set()
self.dollar_params = list(self.ExtractDollarParams(rules))
self.table_aliases = table_aliases or {}
self.execution = None
self.user_flags = user_flags or {}
self.annotations = Annotations(rules, self.user_flags)
self.flag_values = self.annotations.flag_values
# Dictionary custom_udfs maps function name to a format string to use
# in queries.
self.custom_udfs = collections.OrderedDict()
# Dictionary custom_udf_definitions maps function name to SQL defining the
# function.
self.custom_udf_definitions = collections.OrderedDict()
if not set(self.dollar_params) <= set(self.flag_values):
raise rule_translate.RuleCompileException(
'Parameters %s are undefined.' %
(list(set(self.dollar_params) - set(self.flag_values))),
str(list(set(self.dollar_params) - set(self.flag_values))))
self.functors = None
# Extending rules with functors.
extended_rules = self.RunMakes(rules) # Populates self.functors.
# Extending rules with the library of the dialect.
library_rules = parse.ParseFile(
dialects.Get(self.annotations.Engine()).LibraryProgram())['rule']
extended_rules.extend(library_rules)
for rule in extended_rules:
predicate_name = rule['head']['predicate_name']
self.defined_predicates.add(predicate_name)
self.rules.append((predicate_name, rule))
# We need to recompute annotations, because 'Make' created more rules and
# annotations.
self.annotations = Annotations(extended_rules, self.user_flags)
# Build udfs, populating custom_udfs and custom_udf_definitions.
self.BuildUdfs()
# Function compilation may have added irrelevant defines:
self.execution = None
if False:
self.RunTypechecker()
def ParseOrExit(filename):
"""Parse a Logica program."""
with open(filename) as f:
program_text = f.read()
try:
parsed_rules = parse.ParseFile(program_text)['rule']
except parse.ParsingException as parsing_exception:
parsing_exception.ShowMessage()
sys.exit(1)
return parsed_rules
def UnfoldRecursivePredicate(self, predicate, depth, rules):
"""Unfolds recurive predicate."""
new_predicate_name = predicate + '_recursive'
new_predicate_head_name = predicate + '_recursive_head'
def ReplaceRecursivePredicate(x):
if isinstance(x, dict) and 'predicate_name' in x:
if x['predicate_name'] == predicate:
x['predicate_name'] = new_predicate_name
return []
for r in rules:
if r['head']['predicate_name'] == predicate:
r['head']['predicate_name'] = new_predicate_head_name
Walk(r, ReplaceRecursivePredicate, lambda _: True)
elif predicate + '_MultBodyAggAux' == r['head']['predicate_name']:
Walk(r, ReplaceRecursivePredicate, lambda _: True)
else:
# This rule simply uses the predicate, keep the name.
pass
lib = recursion_library.GetRecursionFunctor(depth)
lib = lib.replace('P', predicate)
lib_rules = parse.ParseFile(lib)['rule']
rules.extend(lib_rules)
def main(argv):
if len(argv) <= 1 or argv[1] == 'help':
print('Usage:')
print(' logica <l file> <command> <predicate name> [flags]')
print(' Commands are:')
print(' print: prints the StandardSQL query for the predicate.')
print(
' run: runs the StandardSQL query on BigQuery with pretty output.'
)
print(' run_to_csv: runs the query on BigQuery with csv output.')
print('')
print('')
print('Example:')
print(' python3 logica.py - run GoodIdea <<<\' '
'GoodIdea(snack: "carrots")\'')
return 1
if len(argv) == 3 and argv[2] == 'parse':
pass # compile needs just 2 actual arguments.
else:
if len(argv) < 4:
print('Not enough arguments. Run \'logica help\' for help.',
file=sys.stderr)
return 1
if argv[1] == '-':
filename = '/dev/stdin'
else:
filename = argv[1]
command = argv[2]
commands = ['parse', 'print', 'run', 'run_to_csv']
if command not in commands:
print(
color.Format(
'Unknown command {warning}{command}{end}. '
'Available commands: {commands}.',
dict(command=command, commands=', '.join(commands))))
return 1
if not os.path.exists(filename):
print('File not found: %s' % filename, file=sys.stderr)
return 1
program_text = open(filename).read()
try:
parsed_rules = parse.ParseFile(program_text,
import_root=GetImportRoot())['rule']
except parse.ParsingException as parsing_exception:
parsing_exception.ShowMessage()
sys.exit(1)
if command == 'parse':
# No indentation to avoid file size inflation.
print(json.dumps(parsed_rules, sort_keys=True, indent=''))
return 0
predicates = argv[3]
user_flags = ReadUserFlags(parsed_rules, argv[4:])
predicates_list = predicates.split(',')
for predicate in predicates_list:
try:
logic_program = universe.LogicaProgram(parsed_rules,
user_flags=user_flags)
formatted_sql = logic_program.FormattedPredicateSql(predicate)
preamble = logic_program.execution.preamble
defines_and_exports = logic_program.execution.defines_and_exports
main_predicate_sql = logic_program.execution.main_predicate_sql
except rule_translate.RuleCompileException as rule_compilation_exception:
rule_compilation_exception.ShowMessage()
sys.exit(1)
except functors.FunctorError as functor_exception:
functor_exception.ShowMessage()
sys.exit(1)
if command == 'print':
print(formatted_sql)
engine = logic_program.annotations.Engine()
if command == 'run' or command == 'run_to_csv':
# We should split and move this logic to dialects.
if engine == 'bigquery':
output_format = 'csv' if command == 'run_to_csv' else 'pretty'
p = subprocess.Popen([
'bq', 'query', '--use_legacy_sql=false',
'--format=%s' % output_format
],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
o, _ = p.communicate(formatted_sql.encode())
elif engine == 'sqlite':
# TODO: Make multi-statement scripts work.
format = ('artistictable' if command == 'run' else 'csv')
statements_to_execute = ([preamble] + defines_and_exports +
[main_predicate_sql])
o = sqlite3_logica.RunSqlScript(statements_to_execute,
format).encode()
elif engine == 'psql':
p = subprocess.Popen(
['psql', '--quiet'] +
(['--csv'] if command == 'run_to_csv' else []),
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
commands = []
o, _ = p.communicate('\n'.join(commands +
[formatted_sql]).encode())
elif engine == 'trino':
a = logic_program.annotations.annotations['@Engine']['trino']
params = GetTrinoParameters(a)
p = subprocess.Popen(
['trino'] + params +
(['--output-format=CSV_HEADER_UNQUOTED'] if command
== 'run_to_csv' else ['--output-format=ALIGNED']),
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
o, _ = p.communicate(formatted_sql.encode())
elif engine == 'presto':
a = logic_program.annotations.annotations['@Engine']['presto']
catalog = a.get('catalog', 'memory')
server = a.get('server', 'localhost:8080')
p = subprocess.Popen([
'presto',
'--catalog=%s' % catalog,
'--server=%s' % server, '--file=/dev/stdin'
] + (['--output-format=CSV_HEADER_UNQUOTED'] if command
== 'run_to_csv' else ['--output-format=ALIGNED']),
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
o, _ = p.communicate(formatted_sql.encode())
else:
assert False, 'Unknown engine: %s' % engine
print(o.decode())
def main(argv):
if len(argv) <= 1 or argv[1] == 'help':
print('Usage:')
print(' logica <l file> <command> <predicate name> [flags]')
print(' Commands are:')
print(' print: prints the StandardSQL query for the predicate.')
print(' run: runs the StandardSQL query on BigQuery with pretty output.')
print(' run_to_csv: runs the query on BigQuery with csv output.')
print('')
print('')
print('Example:')
print(' python3 logica.py - run GoodIdea <<<\' '
'GoodIdea(snack: "carrots")\'')
return 1
if len(argv) == 3 and argv[2] == 'parse':
pass # compile needs just 2 actual arguments.
else:
if len(argv) < 4:
print('Not enought arguments. Run \'logica help\' for help.',
file=sys.stderr)
return 1
if argv[1] == '-':
filename = '/dev/stdin'
else:
filename = argv[1]
command = argv[2]
commands = ['parse', 'print', 'run', 'run_to_csv']
if command not in commands:
print(color.Format('Unknown command {warning}{command}{end}. '
'Available commands: {commands}.',
dict(command=command, commands=', '.join(commands))))
return 1
if not os.path.exists(filename):
print('File not found: %s' % filename, file=sys.stderr)
return 1
program_text = open(filename).read()
try:
parsed_rules = parse.ParseFile(program_text)['rule']
except parse.ParsingException as parsing_exception:
parsing_exception.ShowMessage()
sys.exit(1)
if command == 'parse':
# No indentation to avoid file size inflation.
print(json.dumps(parsed_rules, sort_keys=True, indent=''))
return 0
predicates = argv[3]
user_flags = ReadUserFlags(parsed_rules, argv[4:])
predicates_list = predicates.split(',')
for predicate in predicates_list:
try:
p = universe.LogicaProgram(parsed_rules, user_flags=user_flags)
formatted_sql = p.FormattedPredicateSql(predicate)
except rule_translate.RuleCompileException as rule_compilation_exception:
rule_compilation_exception.ShowMessage()
sys.exit(1)
except functors.FunctorError as functor_exception:
functor_exception.ShowMessage()
sys.exit(1)
if command == 'print':
print(formatted_sql)
engine = p.annotations.Engine()
if command == 'run' or command == 'run_to_csv':
if engine == 'bigquery':
output_format = 'csv' if command == 'run_to_csv' else 'pretty'
p = subprocess.Popen(['bq', 'query',
'--use_legacy_sql=false',
'--format=%s' % output_format],
stdin=subprocess.PIPE, stdout=subprocess.PIPE)
o, _ = p.communicate(formatted_sql.encode())
elif engine == 'sqlite':
p = subprocess.Popen(['sqlite3'],
stdin=subprocess.PIPE, stdout=subprocess.PIPE)
commands = []
if command == 'run_to_csv':
commands.append('.mode csv')
o, _ = p.communicate(
'\n'.join(commands + [formatted_sql]).encode())
elif engine == 'psql':
p = subprocess.Popen(['psql', '--quiet'] +
(['--csv'] if command == 'run_to_csv' else []),
stdin=subprocess.PIPE, stdout=subprocess.PIPE)
commands = []
o, _ = p.communicate(
'\n'.join(commands + [formatted_sql]).encode())
else:
assert False, 'Unknown engine: %s' % engine
print(o.decode())