def UnfoldRecursivePredicate(self, predicate, cover, depth, rules): """Unfolds recurive predicate.""" new_predicate_name = predicate + '_recursive' new_predicate_head_name = predicate + '_recursive_head' def ReplaceRecursivePredicate(x): if isinstance(x, dict) and 'predicate_name' in x: if x['predicate_name'] == predicate: x['predicate_name'] = new_predicate_name return [] def ReplaceRecursiveHeadPredicate(x): if isinstance(x, dict) and 'predicate_name' in x: if x['predicate_name'] == predicate: x['predicate_name'] = new_predicate_head_name return [] def ReplacerOfCoverMember(member): def Replace(x): if isinstance(x, dict) and 'predicate_name' in x: if x['predicate_name'] == member: x['predicate_name'] = member + '_recursive_head' return [] return Replace for r in rules: if r['head']['predicate_name'] == predicate: r['head']['predicate_name'] = new_predicate_head_name Walk(r, ReplaceRecursivePredicate) for c in cover - {predicate}: Walk(r, ReplacerOfCoverMember(c)) elif r['head']['predicate_name'] in cover: Walk(r, ReplaceRecursivePredicate) for c in cover - {predicate}: Walk(r, ReplacerOfCoverMember(c)) elif (r['head']['predicate_name'][0] == '@' and r['head']['predicate_name'] != '@Make'): Walk(r, ReplaceRecursiveHeadPredicate) for c in cover - {predicate}: Walk(r, ReplacerOfCoverMember(c)) else: # This rule simply uses the predicate, keep the name. pass lib = recursion_library.GetRecursionFunctor(depth) lib = lib.replace('P', predicate) lib_rules = parse.ParseFile(lib)['rule'] rules.extend(lib_rules) for c in cover - {predicate}: rename_lib = recursion_library.GetRenamingFunctor(c, predicate) rename_lib_rules = parse.ParseFile(rename_lib)['rule'] rules.extend(rename_lib_rules)
def __init__(self, rules, table_aliases=None, user_flags=None): """Initializes the program. Args: rules: A list of dictionary representations of parsed Logica rules. table_aliases: A map from an undefined Logica predicate name to a BigQuery table name. This table will be used in place of predicate. user_flags: Dictionary of user specified flags. """ rules = self.UnfoldRecursion(rules) # TODO: Should allocator be a member of Logica? self.preparsed_rules = rules self.rules = [] self.defined_predicates = set() self.dollar_params = list(self.ExtractDollarParams(rules)) self.table_aliases = table_aliases or {} self.execution = None self.user_flags = user_flags or {} self.annotations = Annotations(rules, self.user_flags) self.flag_values = self.annotations.flag_values # Dictionary custom_udfs maps function name to a format string to use # in queries. self.custom_udfs = collections.OrderedDict() # Dictionary custom_udf_definitions maps function name to SQL defining the # function. self.custom_udf_definitions = collections.OrderedDict() if not set(self.dollar_params) <= set(self.flag_values): raise rule_translate.RuleCompileException( 'Parameters %s are undefined.' % (list(set(self.dollar_params) - set(self.flag_values))), str(list(set(self.dollar_params) - set(self.flag_values)))) self.functors = None # Extending rules with functors. extended_rules = self.RunMakes(rules) # Populates self.functors. # Extending rules with the library of the dialect. library_rules = parse.ParseFile( dialects.Get(self.annotations.Engine()).LibraryProgram())['rule'] extended_rules.extend(library_rules) for rule in extended_rules: predicate_name = rule['head']['predicate_name'] self.defined_predicates.add(predicate_name) self.rules.append((predicate_name, rule)) # We need to recompute annotations, because 'Make' created more rules and # annotations. self.annotations = Annotations(extended_rules, self.user_flags) # Build udfs, populating custom_udfs and custom_udf_definitions. self.BuildUdfs() # Function compilation may have added irrelevant defines: self.execution = None if False: self.RunTypechecker()
def ParseOrExit(filename): """Parse a Logica program.""" with open(filename) as f: program_text = f.read() try: parsed_rules = parse.ParseFile(program_text)['rule'] except parse.ParsingException as parsing_exception: parsing_exception.ShowMessage() sys.exit(1) return parsed_rules
def UnfoldRecursivePredicate(self, predicate, depth, rules): """Unfolds recurive predicate.""" new_predicate_name = predicate + '_recursive' new_predicate_head_name = predicate + '_recursive_head' def ReplaceRecursivePredicate(x): if isinstance(x, dict) and 'predicate_name' in x: if x['predicate_name'] == predicate: x['predicate_name'] = new_predicate_name return [] for r in rules: if r['head']['predicate_name'] == predicate: r['head']['predicate_name'] = new_predicate_head_name Walk(r, ReplaceRecursivePredicate, lambda _: True) elif predicate + '_MultBodyAggAux' == r['head']['predicate_name']: Walk(r, ReplaceRecursivePredicate, lambda _: True) else: # This rule simply uses the predicate, keep the name. pass lib = recursion_library.GetRecursionFunctor(depth) lib = lib.replace('P', predicate) lib_rules = parse.ParseFile(lib)['rule'] rules.extend(lib_rules)
def main(argv): if len(argv) <= 1 or argv[1] == 'help': print('Usage:') print(' logica <l file> <command> <predicate name> [flags]') print(' Commands are:') print(' print: prints the StandardSQL query for the predicate.') print( ' run: runs the StandardSQL query on BigQuery with pretty output.' ) print(' run_to_csv: runs the query on BigQuery with csv output.') print('') print('') print('Example:') print(' python3 logica.py - run GoodIdea <<<\' ' 'GoodIdea(snack: "carrots")\'') return 1 if len(argv) == 3 and argv[2] == 'parse': pass # compile needs just 2 actual arguments. else: if len(argv) < 4: print('Not enough arguments. Run \'logica help\' for help.', file=sys.stderr) return 1 if argv[1] == '-': filename = '/dev/stdin' else: filename = argv[1] command = argv[2] commands = ['parse', 'print', 'run', 'run_to_csv'] if command not in commands: print( color.Format( 'Unknown command {warning}{command}{end}. ' 'Available commands: {commands}.', dict(command=command, commands=', '.join(commands)))) return 1 if not os.path.exists(filename): print('File not found: %s' % filename, file=sys.stderr) return 1 program_text = open(filename).read() try: parsed_rules = parse.ParseFile(program_text, import_root=GetImportRoot())['rule'] except parse.ParsingException as parsing_exception: parsing_exception.ShowMessage() sys.exit(1) if command == 'parse': # No indentation to avoid file size inflation. print(json.dumps(parsed_rules, sort_keys=True, indent='')) return 0 predicates = argv[3] user_flags = ReadUserFlags(parsed_rules, argv[4:]) predicates_list = predicates.split(',') for predicate in predicates_list: try: logic_program = universe.LogicaProgram(parsed_rules, user_flags=user_flags) formatted_sql = logic_program.FormattedPredicateSql(predicate) preamble = logic_program.execution.preamble defines_and_exports = logic_program.execution.defines_and_exports main_predicate_sql = logic_program.execution.main_predicate_sql except rule_translate.RuleCompileException as rule_compilation_exception: rule_compilation_exception.ShowMessage() sys.exit(1) except functors.FunctorError as functor_exception: functor_exception.ShowMessage() sys.exit(1) if command == 'print': print(formatted_sql) engine = logic_program.annotations.Engine() if command == 'run' or command == 'run_to_csv': # We should split and move this logic to dialects. if engine == 'bigquery': output_format = 'csv' if command == 'run_to_csv' else 'pretty' p = subprocess.Popen([ 'bq', 'query', '--use_legacy_sql=false', '--format=%s' % output_format ], stdin=subprocess.PIPE, stdout=subprocess.PIPE) o, _ = p.communicate(formatted_sql.encode()) elif engine == 'sqlite': # TODO: Make multi-statement scripts work. format = ('artistictable' if command == 'run' else 'csv') statements_to_execute = ([preamble] + defines_and_exports + [main_predicate_sql]) o = sqlite3_logica.RunSqlScript(statements_to_execute, format).encode() elif engine == 'psql': p = subprocess.Popen( ['psql', '--quiet'] + (['--csv'] if command == 'run_to_csv' else []), stdin=subprocess.PIPE, stdout=subprocess.PIPE) commands = [] o, _ = p.communicate('\n'.join(commands + [formatted_sql]).encode()) elif engine == 'trino': a = logic_program.annotations.annotations['@Engine']['trino'] params = GetTrinoParameters(a) p = subprocess.Popen( ['trino'] + params + (['--output-format=CSV_HEADER_UNQUOTED'] if command == 'run_to_csv' else ['--output-format=ALIGNED']), stdin=subprocess.PIPE, stdout=subprocess.PIPE) o, _ = p.communicate(formatted_sql.encode()) elif engine == 'presto': a = logic_program.annotations.annotations['@Engine']['presto'] catalog = a.get('catalog', 'memory') server = a.get('server', 'localhost:8080') p = subprocess.Popen([ 'presto', '--catalog=%s' % catalog, '--server=%s' % server, '--file=/dev/stdin' ] + (['--output-format=CSV_HEADER_UNQUOTED'] if command == 'run_to_csv' else ['--output-format=ALIGNED']), stdin=subprocess.PIPE, stdout=subprocess.PIPE) o, _ = p.communicate(formatted_sql.encode()) else: assert False, 'Unknown engine: %s' % engine print(o.decode())
def main(argv): if len(argv) <= 1 or argv[1] == 'help': print('Usage:') print(' logica <l file> <command> <predicate name> [flags]') print(' Commands are:') print(' print: prints the StandardSQL query for the predicate.') print(' run: runs the StandardSQL query on BigQuery with pretty output.') print(' run_to_csv: runs the query on BigQuery with csv output.') print('') print('') print('Example:') print(' python3 logica.py - run GoodIdea <<<\' ' 'GoodIdea(snack: "carrots")\'') return 1 if len(argv) == 3 and argv[2] == 'parse': pass # compile needs just 2 actual arguments. else: if len(argv) < 4: print('Not enought arguments. Run \'logica help\' for help.', file=sys.stderr) return 1 if argv[1] == '-': filename = '/dev/stdin' else: filename = argv[1] command = argv[2] commands = ['parse', 'print', 'run', 'run_to_csv'] if command not in commands: print(color.Format('Unknown command {warning}{command}{end}. ' 'Available commands: {commands}.', dict(command=command, commands=', '.join(commands)))) return 1 if not os.path.exists(filename): print('File not found: %s' % filename, file=sys.stderr) return 1 program_text = open(filename).read() try: parsed_rules = parse.ParseFile(program_text)['rule'] except parse.ParsingException as parsing_exception: parsing_exception.ShowMessage() sys.exit(1) if command == 'parse': # No indentation to avoid file size inflation. print(json.dumps(parsed_rules, sort_keys=True, indent='')) return 0 predicates = argv[3] user_flags = ReadUserFlags(parsed_rules, argv[4:]) predicates_list = predicates.split(',') for predicate in predicates_list: try: p = universe.LogicaProgram(parsed_rules, user_flags=user_flags) formatted_sql = p.FormattedPredicateSql(predicate) except rule_translate.RuleCompileException as rule_compilation_exception: rule_compilation_exception.ShowMessage() sys.exit(1) except functors.FunctorError as functor_exception: functor_exception.ShowMessage() sys.exit(1) if command == 'print': print(formatted_sql) engine = p.annotations.Engine() if command == 'run' or command == 'run_to_csv': if engine == 'bigquery': output_format = 'csv' if command == 'run_to_csv' else 'pretty' p = subprocess.Popen(['bq', 'query', '--use_legacy_sql=false', '--format=%s' % output_format], stdin=subprocess.PIPE, stdout=subprocess.PIPE) o, _ = p.communicate(formatted_sql.encode()) elif engine == 'sqlite': p = subprocess.Popen(['sqlite3'], stdin=subprocess.PIPE, stdout=subprocess.PIPE) commands = [] if command == 'run_to_csv': commands.append('.mode csv') o, _ = p.communicate( '\n'.join(commands + [formatted_sql]).encode()) elif engine == 'psql': p = subprocess.Popen(['psql', '--quiet'] + (['--csv'] if command == 'run_to_csv' else []), stdin=subprocess.PIPE, stdout=subprocess.PIPE) commands = [] o, _ = p.communicate( '\n'.join(commands + [formatted_sql]).encode()) else: assert False, 'Unknown engine: %s' % engine print(o.decode())