def PredicateSql(self, name, allocator=None, external_vocabulary=None): """Producing SQL for a predicate.""" # Load proto if necessary. rules = list(self.GetPredicateRules(name)) if len(rules) == 1: [rule] = rules return (self.SingleRuleSql(rule, allocator, external_vocabulary) + self.annotations.OrderByClause(name) + self.annotations.LimitClause(name)) elif len(rules) > 1: rules_sql = [] for rule in rules: if 'distinct_denoted' in rule: raise rule_translate.RuleCompileException( color.Format( 'For distinct denoted predicates multiple rules are not ' 'currently supported. Consider taking ' '{warning}union of bodies manually{end}, if that was what ' 'you intended.'), rule['full_text']) rules_sql.append('\n%s\n' % Indent2( self.SingleRuleSql(rule, allocator, external_vocabulary))) rules_sql = [ '\n'.join(' ' + l for l in r.split('\n')) for r in rules_sql ] return 'SELECT * FROM (\n%s\n) AS UNUSED_TABLE_NAME %s %s' % ( ' UNION ALL\n'.join(rules_sql), self.annotations.OrderByClause(name), self.annotations.LimitClause(name)) else: raise rule_translate.RuleCompileException( color.Format( 'No rules are defining {warning}{name}{end}, but compilation ' 'was requested.', dict(name=name)), r' ¯\_(ツ)_/¯')
def RunTest(name, src, predicate, golden, user_flags=None, overwrite=False, announce=False): """Run one test.""" if announce: print('Running test:', name) test_result = '{warning}RUNNING{end}' print(color.Format('% 50s %s' % (name, test_result))) result = logica_lib.RunPredicate(src, predicate, user_flags=user_flags) # Hacky way to remove query that BQ prints. if '+---' in result[200:]: result = result[result.index('+---'):] if overwrite: with open(golden, 'w') as w: w.write(result) golden_result = open(golden).read() if result == golden_result: test_result = '{ok}PASSED{end}' else: p = subprocess.Popen(['diff', '-', golden], stdin=subprocess.PIPE) p.communicate(result.encode()) test_result = '{error}FAILED{end}' print('\033[F\033[K' + color.Format('% 50s %s' % (name, test_result)))
def PrintHeader(): """Print header for all tests.""" print( color.Format('% 64s %s' % ('{warning}TEST{end}', '{warning}RESULT{end}'))) print( color.Format('% 64s %s' % ('{warning}----{end}', '{warning}------{end}')))
def SortUnnestings(self): """Sorts unnestings in dependency order.""" unnesting_of = { u[0]['variable']['var_name']: u for u in self.unnestings } unnesting_variables = set(unnesting_of) depends_on = { u[0]['variable']['var_name']: set(AllMentionedVariables(u[1], dive_in_combines=True)) & unnesting_variables for u in self.unnestings } unnested = set() ordered_unnestings = [] while unnesting_of: for v, u in sorted(unnesting_of.items()): if depends_on[v] <= unnested: ordered_unnestings.append(unnesting_of[v]) del unnesting_of[v] unnested.add(v) break else: raise RuleCompileException( color.Format( 'There seem to be a circular dependency of {warning}In{end} ' 'calls. ' 'This error might also come from injected sub-rules.'), self.full_rule_text) self.unnestings = ordered_unnestings
def ShowMessage(self, stream=sys.stderr): """Printing error message.""" print(color.Format('{underline}Parsing{end}:'), file=stream) before, error, after = self.location.Pieces() if len(before) > 300: before = before[-300:] if len(after) > 300: after = after[:300] if not error: error = '<EMPTY>' print(color.Format('{before}{warning}{error_text}{end}{after}', dict(before=before, error_text=error, after=after)), file=stream) print( color.Format('\n[ {error}Error{end} ] ') + str(self), file=stream)
def With(self, predicate_name): """Return whether this predicate should be compiled to a WITH-table. This only applies if the predicate is not inlined earlier in the flow. """ is_with = self.ForceWith(predicate_name) is_nowith = self.ForceNoWith(predicate_name) if is_with and is_nowith: raise rule_translate.RuleCompileException( color.Format('Predicate is annotated both with @With and @NoWith.'), 'Predicate: %s' % predicate_name) if is_with: return True if is_nowith or self.Ground(predicate_name): return False # TODO: return false for predicates that will be injected. return True
def ExtractRuleStructure(rule, names_allocator=None, external_vocabulary=None): """Extracts RuleStructure from rule.""" rule = copy.deepcopy(rule) # Not disambiguating if this rule is extracting structure of the combine # itself, as variables of this combine were already disambiguated from # parent rule. if rule['head']['predicate_name'] != 'Combine': DisambiguateCombineVariables(rule, names_allocator) s = RuleStructure(names_allocator, external_vocabulary) InlinePredicateValues(rule, names_allocator) s.full_rule_text = rule['full_text'] s.this_predicate_name = rule['head']['predicate_name'] (s.select, aggregated_vars) = HeadToSelect(rule['head']) # Adding internal variable unification with select arguments to avoid # confusion of user variables between injected predicates. for k, expr in s.select.items(): if 'variable' in expr: s.vars_unification.append({ 'left': expr, 'right': { 'variable': { 'var_name': names_allocator.AllocateVar('extract_%s_%s' % (s.this_predicate_name, k)) } } }) if 'body' in rule: ExtractConjunctiveStructure(rule['body']['conjunction']['conjunct'], s) distinct_denoted = 'distinct_denoted' in rule s.distinct_denoted = distinct_denoted if aggregated_vars and not distinct_denoted: raise RuleCompileException( color.Format( 'Aggregating predicate must be {warning}distinct{end} denoted.' ), s.full_rule_text) if distinct_denoted: s.distinct_vars = sorted( list(set(s.select.keys()) - set(aggregated_vars)), key=str) return s
def ExtractAnnotations(cls, rules, restrict_to=None, flag_values=None): """Extracting annotations from the rules.""" result = { p: collections.OrderedDict() for p in cls.ANNOTATING_PREDICATES } for rule in rules: rule_predicate = rule['head']['predicate_name'] if restrict_to and rule_predicate not in restrict_to: continue if (rule_predicate[0] == '@' and rule_predicate not in cls.ANNOTATING_PREDICATES): raise rule_translate.RuleCompileException( 'Only {0} and {1} special predicates are allowed.'.format( ', '.join(cls.ANNOTATING_PREDICATES[:-1]), cls.ANNOTATING_PREDICATES[-1]), rule['full_text']) if rule_predicate in cls.ANNOTATING_PREDICATES: rule_text = rule['full_text'] # pylint: disable=cell-var-from-loop def ThrowException(*args, **xargs): _ = args _ = xargs if rule_predicate == '@Make': # pylint: disable=raising-format-tuple raise rule_translate.RuleCompileException( 'Incorrect syntax for functor call. ' 'Functor call to be made as\n' ' R := F(A: V, ...)\n' 'or\n' ' @Make(R, F, {A: V, ...})\n' 'Where R, F, A\'s and V\'s are all ' 'predicate names.', rule_text) else: raise rule_translate.RuleCompileException( 'Can not understand annotation.', rule_text) class Thrower(object): def __contains__(self, key): if rule_predicate == '@Make': ThrowException() return raise rule_translate.RuleCompileException( 'Annotation may not use variables, but this one uses ' 'variable %s.' % (color.Warn(key)), rule_text) flag_values = flag_values or Thrower() ql = expr_translate.QL(Thrower(), ThrowException, ThrowException, flag_values) ql.convert_to_json = True annotation = rule['head']['predicate_name'] field_values_json_str = ql.ConvertToSql( {'record': rule['head']['record']}) try: field_values = json.loads(field_values_json_str) except: raise rule_translate.RuleCompileException( 'Could not understand arguments of annotation.', rule['full_text']) if ('0' in field_values and isinstance(field_values['0'], dict) and 'predicate_name' in field_values['0']): subject = field_values['0']['predicate_name'] else: subject = field_values['0'] del field_values['0'] if rule_predicate in ['@OrderBy', '@Limit', '@NoInject']: field_values_list = FieldValuesAsList(field_values) if field_values_list is None: raise rule_translate.RuleCompileException( '@OrderBy and @Limit may only have positional ' 'arguments.', rule['full_text']) if rule_predicate == '@Limit' and len( field_values_list) != 1: raise rule_translate.RuleCompileException( 'Annotation @Limit must have exactly two arguments: ' 'predicate and limit.', rule['full_text']) updated_annotation = result.get(annotation, {}) field_values['__rule_text'] = rule['full_text'] if subject in updated_annotation: raise rule_translate.RuleCompileException( color.Format( '{annotation} annotates {warning}{subject}{end} more ' 'than once: {before}, {after}', dict(annotation=annotation, subject=subject, before=updated_annotation[subject] ['__rule_text'], after=field_values['__rule_text'])), rule['full_text']) updated_annotation[subject] = field_values result[annotation] = updated_annotation return result
def RecursionError(): return color.Format('The rule appears to use recursion. ' '{warning}Recursion{end} is neither supported by ' 'Logica nor by StandardSQL.')
def ConvertToSql(self, expression): """Converting Logica expression into SQL.""" # print('EXPR:', expression) # Variables. if 'variable' in expression: return self.Variable(expression['variable']) # Literals. if 'literal' in expression: literal = expression['literal'] if 'the_number' in literal: return self.IntLiteral(literal['the_number']) if 'the_string' in literal: return self.StrLiteral(literal['the_string']) if 'the_list' in literal: return self.ListLiteral(literal['the_list']) if 'the_bool' in literal: return self.BoolLiteral(literal['the_bool']) if 'the_null' in literal: return self.NullLiteral(literal['the_null']) if 'the_predicate' in literal: return self.PredicateLiteral(literal['the_predicate']) assert False, 'Logica bug: unsupported literal parsed: %s' % literal if 'call' in expression: call = expression['call'] arguments = self.ConvertRecord(call['record']) if call['predicate_name'] in self.ANALYTIC_FUNCTIONS: return self.ConvertAnalytic(call) if call['predicate_name'] == 'SqlExpr': return self.GenericSqlExpression(call['record']) if call['predicate_name'] == 'Cast': if (len(arguments) != 2 or 'literal' not in call['record'] ['field_value'][1]['value']['expression'] or 'the_string' not in call['record']['field_value'][1] ['value']['expression']['literal']): raise self.exception_maker( 'Cast must have 2 arguments and the second argument must be a ' 'string literal.') cast_to = ( call['record']['field_value'][1]['value']['expression'] ['literal']['the_string']['the_string']) return 'CAST(%s AS %s)' % (self.ConvertToSql( call['record']['field_value'][0]['value']['expression']), cast_to) if call['predicate_name'] == 'FlagValue': if (len(arguments) != 1 or 'literal' not in call['record'] ['field_value'][0]['value']['expression'] or 'the_string' not in call['record']['field_value'][0] ['value']['expression']['literal']): raise self.exception_maker( 'FlagValue argument must be a string literal.') flag = (call['record']['field_value'][0]['value']['expression'] ['literal']['the_string']['the_string']) if flag not in self.flag_values: raise self.exception_maker('Unspecified flag: %s' % flag) return self.StrLiteral({'the_string': self.flag_values[flag]}) for ydg_f, sql_f in self.built_in_functions.items(): if call['predicate_name'] == ydg_f: if not sql_f: raise self.exception_maker( 'Function %s is not supported by %s dialect.' % (color.Warn(ydg_f), color.Warn( self.dialect.Name()))) if len(arguments) == 2 and ydg_f == '-': continue # '-' is the only operator with variable arity. arity_range = self.BuiltInFunctionArityRange(ydg_f) if not arity_range[0] <= len(arguments) <= arity_range[1]: raise self.exception_maker( color.Format( 'Built-in function {warning}{ydg_f}{end} takes {a} ' 'arguments, but {b} arguments were given.', dict(ydg_f=ydg_f, a=arity_range, b=len(arguments)))) return self.Function(sql_f, arguments) for udf, udf_sql in self.custom_udfs.items(): if call['predicate_name'] == udf: # TODO: Treatment of positional arguments should be # simplified everywhere. arguments = dict( (k, v) if isinstance(k, str) else ('col%d' % k, v) for k, v in arguments.items()) try: result = udf_sql.format(**arguments) except KeyError: raise self.exception_maker( 'Function %s call is inconsistent with its signature %s.' % (color.Warn(udf), udf_sql)) return result for ydg_op, sql_op in self.built_in_infix_operators.items(): if call['predicate_name'] == ydg_op: result = self.Infix(sql_op, arguments) result = '(' + result + ')' return result if 'subscript' in expression: sub = expression['subscript'] subscript = sub['subscript']['literal']['the_symbol']['symbol'] # TODO: Record literal and record of subscript should have # different keys. # Try to opimize and return the field from a record. if 'record' in sub['record']: for f_v in sub['record']['record']['field_value']: if f_v['field'] == subscript: # Optimizing and returning the field directly. return self.ConvertToSql(f_v['value']['expression']) # Trying to optmize subscript of implication. if 'implication' in sub['record']: simplified_sub = self.SubIfStruct(sub['record']['implication'], subscript) if simplified_sub: return simplified_sub # Couldn't optimize, just return the '.' expression. record = self.ConvertToSql(sub['record']) return self.Subscript(record, subscript) if 'record' in expression: record = expression['record'] return self.Record(record) if 'combine' in expression: return '(%s)' % (self.subquery_translator.TranslateRule( expression['combine'], self.vocabulary, is_combine=True)) if 'implication' in expression: implication = expression['implication'] return self.Implication(implication) if 'call' in expression and 'predicate_name' in expression['call']: raise self.exception_maker( color.Format( 'Unsupported supposedly built-in function: ' '{warning}{predicate}{end}.', dict(predicate=expression['call']['predicate_name']))) assert False, ( 'Logica bug: expression %s failed to compile for unknown reason.' % str(expression))
def ShowMessage(self, stream=sys.stderr): print(color.Format('{underline}Compiling{end}:'), file=stream) print(self.rule_str, file=stream) print(color.Format('\n[ {error}Error{end} ] ') + str(self), file=stream)
def AsSql(self, subquery_encoder=None, flag_values=None): """Outputing SQL representing this structure.""" # pylint: disable=g-long-lambda ql = expr_translate.QL( self.VarsVocabulary(), subquery_encoder, lambda message: RuleCompileException(message, self.full_rule_text), flag_values, custom_udfs=subquery_encoder.execution.custom_udfs, dialect=subquery_encoder.execution.dialect) r = 'SELECT\n' fields = [] if not self.select: raise RuleCompileException( color.Format( 'Tables with {warning}no columns{end} are not allowed in ' 'StandardSQL, so they are not allowed in Logica.'), self.full_rule_text) for k, v in self.select.items(): if k == '*': fields.append('%s.*' % ql.ConvertToSql(v)) else: fields.append('%s AS %s' % (ql.ConvertToSql(v), LogicaFieldToSqlField(k))) r += ',\n'.join(' ' + f for f in fields) if (self.tables or self.unnestings or self.constraints or self.distinct_denoted): r += '\nFROM\n' tables = [] for k, v in self.tables.items(): if subquery_encoder: # Note that we are passing external_vocabulary, not VarsVocabulary # here. I.e. if this is a sub-query then variables of outer tables # can be used. sql = subquery_encoder.TranslateTable( v, self.external_vocabulary) if not sql: raise RuleCompileException( color.Format( 'Rule uses table {warning}{table}{end}, which is not ' 'defined. External tables can not be used in ' '{warning}\'testrun\'{end} mode. This error may come ' 'from injected sub-rules.', dict(table=v)), self.full_rule_text) if sql != k: tables.append(sql + ' AS ' + k) else: tables.append(sql) self.SortUnnestings() for element, the_list in self.unnestings: tables.append( 'UNNEST(%s) as %s' % (ql.ConvertToSql(the_list), ql.ConvertToSql(element))) if not tables: tables.append('UNNEST(ARRAY[\'UNUSED\']) as unused_unnest') from_str = ', '.join(tables) # Indent the from_str. from_str = '\n'.join(' ' + l for l in from_str.split('\n')) r += from_str if self.constraints: r += '\nWHERE\n' constraints = [] for c in self.constraints: constraints.append(ql.ConvertToSql(c)) r += ' AND\n'.join(map(Indent2, constraints)) if self.distinct_vars: ordered_distinct_vars = [ v for v in self.select.keys() if v in self.distinct_vars ] r += '\nGROUP BY ' r += ', '.join( map(LogicaFieldToSqlField, ordered_distinct_vars)) return r
def FunctionSql(self, name, allocator=None, internal_mode=False): """Print formatted SQL function creation statement.""" # TODO: Refactor this into FunctionSqlInternal and FunctionSql. if not allocator: allocator = self.NewNamesAllocator() rules = list(self.GetPredicateRules(name)) # Check that the predicate is defined via a single rule. if not rules: raise rule_translate.RuleCompileException( color.Format( 'No rules are defining {warning}{name}{end}, but compilation ' 'was requested.', dict(name=name)), r' ¯\_(ツ)_/¯') elif len(rules) > 1: raise rule_translate.RuleCompileException( color.Format( 'Predicate {warning}{name}{end} is defined by more than 1 rule ' 'and can not be compiled into a function.', dict(name=name)), '\n\n'.join(r['full_text'] for r in rules)) [rule] = rules # Extract structure and assert that it is isomorphic to a function. s = rule_translate.ExtractRuleStructure(rule, external_vocabulary=None, names_allocator=allocator) udf_variables = [ v if isinstance(v, str) else 'col%d' % v for v in s.select if v != 'logica_value' ] s.select = self.TurnPositionalIntoNamed(s.select) variables = [v for v in s.select if v != 'logica_value'] if 0 in variables: raise rule_translate.RuleCompileException( color.Format( 'Predicate {warning}{name}{end} must have all aruments named for ' 'compilation as a function.', dict(name=name)), rule['full_text']) for v in variables: if ('variable' not in s.select[v] or s.select[v]['variable']['var_name'] != v): raise rule_translate.RuleCompileException( color.Format( 'Predicate {warning}{name}{end} must not rename arguments ' 'for compilation as a function.', dict(name=name)), rule['full_text']) vocabulary = {v: v for v in variables} s.external_vocabulary = vocabulary self.RunInjections(s, allocator) s.ElliminateInternalVariables(assert_full_ellimination=True) s.UnificationsToConstraints() sql = s.AsSql(subquery_encoder=self.MakeSubqueryTranslator(allocator)) if s.constraints or s.unnestings or s.tables: raise rule_translate.RuleCompileException( color.Format( 'Predicate {warning}{name}{end} is not a simple function, but ' 'compilation as function was requested. Full SQL:\n{sql}', dict(name=name, sql=sql)), rule['full_text']) if 'logica_value' not in s.select: raise rule_translate.RuleCompileException( color.Format( 'Predicate {warning}{name}{end} does not have a value, but ' 'compilation as function was requested. Full SQL:\n%s' % sql), rule['full_text']) # pylint: disable=g-long-lambda # Compile the function! ql = expr_translate.QL( vocabulary, self.MakeSubqueryTranslator(allocator), lambda message: rule_translate.RuleCompileException( message, rule['full_text']), self.flag_values, custom_udfs=self.custom_udfs, dialect=self.execution.dialect) value_sql = ql.ConvertToSql(s.select['logica_value']) sql = 'CREATE TEMP FUNCTION {name}({signature}) AS ({value})'.format( name=name, signature=', '.join('%s ANY TYPE' % v for v in variables), value=value_sql) sql = FormatSql(sql) if internal_mode: return ('%s(%s)' % (name, ', '.join('{%s}' % v for v in udf_variables)), sql) return sql
def Error(msg): print(color.Format('[ {error}Error{end} ] {msg}', {'msg': msg})) sys.exit(1)
def main(argv): if len(argv) <= 1 or argv[1] == 'help': print('Usage:') print(' logica <l file> <command> <predicate name> [flags]') print(' Commands are:') print(' print: prints the StandardSQL query for the predicate.') print( ' run: runs the StandardSQL query on BigQuery with pretty output.' ) print(' run_to_csv: runs the query on BigQuery with csv output.') print('') print('') print('Example:') print(' python3 logica.py - run GoodIdea <<<\' ' 'GoodIdea(snack: "carrots")\'') return 1 if len(argv) == 3 and argv[2] == 'parse': pass # compile needs just 2 actual arguments. else: if len(argv) < 4: print('Not enough arguments. Run \'logica help\' for help.', file=sys.stderr) return 1 if argv[1] == '-': filename = '/dev/stdin' else: filename = argv[1] command = argv[2] commands = ['parse', 'print', 'run', 'run_to_csv'] if command not in commands: print( color.Format( 'Unknown command {warning}{command}{end}. ' 'Available commands: {commands}.', dict(command=command, commands=', '.join(commands)))) return 1 if not os.path.exists(filename): print('File not found: %s' % filename, file=sys.stderr) return 1 program_text = open(filename).read() try: parsed_rules = parse.ParseFile(program_text, import_root=GetImportRoot())['rule'] except parse.ParsingException as parsing_exception: parsing_exception.ShowMessage() sys.exit(1) if command == 'parse': # No indentation to avoid file size inflation. print(json.dumps(parsed_rules, sort_keys=True, indent='')) return 0 predicates = argv[3] user_flags = ReadUserFlags(parsed_rules, argv[4:]) predicates_list = predicates.split(',') for predicate in predicates_list: try: logic_program = universe.LogicaProgram(parsed_rules, user_flags=user_flags) formatted_sql = logic_program.FormattedPredicateSql(predicate) preamble = logic_program.execution.preamble defines_and_exports = logic_program.execution.defines_and_exports main_predicate_sql = logic_program.execution.main_predicate_sql except rule_translate.RuleCompileException as rule_compilation_exception: rule_compilation_exception.ShowMessage() sys.exit(1) except functors.FunctorError as functor_exception: functor_exception.ShowMessage() sys.exit(1) if command == 'print': print(formatted_sql) engine = logic_program.annotations.Engine() if command == 'run' or command == 'run_to_csv': # We should split and move this logic to dialects. if engine == 'bigquery': output_format = 'csv' if command == 'run_to_csv' else 'pretty' p = subprocess.Popen([ 'bq', 'query', '--use_legacy_sql=false', '--format=%s' % output_format ], stdin=subprocess.PIPE, stdout=subprocess.PIPE) o, _ = p.communicate(formatted_sql.encode()) elif engine == 'sqlite': # TODO: Make multi-statement scripts work. format = ('artistictable' if command == 'run' else 'csv') statements_to_execute = ([preamble] + defines_and_exports + [main_predicate_sql]) o = sqlite3_logica.RunSqlScript(statements_to_execute, format).encode() elif engine == 'psql': p = subprocess.Popen( ['psql', '--quiet'] + (['--csv'] if command == 'run_to_csv' else []), stdin=subprocess.PIPE, stdout=subprocess.PIPE) commands = [] o, _ = p.communicate('\n'.join(commands + [formatted_sql]).encode()) elif engine == 'trino': a = logic_program.annotations.annotations['@Engine']['trino'] params = GetTrinoParameters(a) p = subprocess.Popen( ['trino'] + params + (['--output-format=CSV_HEADER_UNQUOTED'] if command == 'run_to_csv' else ['--output-format=ALIGNED']), stdin=subprocess.PIPE, stdout=subprocess.PIPE) o, _ = p.communicate(formatted_sql.encode()) elif engine == 'presto': a = logic_program.annotations.annotations['@Engine']['presto'] catalog = a.get('catalog', 'memory') server = a.get('server', 'localhost:8080') p = subprocess.Popen([ 'presto', '--catalog=%s' % catalog, '--server=%s' % server, '--file=/dev/stdin' ] + (['--output-format=CSV_HEADER_UNQUOTED'] if command == 'run_to_csv' else ['--output-format=ALIGNED']), stdin=subprocess.PIPE, stdout=subprocess.PIPE) o, _ = p.communicate(formatted_sql.encode()) else: assert False, 'Unknown engine: %s' % engine print(o.decode())
def ShowMessage(self): print(color.Format('{underline}Making{end}:'), file=sys.stderr) print(self.functor_name, file=sys.stderr) print(color.Format('\n[ {error}Error{end} ] ') + self.message, file=sys.stderr)
def main(argv): if len(argv) <= 1 or argv[1] == 'help': print('Usage:') print(' logica <l file> <command> <predicate name> [flags]') print(' Commands are:') print(' print: prints the StandardSQL query for the predicate.') print(' run: runs the StandardSQL query on BigQuery with pretty output.') print(' run_to_csv: runs the query on BigQuery with csv output.') print('') print('') print('Example:') print(' python3 logica.py - run GoodIdea <<<\' ' 'GoodIdea(snack: "carrots")\'') return 1 if len(argv) == 3 and argv[2] == 'parse': pass # compile needs just 2 actual arguments. else: if len(argv) < 4: print('Not enought arguments. Run \'logica help\' for help.', file=sys.stderr) return 1 if argv[1] == '-': filename = '/dev/stdin' else: filename = argv[1] command = argv[2] commands = ['parse', 'print', 'run', 'run_to_csv'] if command not in commands: print(color.Format('Unknown command {warning}{command}{end}. ' 'Available commands: {commands}.', dict(command=command, commands=', '.join(commands)))) return 1 if not os.path.exists(filename): print('File not found: %s' % filename, file=sys.stderr) return 1 program_text = open(filename).read() try: parsed_rules = parse.ParseFile(program_text)['rule'] except parse.ParsingException as parsing_exception: parsing_exception.ShowMessage() sys.exit(1) if command == 'parse': # No indentation to avoid file size inflation. print(json.dumps(parsed_rules, sort_keys=True, indent='')) return 0 predicates = argv[3] user_flags = ReadUserFlags(parsed_rules, argv[4:]) predicates_list = predicates.split(',') for predicate in predicates_list: try: p = universe.LogicaProgram(parsed_rules, user_flags=user_flags) formatted_sql = p.FormattedPredicateSql(predicate) except rule_translate.RuleCompileException as rule_compilation_exception: rule_compilation_exception.ShowMessage() sys.exit(1) except functors.FunctorError as functor_exception: functor_exception.ShowMessage() sys.exit(1) if command == 'print': print(formatted_sql) engine = p.annotations.Engine() if command == 'run' or command == 'run_to_csv': if engine == 'bigquery': output_format = 'csv' if command == 'run_to_csv' else 'pretty' p = subprocess.Popen(['bq', 'query', '--use_legacy_sql=false', '--format=%s' % output_format], stdin=subprocess.PIPE, stdout=subprocess.PIPE) o, _ = p.communicate(formatted_sql.encode()) elif engine == 'sqlite': p = subprocess.Popen(['sqlite3'], stdin=subprocess.PIPE, stdout=subprocess.PIPE) commands = [] if command == 'run_to_csv': commands.append('.mode csv') o, _ = p.communicate( '\n'.join(commands + [formatted_sql]).encode()) elif engine == 'psql': p = subprocess.Popen(['psql', '--quiet'] + (['--csv'] if command == 'run_to_csv' else []), stdin=subprocess.PIPE, stdout=subprocess.PIPE) commands = [] o, _ = p.communicate( '\n'.join(commands + [formatted_sql]).encode()) else: assert False, 'Unknown engine: %s' % engine print(o.decode())
def RunInjections(self, s, allocator): iterations = 0 while True: iterations += 1 if iterations > sys.getrecursionlimit(): raise rule_translate.RuleCompileException( RecursionError(), s.full_rule_text) new_tables = collections.OrderedDict() for table_name_rsql, table_predicate_rsql in s.tables.items(): rules = list(self.GetPredicateRules(table_predicate_rsql)) if (len(rules) == 1 and ('distinct_denoted' not in rules[0]) and self.annotations.OkInjection(table_predicate_rsql)): [r] = rules rs = rule_translate.ExtractRuleStructure( r, allocator, None) rs.ElliminateInternalVariables( assert_full_ellimination=False) new_tables.update(rs.tables) InjectStructure(s, rs) new_vars_map = {} new_inv_vars_map = {} for (table_name, table_var), clause_var in s.vars_map.items(): if table_name != table_name_rsql: new_vars_map[table_name, table_var] = clause_var new_inv_vars_map[clause_var] = (table_name, table_var) else: if table_var not in rs.select: if '*' in rs.select: subscript = { 'literal': { 'the_symbol': { 'symbol': table_var } } } s.vars_unification.append({ 'left': { 'variable': { 'var_name': clause_var } }, 'right': { 'subscript': { 'subscript': subscript, 'record': rs.select['*'] } } }) else: extra_hint = '' if table_var != '*' else ( ' Are you using ..<rest of> for injectible predicate? ' 'Please list the fields that you extract explicitly. ' 'Tracking bug: b/131759583.') raise rule_translate.RuleCompileException( color.Format( 'Predicate {warning}{table_predicate_rsql}{end} ' 'does not have an argument ' '{warning}{table_var}{end}, but ' 'this rule tries to access it. {extra_hint}', dict(table_predicate_rsql= table_predicate_rsql, table_var=table_var, extra_hint=extra_hint)), s.full_rule_text) else: s.vars_unification.append({ 'left': { 'variable': { 'var_name': clause_var } }, 'right': rs.select[table_var] }) s.vars_map = new_vars_map s.inv_vars_map = new_inv_vars_map else: new_tables[table_name_rsql] = table_predicate_rsql if s.tables == new_tables: break s.tables = new_tables
def ElliminateInternalVariables(self, assert_full_ellimination=False): """Elliminates internal variables via substitution.""" variables = self.InternalVariables() while True: done = True for u in self.vars_unification: for k, r in [['left', 'right'], ['right', 'left']]: if u[k] == u[r]: continue ur_variables = AllMentionedVariables(u[r]) ur_variables_incl_combines = AllMentionedVariables( u[r], dive_in_combines=True) if (isinstance(u[k], dict) and 'variable' in u[k] and u[k]['variable']['var_name'] in variables and u[k]['variable']['var_name'] not in ur_variables_incl_combines and (ur_variables <= self.ExtractedVariables() or not str( u[k]['variable']['var_name']).startswith('x_'))): u_left = u[k]['variable']['var_name'] u_right = u[r] if 'variable' in u_right: l = self.synonym_log.get( u_right['variable']['var_name'], []) l.append(u_left) l.extend(self.synonym_log.get(u_left, [])) self.synonym_log[u_right['variable'] ['var_name']] = l ReplaceVariable(u_left, u_right, self.unnestings) ReplaceVariable(u_left, u_right, self.select) ReplaceVariable(u_left, u_right, self.vars_unification) ReplaceVariable(u_left, u_right, self.constraints) done = False if done: variables = self.InternalVariables() if assert_full_ellimination: if True: if variables: violators = [] for v in variables: violators.extend(self.synonym_log.get(v, [])) violators.append(v) violators = { v for v in violators if not v.startswith('x_') } assert violators, ( 'Logica needs better error messages: purely internal ' 'variable was not eliminated. It looks like you have ' 'not passed a required argument to some called predicate. ' 'Use --add_debug_info_to_var_names flag to make this message ' 'a little more informatvie. ' 'Variables: %s, synonym_log: %s' % (str(variables), str(self.synonym_log))) # Remove disambiguation suffixes from variables not to confuse # the user. violators = { v.split(' # disambiguated')[0] for v in violators } raise RuleCompileException( color.Format( 'Found no way to assign variables: ' '{warning}{violators}{end}. ' 'This error might also come from injected sub-rules.', dict(violators=', '.join(sorted( violators)))), self.full_rule_text) else: assert not variables, ( 'Not all internal variables were eliminated. Violators:\n' + ',\n'.join('%s (aka %s)' % (v, self.synonym_log[v]) for v in variables) + '\nRule: %s' % self) else: unassigned_variables = [] for v in variables: if not v.startswith('x_'): unassigned_variables.append(v) # Remove disambiguation suffixes from variables not to confuse # the user. unassigned_variables = { v.split(' # disambiguated')[0] for v in unassigned_variables } if unassigned_variables: raise RuleCompileException( color.Format( 'Found no way to assign variables: ' '{warning}{violators}{end}. ' 'This error might also come from injected sub-rules.', dict(violators=', '.join( sorted(unassigned_variables)))), self.full_rule_text) break