def PredicateSql(self, name, allocator=None, external_vocabulary=None): """Producing SQL for a predicate.""" # Load proto if necessary. rules = list(self.GetPredicateRules(name)) if len(rules) == 1: [rule] = rules return (self.SingleRuleSql(rule, allocator, external_vocabulary) + self.annotations.OrderByClause(name) + self.annotations.LimitClause(name)) elif len(rules) > 1: rules_sql = [] for rule in rules: if 'distinct_denoted' in rule: raise rule_translate.RuleCompileException( color.Format( 'For distinct denoted predicates multiple rules are not ' 'currently supported. Consider taking ' '{warning}union of bodies manually{end}, if that was what ' 'you intended.'), rule['full_text']) rules_sql.append('\n%s\n' % Indent2( self.SingleRuleSql(rule, allocator, external_vocabulary))) rules_sql = [ '\n'.join(' ' + l for l in r.split('\n')) for r in rules_sql ] return 'SELECT * FROM (\n%s\n) AS UNUSED_TABLE_NAME %s %s' % ( ' UNION ALL\n'.join(rules_sql), self.annotations.OrderByClause(name), self.annotations.LimitClause(name)) else: raise rule_translate.RuleCompileException( color.Format( 'No rules are defining {warning}{name}{end}, but compilation ' 'was requested.', dict(name=name)), r' ¯\_(ツ)_/¯')
def SingleRuleSql(self, rule, allocator=None, external_vocabulary=None, is_combine=False): """Producing SQL for a given rule in the program.""" allocator = allocator or self.NewNamesAllocator() r = rule if (is_combine): r = self.execution.dialect.DecorateCombineRule( r, allocator.AllocateVar()) s = rule_translate.ExtractRuleStructure(r, allocator, external_vocabulary) s.ElliminateInternalVariables(assert_full_ellimination=False) self.RunInjections(s, allocator) s.ElliminateInternalVariables(assert_full_ellimination=True) s.UnificationsToConstraints() try: sql = s.AsSql(self.MakeSubqueryTranslator(allocator), self.flag_values) except RuntimeError as runtime_error: if (str(runtime_error).startswith('maximum recursion')): raise rule_translate.RuleCompileException( RecursionError(), s.full_rule_text) else: raise runtime_error if 'nil' in s.tables.values(): # Mark rule for deletion. sql = '/* nil */' + sql return sql
def __contains__(self, key): if rule_predicate == '@Make': ThrowException() return raise rule_translate.RuleCompileException( 'Annotation may not use variables, but this one uses ' 'variable %s.' % (color.Warn(key)), rule_text)
def ThrowException(*args, **xargs): _ = args _ = xargs if rule_predicate == '@Make': # pylint: disable=raising-format-tuple raise rule_translate.RuleCompileException( 'Incorrect syntax for functor call. ' 'Functor call to be made as\n' ' R := F(A: V, ...)\n' 'or\n' ' @Make(R, F, {A: V, ...})\n' 'Where R, F, A\'s and V\'s are all ' 'predicate names.', rule_text) else: raise rule_translate.RuleCompileException( 'Can not understand annotation.', rule_text)
def CompileAsUdf(self, predicate_name): result = predicate_name in self.annotations['@CompileAsUdf'] if result and self.TvfSignature(predicate_name): raise rule_translate.RuleCompileException( 'A predicate can not be UDF and TVF at the ' 'same time %s.' % predicate_name, 'Predicate: ' + predicate_name) return result
def __init__(self, rules, table_aliases=None, user_flags=None): """Initializes the program. Args: rules: A list of dictionary representations of parsed Logica rules. table_aliases: A map from an undefined Logica predicate name to a BigQuery table name. This table will be used in place of predicate. user_flags: Dictionary of user specified flags. """ rules = self.UnfoldRecursion(rules) # TODO: Should allocator be a member of Logica? self.preparsed_rules = rules self.rules = [] self.defined_predicates = set() self.dollar_params = list(self.ExtractDollarParams(rules)) self.table_aliases = table_aliases or {} self.execution = None self.user_flags = user_flags or {} self.annotations = Annotations(rules, self.user_flags) self.flag_values = self.annotations.flag_values # Dictionary custom_udfs maps function name to a format string to use # in queries. self.custom_udfs = collections.OrderedDict() # Dictionary custom_udf_definitions maps function name to SQL defining the # function. self.custom_udf_definitions = collections.OrderedDict() if not set(self.dollar_params) <= set(self.flag_values): raise rule_translate.RuleCompileException( 'Parameters %s are undefined.' % (list(set(self.dollar_params) - set(self.flag_values))), str(list(set(self.dollar_params) - set(self.flag_values)))) self.functors = None # Extending rules with functors. extended_rules = self.RunMakes(rules) # Populates self.functors. # Extending rules with the library of the dialect. library_rules = parse.ParseFile( dialects.Get(self.annotations.Engine()).LibraryProgram())['rule'] extended_rules.extend(library_rules) for rule in extended_rules: predicate_name = rule['head']['predicate_name'] self.defined_predicates.add(predicate_name) self.rules.append((predicate_name, rule)) # We need to recompute annotations, because 'Make' created more rules and # annotations. self.annotations = Annotations(extended_rules, self.user_flags) # Build udfs, populating custom_udfs and custom_udf_definitions. self.BuildUdfs() # Function compilation may have added irrelevant defines: self.execution = None if False: self.RunTypechecker()
def ExtractSingleton(self, annotation_name, default_value): if not self.annotations[annotation_name]: return default_value results = list(self.annotations[annotation_name].keys()) if len(results) > 1: raise rule_translate.RuleCompileException( 'Single %s must be provided. Provided: %s' % (annotation_name, results), self.annotations[annotation_name][results[0]]['__rule_text']) return results[0]
def LimitOf(self, predicate_name): """Limit of the query corresponding to the predicate as per annotation.""" if predicate_name not in self.annotations['@Limit']: return None annotation = FieldValuesAsList( self.annotations['@Limit'][predicate_name]) if (len(annotation) != 1 or not isinstance(annotation[0], int)): raise rule_translate.RuleCompileException( 'Bad limit specification for predicate %s.' % predicate_name, 'Predicate: ' + predicate_name) return annotation[0]
def UseFlagsAsParameters(self, sql): """Running flag substitution in a loop to the fixed point.""" # We do it in a loop to deal with flags that refer to other flags. prev_sql = '' num_subs = 0 while sql != prev_sql: num_subs += 1 prev_sql = sql if num_subs > 100: raise rule_translate.RuleCompileException( 'You seem to have recursive flags. It is disallowed.', 'Flags:\n' + '\n'.join('--{0}={1}'.format(*i) for i in self.flag_values.items())) # Do the substitution! for flag, value in self.flag_values.items(): sql = sql.replace('${%s}' % flag, value) return sql
def With(self, predicate_name): """Return whether this predicate should be compiled to a WITH-table. This only applies if the predicate is not inlined earlier in the flow. """ is_with = self.ForceWith(predicate_name) is_nowith = self.ForceNoWith(predicate_name) if is_with and is_nowith: raise rule_translate.RuleCompileException( color.Format('Predicate is annotated both with @With and @NoWith.'), 'Predicate: %s' % predicate_name) if is_with: return True if is_nowith or self.Ground(predicate_name): return False # TODO: return false for predicates that will be injected. return True
def BuildFlagValues(self): """Building values by overriding defaults with user flags.""" default_values = {} for flag, a in self.annotations['@DefineFlag'].items(): default_values[flag] = a.get('1', '${%s}' % flag) programmatic_flag_values = {} for flag, a in self.annotations['@ResetFlagValue'].items(): programmatic_flag_values[flag] = a.get('1', '${%s}' % flag) if not set(self.user_flags) <= set(default_values): raise rule_translate.RuleCompileException( 'Undefined flags used: %s' % list(set(self.user_flags) - set(default_values)), str(set(self.user_flags) - set(default_values))) flag_values = default_values flag_values.update(**programmatic_flag_values) flag_values.update(**self.user_flags) return flag_values
def RunInjections(self, s, allocator): iterations = 0 while True: iterations += 1 if iterations > sys.getrecursionlimit(): raise rule_translate.RuleCompileException( RecursionError(), s.full_rule_text) new_tables = collections.OrderedDict() for table_name_rsql, table_predicate_rsql in s.tables.items(): rules = list(self.GetPredicateRules(table_predicate_rsql)) if (len(rules) == 1 and ('distinct_denoted' not in rules[0]) and self.annotations.OkInjection(table_predicate_rsql)): [r] = rules rs = rule_translate.ExtractRuleStructure( r, allocator, None) rs.ElliminateInternalVariables( assert_full_ellimination=False) new_tables.update(rs.tables) InjectStructure(s, rs) new_vars_map = {} new_inv_vars_map = {} for (table_name, table_var), clause_var in s.vars_map.items(): if table_name != table_name_rsql: new_vars_map[table_name, table_var] = clause_var new_inv_vars_map[clause_var] = (table_name, table_var) else: if table_var not in rs.select: if '*' in rs.select: subscript = { 'literal': { 'the_symbol': { 'symbol': table_var } } } s.vars_unification.append({ 'left': { 'variable': { 'var_name': clause_var } }, 'right': { 'subscript': { 'subscript': subscript, 'record': rs.select['*'] } } }) else: extra_hint = '' if table_var != '*' else ( ' Are you using ..<rest of> for injectible predicate? ' 'Please list the fields that you extract explicitly. ' 'Tracking bug: b/131759583.') raise rule_translate.RuleCompileException( color.Format( 'Predicate {warning}{table_predicate_rsql}{end} ' 'does not have an argument ' '{warning}{table_var}{end}, but ' 'this rule tries to access it. {extra_hint}', dict(table_predicate_rsql= table_predicate_rsql, table_var=table_var, extra_hint=extra_hint)), s.full_rule_text) else: s.vars_unification.append({ 'left': { 'variable': { 'var_name': clause_var } }, 'right': rs.select[table_var] }) s.vars_map = new_vars_map s.inv_vars_map = new_inv_vars_map else: new_tables[table_name_rsql] = table_predicate_rsql if s.tables == new_tables: break s.tables = new_tables
def FunctionSql(self, name, allocator=None, internal_mode=False): """Print formatted SQL function creation statement.""" # TODO: Refactor this into FunctionSqlInternal and FunctionSql. if not allocator: allocator = self.NewNamesAllocator() rules = list(self.GetPredicateRules(name)) # Check that the predicate is defined via a single rule. if not rules: raise rule_translate.RuleCompileException( color.Format( 'No rules are defining {warning}{name}{end}, but compilation ' 'was requested.', dict(name=name)), r' ¯\_(ツ)_/¯') elif len(rules) > 1: raise rule_translate.RuleCompileException( color.Format( 'Predicate {warning}{name}{end} is defined by more than 1 rule ' 'and can not be compiled into a function.', dict(name=name)), '\n\n'.join(r['full_text'] for r in rules)) [rule] = rules # Extract structure and assert that it is isomorphic to a function. s = rule_translate.ExtractRuleStructure(rule, external_vocabulary=None, names_allocator=allocator) udf_variables = [ v if isinstance(v, str) else 'col%d' % v for v in s.select if v != 'logica_value' ] s.select = self.TurnPositionalIntoNamed(s.select) variables = [v for v in s.select if v != 'logica_value'] if 0 in variables: raise rule_translate.RuleCompileException( color.Format( 'Predicate {warning}{name}{end} must have all aruments named for ' 'compilation as a function.', dict(name=name)), rule['full_text']) for v in variables: if ('variable' not in s.select[v] or s.select[v]['variable']['var_name'] != v): raise rule_translate.RuleCompileException( color.Format( 'Predicate {warning}{name}{end} must not rename arguments ' 'for compilation as a function.', dict(name=name)), rule['full_text']) vocabulary = {v: v for v in variables} s.external_vocabulary = vocabulary self.RunInjections(s, allocator) s.ElliminateInternalVariables(assert_full_ellimination=True) s.UnificationsToConstraints() sql = s.AsSql(subquery_encoder=self.MakeSubqueryTranslator(allocator)) if s.constraints or s.unnestings or s.tables: raise rule_translate.RuleCompileException( color.Format( 'Predicate {warning}{name}{end} is not a simple function, but ' 'compilation as function was requested. Full SQL:\n{sql}', dict(name=name, sql=sql)), rule['full_text']) if 'logica_value' not in s.select: raise rule_translate.RuleCompileException( color.Format( 'Predicate {warning}{name}{end} does not have a value, but ' 'compilation as function was requested. Full SQL:\n%s' % sql), rule['full_text']) # pylint: disable=g-long-lambda # Compile the function! ql = expr_translate.QL( vocabulary, self.MakeSubqueryTranslator(allocator), lambda message: rule_translate.RuleCompileException( message, rule['full_text']), self.flag_values, custom_udfs=self.custom_udfs, dialect=self.execution.dialect) value_sql = ql.ConvertToSql(s.select['logica_value']) sql = 'CREATE TEMP FUNCTION {name}({signature}) AS ({value})'.format( name=name, signature=', '.join('%s ANY TYPE' % v for v in variables), value=value_sql) sql = FormatSql(sql) if internal_mode: return ('%s(%s)' % (name, ', '.join('{%s}' % v for v in udf_variables)), sql) return sql
def ExtractAnnotations(cls, rules, restrict_to=None, flag_values=None): """Extracting annotations from the rules.""" result = { p: collections.OrderedDict() for p in cls.ANNOTATING_PREDICATES } for rule in rules: rule_predicate = rule['head']['predicate_name'] if restrict_to and rule_predicate not in restrict_to: continue if (rule_predicate[0] == '@' and rule_predicate not in cls.ANNOTATING_PREDICATES): raise rule_translate.RuleCompileException( 'Only {0} and {1} special predicates are allowed.'.format( ', '.join(cls.ANNOTATING_PREDICATES[:-1]), cls.ANNOTATING_PREDICATES[-1]), rule['full_text']) if rule_predicate in cls.ANNOTATING_PREDICATES: rule_text = rule['full_text'] # pylint: disable=cell-var-from-loop def ThrowException(*args, **xargs): _ = args _ = xargs if rule_predicate == '@Make': # pylint: disable=raising-format-tuple raise rule_translate.RuleCompileException( 'Incorrect syntax for functor call. ' 'Functor call to be made as\n' ' R := F(A: V, ...)\n' 'or\n' ' @Make(R, F, {A: V, ...})\n' 'Where R, F, A\'s and V\'s are all ' 'predicate names.', rule_text) else: raise rule_translate.RuleCompileException( 'Can not understand annotation.', rule_text) class Thrower(object): def __contains__(self, key): if rule_predicate == '@Make': ThrowException() return raise rule_translate.RuleCompileException( 'Annotation may not use variables, but this one uses ' 'variable %s.' % (color.Warn(key)), rule_text) flag_values = flag_values or Thrower() ql = expr_translate.QL(Thrower(), ThrowException, ThrowException, flag_values) ql.convert_to_json = True annotation = rule['head']['predicate_name'] field_values_json_str = ql.ConvertToSql( {'record': rule['head']['record']}) try: field_values = json.loads(field_values_json_str) except: raise rule_translate.RuleCompileException( 'Could not understand arguments of annotation.', rule['full_text']) if ('0' in field_values and isinstance(field_values['0'], dict) and 'predicate_name' in field_values['0']): subject = field_values['0']['predicate_name'] else: subject = field_values['0'] del field_values['0'] if rule_predicate in ['@OrderBy', '@Limit', '@NoInject']: field_values_list = FieldValuesAsList(field_values) if field_values_list is None: raise rule_translate.RuleCompileException( '@OrderBy and @Limit may only have positional ' 'arguments.', rule['full_text']) if rule_predicate == '@Limit' and len( field_values_list) != 1: raise rule_translate.RuleCompileException( 'Annotation @Limit must have exactly two arguments: ' 'predicate and limit.', rule['full_text']) updated_annotation = result.get(annotation, {}) field_values['__rule_text'] = rule['full_text'] if subject in updated_annotation: raise rule_translate.RuleCompileException( color.Format( '{annotation} annotates {warning}{subject}{end} more ' 'than once: {before}, {after}', dict(annotation=annotation, subject=subject, before=updated_annotation[subject] ['__rule_text'], after=field_values['__rule_text'])), rule['full_text']) updated_annotation[subject] = field_values result[annotation] = updated_annotation return result
def AnnotationError(message, annotation_value): raise rule_translate.RuleCompileException(message, annotation_value['__rule_text'])
def RaiseCompilerError(message, context): raise rule_translate.RuleCompileException(message, context)