def testToBase64(self): """Test built-in function TO_BASE64(string).""" stack = [ util.StringLiteralToken('"hello test"'), util.BuiltInFunctionToken('to_base64') ] self.assertEqual(interpreter.Evaluate(stack), 'aGVsbG8gdGVzdA==')
def testString(self): stack = [ util.StringLiteralToken('"TESTING IS FUN."'), 4, util.BuiltInFunctionToken('left') ] self.assertEqual(interpreter.ToInfix(list(stack)), 'left("TESTING IS FUN.", 4)') self.assertEqual(interpreter.Evaluate(stack), 'TEST')
def testOneArgumentFunction(self): stack = [ 0, util.BuiltInFunctionToken('cos'), util.BuiltInFunctionToken('ln'), util.BuiltInFunctionToken('sqrt') ] self.assertEqual(interpreter.ToInfix(list(stack)), 'sqrt(ln(cos(0)))') self.assertEqual(interpreter.Evaluate(stack), 0)
def testUnary(self): stack = [ 1, 2, util.OperatorToken('~', 1), util.OperatorToken('<', 2), util.OperatorToken('not', 1) ] self.assertEqual(interpreter.ToInfix(list(stack)), 'not (1 < ~ 2)') self.assertEqual(interpreter.Evaluate(stack), True)
def testToBase64RawBytes(self): """Test built-in function TO_BASE64(arbitrary bytes).""" bytes_str = '\xb4\x00\xb0\x09\xcd\x10' bytes_str_b64 = 'tACwCc0Q' stack = [ util.StringLiteralToken('"%s"' % bytes_str), util.BuiltInFunctionToken('to_base64') ] self.assertEqual(interpreter.Evaluate(stack), bytes_str_b64)
def testToBase64Utf8(self): """Test built-in function TO_BASE64(utf8 string).""" unicode_str = u'M\u00fcnchen' unicode_str_b64 = 'TcO8bmNoZW4=' stack = [ util.StringLiteralToken('"%s"' % unicode_str.encode('utf-8')), util.BuiltInFunctionToken('to_base64') ] self.assertEqual(interpreter.Evaluate(stack), unicode_str_b64)
def testToBase64Ascii(self): """Test built-in function TO_BASE64(string).""" ascii_str = 'hello test' ascii_str_b64 = 'aGVsbG8gdGVzdA==' stack = [ util.StringLiteralToken('"%s"' % ascii_str), util.BuiltInFunctionToken('to_base64') ] self.assertEqual(interpreter.Evaluate(stack), ascii_str_b64)
def testFromBase64(self): """Don't implement FROM_BASE64() until BigQuery has BYTES type.""" base64_str = 'aGVsbG8gdGVzdA==' # "hello test" stack = [ util.StringLiteralToken('"%s"' % base64_str), util.BuiltInFunctionToken('from_base64') ] self.assertEqual(interpreter.Evaluate(stack), 'FROM_BASE64("%s")' % base64_str)
def testMultipleArgumentFunction(self): stack = [ 'True', 3, 0, util.BuiltInFunctionToken('if'), 2, 1, util.BuiltInFunctionToken('pow'), util.BuiltInFunctionToken('pow') ] self.assertEqual(interpreter.ToInfix(list(stack)), 'pow(if(True, 3, 0), pow(2, 1))') self.assertEqual(interpreter.Evaluate(stack), 9)
def _ComputeRows(new_postfix_stack, queried_values): """Substitutes queries back to expressions and evaluates them. Arguments: new_postfix_stack: All expressions for each column. queried_values: A dictionary that represents the queried values to a list of values that were received from server (all have been decrypted). Returns: A new table with results of each expression after query substitution. """ table_values = [] if queried_values: num_rows = len(queried_values[queried_values.keys()[0]]) else: for stack in new_postfix_stack: ans = interpreter.Evaluate(stack) if ans is None: ans = 'NULL' table_values.append(str(ans)) return [table_values] # Substitute queried values back into postfix stacks and evaluate them. for i in range(num_rows): row_values = [] for j in range(len(new_postfix_stack)): temp_stack = list(new_postfix_stack[j]) for k in xrange(len(temp_stack)): if (isinstance(temp_stack[k], util.AggregationQueryToken) or isinstance(temp_stack[k], util.UnencryptedQueryToken) or isinstance(temp_stack[k], util.FieldToken)): if str(temp_stack[k]) not in queried_values: raise bigquery_client.BigqueryInvalidQueryError( '%s column does not exist.' % temp_stack[k], None, None, None) temp_stack[k] = queried_values[str(temp_stack[k])][i] ans = interpreter.Evaluate(temp_stack) if ans is None: ans = 'NULL' row_values.append(str(ans)) table_values.append(row_values) return table_values
def testBinary(self): stack = [ 1, 2, util.OperatorToken('+', 2), 3, util.OperatorToken('*', 2), 9, util.OperatorToken('/', 2), 2, util.OperatorToken('-', 2) ] self.assertEqual(interpreter.ToInfix(list(stack)), '((((1 + 2) * 3) / 9) - 2)') self.assertEqual(interpreter.Evaluate(stack), -1)
def testBooleanLiterals(self): stack = [ util.LiteralToken('True', True), util.LiteralToken('False', False), util.OperatorToken('or', 2), 1, 2, util.OperatorToken('=', 2), util.OperatorToken('or', 2) ] self.assertEqual(interpreter.ToInfix(list(stack)), '((True or False) or (1 = 2))') self.assertEqual(interpreter.Evaluate(stack), True)
def _CollapseFunctions(stack): """Collapses functions by evaluating them for actual values. Replaces a function's postfix expression with a single token. If the function can be evaluated (no fields included as arguments), the single token is the value of function's evaluation. Otherwise, the function is collapsed into a single token without evaluation. Arguments: stack: The stack whose functions are to be collapsed and resolved. Raises: bigquery_client.BigqueryInvalidQueryError: If a field exists inside the arguments of a function. Returns: True iff a function is found and collapsed. In other words, another potential function can still exist. """ for i in xrange(len(stack)): if isinstance(stack[i], util.BuiltInFunctionToken): start_idx, postfix_expr = interpreter.GetSingleValue(stack[:i + 1]) if util.IsEncryptedExpression(postfix_expr): raise bigquery_client.BigqueryInvalidQueryError( 'Invalid aggregation function argument: Cannot put an encrypted ' 'field as an argument to a built-in function.', None, None, None) # If the expression has no fields, we want to get the actual value. # But, if the field has a field, we have to get the infix string instead. try: result = interpreter.Evaluate(list(postfix_expr)) if isinstance(result, basestring): result = util.StringLiteralToken('"%s"' % result) elif result is None: result = util.LiteralToken('NULL', None) elif str(result).lower() in ['true', 'false']: result = util.LiteralToken(str(result).lower(), result) stack[start_idx:i + 1] = [result] except bigquery_client.BigqueryInvalidQueryError: result = interpreter.ToInfix(list(postfix_expr)) stack[start_idx:i + 1] = [util.FieldToken(result)] return True return False
def testNull(self): stack = [util.LiteralToken('null', None)] self.assertEqual(interpreter.ToInfix(list(stack)), 'null') self.assertEqual(interpreter.Evaluate(stack), None)
def testSimpleExpression(self): stack = [1, 2, util.OperatorToken('+', 2)] self.assertEqual(interpreter.ToInfix(list(stack)), '(1 + 2)') self.assertEqual(interpreter.Evaluate(stack), 3)
def _ComputeRows(new_postfix_stack, queried_values, manifest=None): """Substitutes queries back to expressions and evaluates them. Args: new_postfix_stack: All expressions for each column. queried_values: A dictionary that represents the queried values to a list of values that were received from server (all have been decrypted). manifest: optional but recommended, query_lib.QueryManifest object. Returns: A new table with results of each expression after query substitution. Raises: BigqueryInvalidQueryError: When a query request has parsed the query response and has determined the response is invalid. """ table_values = [] num_rows = 0 if queried_values: # Try to obtain the number of rows (records) returned from the manifest. if manifest is not None: num_rows = manifest.statistics.get(manifest.RECORDS_WRITTEN, 0) # No manifest or no value obtained. Try to find the length of the FIRST # key:value pair where len(value) > 0. if num_rows <= 0: for k in queried_values: if len(queried_values[k]) > num_rows: num_rows = len(queried_values[k]) break else: for stack in new_postfix_stack: ans = interpreter.Evaluate(stack) if ans is None: ans = 'NULL' table_values.append(str(ans)) return [table_values] # No num_rows able to be found or calculated, or num_rows too few if num_rows <= 0: return [] # Substitute queried values back into postfix stacks and evaluate them. for i in xrange(num_rows): row_values = [] for j in xrange(len(new_postfix_stack)): temp_stack = list(new_postfix_stack[j]) for k in xrange(len(temp_stack)): if (isinstance(temp_stack[k], util.AggregationQueryToken) or isinstance(temp_stack[k], util.UnencryptedQueryToken) or isinstance(temp_stack[k], util.FieldToken)): k_use = None for k_try in [temp_stack[k].alias, temp_stack[k]]: if k_try and k_try in queried_values: k_use = k_try if not k_use: raise bigquery_client.BigqueryInvalidQueryError( 'Required %s column does not exist.' % temp_stack[k], None, None, None) temp_stack[k] = queried_values[k_use][i] ans = interpreter.Evaluate(temp_stack) if ans is None: ans = 'NULL' row_values.append(str(ans)) table_values.append(row_values) return table_values
def testNoArgumentFunction(self): stack = [util.BuiltInFunctionToken('pi')] self.assertEqual(interpreter.ToInfix(list(stack)), 'pi()') self.assertEqual(interpreter.Evaluate(stack), math.pi)