def _create_sql_parser(): sql_parser = CommandParser( prog="%%sql", formatter_class=argparse.RawDescriptionHelpFormatter, description=""" Create a named SQL module with one or more queries. The cell body should contain an optional initial part defining the default values for the variables, if any, using Python code, followed by one or more queries. Queries should start with 'DEFINE QUERY <name>' in order to bind them to <module name>.<query name> in the notebook (as datalab.data.SqlStament instances). The final query can optionally omit 'DEFINE QUERY <name>', as using the module name in places where a SqlStatement is expected will resolve to the final query in the module. Queries can refer to variables with '$<name>', as well as refer to other queries within the same module, making it easy to compose nested queries and test their parts. The Python code defining the variable default values can assign scalar or list/tuple values to variables, or one of the special functions 'datestring' and 'source'. When a variable with a 'datestring' default is expanded it will expand to a formatted string based on the current date, while a 'source' default will expand to a table whose name is based on the current date. datestring() takes two named arguments, 'format' and 'offset'. The former is a format string that is the same as for Python's time.strftime function. The latter is a string containing a comma-separated list of expressions such as -1y, +2m, etc; these are offsets from the time of expansion that are applied in order. The suffix (y, m, d, h, M) correspond to units of years, months, days, hours and minutes, while the +n or -n prefix is the number of units to add or subtract from the time of expansion. Three special values 'now', 'today' and 'yesterday' are also supported; 'today' and 'yesterday' will be midnight UTC on the current date or previous days date. source() can take a 'name' argument for a fixed table name, or 'format' and 'offset' arguments similar to datestring(), but unlike datestring() will resolve to a Table with the specified name. """) sql_parser.add_argument('-m', '--module', help='The name for this SQL module') sql_parser.add_argument('-d', '--dialect', help='BigQuery SQL dialect', choices=['legacy', 'standard'], default='legacy') sql_parser.add_argument('-b', '--billing', type=int, help='BigQuery billing tier') sql_parser.set_defaults(func=lambda args, cell: sql_cell(args, cell)) return sql_parser
def _create_sql_parser(): sql_parser = CommandParser(prog="%%sql", formatter_class=argparse.RawDescriptionHelpFormatter, description=""" Create a named SQL module with one or more queries. The cell body should contain an optional initial part defining the default values for the variables, if any, using Python code, followed by one or more queries. Queries should start with 'DEFINE QUERY <name>' in order to bind them to <module name>.<query name> in the notebook (as datalab.data.SqlStament instances). The final query can optionally omit 'DEFINE QUERY <name>', as using the module name in places where a SqlStatement is expected will resolve to the final query in the module. Queries can refer to variables with '$<name>', as well as refer to other queries within the same module, making it easy to compose nested queries and test their parts. The Python code defining the variable default values can assign scalar or list/tuple values to variables, or one of the special functions 'datestring' and 'source'. When a variable with a 'datestring' default is expanded it will expand to a formatted string based on the current date, while a 'source' default will expand to a table whose name is based on the current date. datestring() takes two named arguments, 'format' and 'offset'. The former is a format string that is the same as for Python's time.strftime function. The latter is a string containing a comma-separated list of expressions such as -1y, +2m, etc; these are offsets from the time of expansion that are applied in order. The suffix (y, m, d, h, M) correspond to units of years, months, days, hours and minutes, while the +n or -n prefix is the number of units to add or subtract from the time of expansion. Three special values 'now', 'today' and 'yesterday' are also supported; 'today' and 'yesterday' will be midnight UTC on the current date or previous days date. source() can take a 'name' argument for a fixed table name, or 'format' and 'offset' arguments similar to datestring(), but unlike datestring() will resolve to a Table with the specified name. """) sql_parser.add_argument('-m', '--module', help='The name for this SQL module') sql_parser.set_defaults(func=lambda args, cell: sql_cell(args, cell)) return sql_parser
def _arguments(code, module): """Define pipeline arguments. Args: code: the Python code to execute that defines the arguments. """ arg_parser = CommandParser.create('') try: # Define our special argument 'types' and add them to the environment. builtins = {'source': _table, 'datestring': _datestring} env = {} env.update(builtins) # Execute the cell which should be one or more calls to arg(). exec(code, env) # Iterate through the module dictionary. For any newly defined objects, # add args to the parser. for key in env: # Skip internal/private stuff. if key in builtins or key[0] == '_': continue # If we want to support importing query modules into other query modules, uncomment next 4 # Skip imports but add them to the module # if isinstance(env[key], types.ModuleType): # module.__dict__[key] = env[key] # continue val = env[key] key = '--%s' % key if isinstance(val, bool): if val: arg_parser.add_argument(key, default=val, action='store_true') else: arg_parser.add_argument(key, default=val, action='store_false') elif isinstance(val, basestring) or isinstance(val, int) or isinstance(val, float) \ or isinstance(val, int): arg_parser.add_argument(key, default=val) elif isinstance(val, list): arg_parser.add_argument(key, default=val, nargs='+') elif isinstance(val, tuple): arg_parser.add_argument(key, default=list(val), nargs='+') # Is this one of our pseudo-types for dates/tables? elif isinstance(val, dict) and 'type' in val: if val['type'] == 'datestring': arg_parser.add_argument(key, default='', type=_make_string_formatter( val['format'], offset=val['offset'])) elif val['type'] == 'table': if val['format'] is not None: arg_parser.add_argument(key, default='', type=_make_table_formatter( val['format'], offset=val['offset'])) else: arg_parser.add_argument(key, default=val['name'], type=_make_table) else: raise Exception( 'Cannot generate argument for %s of type %s' % (key, type(val))) else: raise Exception('Cannot generate argument for %s of type %s' % (key, type(val))) except Exception as e: print("%%sql arguments: %s from code '%s'" % (str(e), str(code))) return arg_parser
def _arguments(code, module): """Define pipeline arguments. Args: code: the Python code to execute that defines the arguments. """ arg_parser = CommandParser.create('') try: # Define our special argument 'types' and add them to the environment. builtins = {'source': _table, 'datestring': _datestring} env = {} env.update(builtins) # Execute the cell which should be one or more calls to arg(). exec(code, env) # Iterate through the module dictionary. For any newly defined objects, # add args to the parser. for key in env: # Skip internal/private stuff. if key in builtins or key[0] == '_': continue # If we want to support importing query modules into other query modules, uncomment next 4 # Skip imports but add them to the module # if isinstance(env[key], types.ModuleType): # module.__dict__[key] = env[key] # continue val = env[key] key = '--%s' % key if isinstance(val, bool): if val: arg_parser.add_argument(key, default=val, action='store_true') else: arg_parser.add_argument(key, default=val, action='store_false') elif isinstance(val, basestring) or isinstance(val, int) or isinstance(val, float) \ or isinstance(val, int): arg_parser.add_argument(key, default=val) elif isinstance(val, list): arg_parser.add_argument(key, default=val, nargs='+') elif isinstance(val, tuple): arg_parser.add_argument(key, default=list(val), nargs='+') # Is this one of our pseudo-types for dates/tables? elif isinstance(val, dict) and 'type' in val: if val['type'] == 'datestring': arg_parser.add_argument(key, default='', type=_make_string_formatter(val['format'], offset=val['offset'])) elif val['type'] == 'table': if val['format'] is not None: arg_parser.add_argument(key, default='', type=_make_table_formatter(val['format'], offset=val['offset'])) else: arg_parser.add_argument(key, default=val['name'], type=_make_table) else: raise Exception('Cannot generate argument for %s of type %s' % (key, type(val))) else: raise Exception('Cannot generate argument for %s of type %s' % (key, type(val))) except Exception as e: print("%%sql arguments: %s from code '%s'" % (str(e), str(code))) return arg_parser