def _create_sql_parser():
sql_parser = CommandParser(
prog="%%sql",
formatter_class=argparse.RawDescriptionHelpFormatter,
description="""
Create a named SQL module with one or more queries.
The cell body should contain an optional initial part defining the default
values for the variables, if any, using Python code, followed by one or more
queries.
Queries should start with 'DEFINE QUERY <name>' in order to bind them to
<module name>.<query name> in the notebook (as datalab.data.SqlStament instances).
The final query can optionally omit 'DEFINE QUERY <name>', as using the module
name in places where a SqlStatement is expected will resolve to the final query
in the module.
Queries can refer to variables with '$<name>', as well as refer to other queries
within the same module, making it easy to compose nested queries and test their
parts.
The Python code defining the variable default values can assign scalar or list/tuple values to
variables, or one of the special functions 'datestring' and 'source'.
When a variable with a 'datestring' default is expanded it will expand to a formatted
string based on the current date, while a 'source' default will expand to a table whose
name is based on the current date.
datestring() takes two named arguments, 'format' and 'offset'. The former is a
format string that is the same as for Python's time.strftime function. The latter
is a string containing a comma-separated list of expressions such as -1y, +2m,
etc; these are offsets from the time of expansion that are applied in order. The
suffix (y, m, d, h, M) correspond to units of years, months, days, hours and
minutes, while the +n or -n prefix is the number of units to add or subtract from
the time of expansion. Three special values 'now', 'today' and 'yesterday' are
also supported; 'today' and 'yesterday' will be midnight UTC on the current date
or previous days date.
source() can take a 'name' argument for a fixed table name, or 'format' and 'offset'
arguments similar to datestring(), but unlike datestring() will resolve to a Table
with the specified name.
""")
sql_parser.add_argument('-m',
'--module',
help='The name for this SQL module')
sql_parser.add_argument('-d',
'--dialect',
help='BigQuery SQL dialect',
choices=['legacy', 'standard'],
default='legacy')
sql_parser.add_argument('-b',
'--billing',
type=int,
help='BigQuery billing tier')
sql_parser.set_defaults(func=lambda args, cell: sql_cell(args, cell))
return sql_parser
def _create_sql_parser():
sql_parser = CommandParser(prog="%%sql",
formatter_class=argparse.RawDescriptionHelpFormatter,
description="""
Create a named SQL module with one or more queries.
The cell body should contain an optional initial part defining the default
values for the variables, if any, using Python code, followed by one or more
queries.
Queries should start with 'DEFINE QUERY <name>' in order to bind them to
<module name>.<query name> in the notebook (as datalab.data.SqlStament instances).
The final query can optionally omit 'DEFINE QUERY <name>', as using the module
name in places where a SqlStatement is expected will resolve to the final query
in the module.
Queries can refer to variables with '$<name>', as well as refer to other queries
within the same module, making it easy to compose nested queries and test their
parts.
The Python code defining the variable default values can assign scalar or list/tuple values to
variables, or one of the special functions 'datestring' and 'source'.
When a variable with a 'datestring' default is expanded it will expand to a formatted
string based on the current date, while a 'source' default will expand to a table whose
name is based on the current date.
datestring() takes two named arguments, 'format' and 'offset'. The former is a
format string that is the same as for Python's time.strftime function. The latter
is a string containing a comma-separated list of expressions such as -1y, +2m,
etc; these are offsets from the time of expansion that are applied in order. The
suffix (y, m, d, h, M) correspond to units of years, months, days, hours and
minutes, while the +n or -n prefix is the number of units to add or subtract from
the time of expansion. Three special values 'now', 'today' and 'yesterday' are
also supported; 'today' and 'yesterday' will be midnight UTC on the current date
or previous days date.
source() can take a 'name' argument for a fixed table name, or 'format' and 'offset'
arguments similar to datestring(), but unlike datestring() will resolve to a Table
with the specified name.
""")
sql_parser.add_argument('-m', '--module', help='The name for this SQL module')
sql_parser.set_defaults(func=lambda args, cell: sql_cell(args, cell))
return sql_parser
def _arguments(code, module):
"""Define pipeline arguments.
Args:
code: the Python code to execute that defines the arguments.
"""
arg_parser = CommandParser.create('')
try:
# Define our special argument 'types' and add them to the environment.
builtins = {'source': _table, 'datestring': _datestring}
env = {}
env.update(builtins)
# Execute the cell which should be one or more calls to arg().
exec(code, env)
# Iterate through the module dictionary. For any newly defined objects,
# add args to the parser.
for key in env:
# Skip internal/private stuff.
if key in builtins or key[0] == '_':
continue
# If we want to support importing query modules into other query modules, uncomment next 4
# Skip imports but add them to the module
# if isinstance(env[key], types.ModuleType):
# module.__dict__[key] = env[key]
# continue
val = env[key]
key = '--%s' % key
if isinstance(val, bool):
if val:
arg_parser.add_argument(key,
default=val,
action='store_true')
else:
arg_parser.add_argument(key,
default=val,
action='store_false')
elif isinstance(val, basestring) or isinstance(val, int) or isinstance(val, float) \
or isinstance(val, int):
arg_parser.add_argument(key, default=val)
elif isinstance(val, list):
arg_parser.add_argument(key, default=val, nargs='+')
elif isinstance(val, tuple):
arg_parser.add_argument(key, default=list(val), nargs='+')
# Is this one of our pseudo-types for dates/tables?
elif isinstance(val, dict) and 'type' in val:
if val['type'] == 'datestring':
arg_parser.add_argument(key,
default='',
type=_make_string_formatter(
val['format'],
offset=val['offset']))
elif val['type'] == 'table':
if val['format'] is not None:
arg_parser.add_argument(key,
default='',
type=_make_table_formatter(
val['format'],
offset=val['offset']))
else:
arg_parser.add_argument(key,
default=val['name'],
type=_make_table)
else:
raise Exception(
'Cannot generate argument for %s of type %s' %
(key, type(val)))
else:
raise Exception('Cannot generate argument for %s of type %s' %
(key, type(val)))
except Exception as e:
print("%%sql arguments: %s from code '%s'" % (str(e), str(code)))
return arg_parser
def _arguments(code, module):
"""Define pipeline arguments.
Args:
code: the Python code to execute that defines the arguments.
"""
arg_parser = CommandParser.create('')
try:
# Define our special argument 'types' and add them to the environment.
builtins = {'source': _table, 'datestring': _datestring}
env = {}
env.update(builtins)
# Execute the cell which should be one or more calls to arg().
exec(code, env)
# Iterate through the module dictionary. For any newly defined objects,
# add args to the parser.
for key in env:
# Skip internal/private stuff.
if key in builtins or key[0] == '_':
continue
# If we want to support importing query modules into other query modules, uncomment next 4
# Skip imports but add them to the module
# if isinstance(env[key], types.ModuleType):
# module.__dict__[key] = env[key]
# continue
val = env[key]
key = '--%s' % key
if isinstance(val, bool):
if val:
arg_parser.add_argument(key, default=val, action='store_true')
else:
arg_parser.add_argument(key, default=val, action='store_false')
elif isinstance(val, basestring) or isinstance(val, int) or isinstance(val, float) \
or isinstance(val, int):
arg_parser.add_argument(key, default=val)
elif isinstance(val, list):
arg_parser.add_argument(key, default=val, nargs='+')
elif isinstance(val, tuple):
arg_parser.add_argument(key, default=list(val), nargs='+')
# Is this one of our pseudo-types for dates/tables?
elif isinstance(val, dict) and 'type' in val:
if val['type'] == 'datestring':
arg_parser.add_argument(key, default='',
type=_make_string_formatter(val['format'],
offset=val['offset']))
elif val['type'] == 'table':
if val['format'] is not None:
arg_parser.add_argument(key, default='',
type=_make_table_formatter(val['format'],
offset=val['offset']))
else:
arg_parser.add_argument(key, default=val['name'], type=_make_table)
else:
raise Exception('Cannot generate argument for %s of type %s' % (key, type(val)))
else:
raise Exception('Cannot generate argument for %s of type %s' % (key, type(val)))
except Exception as e:
print("%%sql arguments: %s from code '%s'" % (str(e), str(code)))
return arg_parser