def chart(line, cell=None):
""" Generate charts with Google Charts. Use %chart --help for more details.
Args:
line: the %chart magic line.
cell: the contents of the cell. Can contain YAML chart options.
"""
parser = _commands.CommandParser(prog='%%chart', description="""
Generate an inline chart using Google Charts using the data in a Table, Query, dataframe, or list.
Numerous types of charts are supported. Options for the charts can be specified in the cell body
using YAML or JSON.
""")
for chart_type in ['annotation', 'area', 'bars', 'bubbles', 'calendar', 'candlestick', 'columns',
'combo', 'gauge', 'geo', 'histogram', 'line', 'map', 'org', 'paged_table',
'pie', 'sankey', 'scatter', 'stepped_area', 'table', 'timeline', 'treemap']:
subparser = parser.subcommand(chart_type,
'Generate a %s chart.' % chart_type)
subparser.add_argument('-f', '--fields',
help='The field(s) to include in the chart')
subparser.add_argument('-d', '--data',
help='The name of the variable referencing the Table or Query to chart',
required=True)
subparser.set_defaults(chart=chart_type)
parser.set_defaults(func=_chart_cell)
return _utils.handle_magic_line(line, cell, parser)
def extension(line):
""" Load an extension. Use %extension --help for more details. """
parser = _commands.CommandParser(prog='%extension', description="""
Load an extension into Datalab. Currently only mathjax is supported.
""")
subparser = parser.subcommand('mathjax', 'Enabled MathJaX support in Datalab.')
subparser.set_defaults(ext='mathjax')
parser.set_defaults(func=_extension)
return _utils.handle_magic_line(line, None, parser)
def _create_bigquery_parser(): """ Create the parser for the %bigquery magics. Note that because we use the func default handler dispatch mechanism of argparse, our handlers can take only one argument which is the parsed args. So we must create closures for the handlers that bind the cell contents and thus must recreate this parser for each cell upon execution. """ parser = _commands.CommandParser(prog='bigquery', description=""" Execute various BigQuery-related operations. Use "%bigquery <command> -h" for help on a specific command. """) # This is a bit kludgy because we want to handle some line magics and some cell magics # with the bigquery command. # %%bigquery sample _add_command(parser, _create_sample_subparser, _sample_cell) # %%bigquery create _add_command(parser, _create_create_subparser, _create_cell) # %%bigquery delete _add_command(parser, _create_delete_subparser, _delete_cell) # %%bigquery dryrun _add_command(parser, _create_dry_run_subparser, _dryrun_cell) # %%bigquery udf _add_command(parser, _create_udf_subparser, _udf_cell, cell_required=True) # %%bigquery execute _add_command(parser, _create_execute_subparser, _execute_cell) # %%bigquery pipeline _add_command(parser, _create_pipeline_subparser, _pipeline_cell) # %bigquery table _add_command(parser, _create_table_subparser, _table_line, cell_prohibited=True) # %bigquery schema _add_command(parser, _create_schema_subparser, _schema_line, cell_prohibited=True) # %bigquery datasets _add_command(parser, _create_datasets_subparser, _datasets_line, cell_prohibited=True) # %bigquery tables _add_command(parser, _create_tables_subparser, _tables_line, cell_prohibited=True) # % bigquery extract _add_command(parser, _create_extract_subparser, _extract_line, cell_prohibited=True) # %bigquery load # TODO(gram): need some additional help, esp. around the option of specifying schema in # cell body and how schema infer may fail. _add_command(parser, _create_load_subparser, _load_cell) return parser
def _create_sql_parser():
sql_parser = _commands.CommandParser(
prog="%%sql",
formatter_class=argparse.RawDescriptionHelpFormatter,
description="""
Create a named SQL module with one or more queries.
The cell body should contain an optional initial part defining the default
values for the variables, if any, using Python code, followed by one or more
queries.
Queries should start with 'DEFINE QUERY <name>' in order to bind them to
<module name>.<query name> in the notebook (as gcp.data.SqlStament instances).
The final query can optionally omit 'DEFINE QUERY <name>', as using the module
name in places where a SqlStatement is expected will resolve to the final query
in the module.
Queries can refer to variables with '$<name>', as well as refer to other queries
within the same module, making it easy to compose nested queries and test their
parts.
The Python code defining the variable default values can assign scalar or list/tuple values to
variables, or one of the special functions 'datestring' and 'source'.
When a variable with a 'datestring' default is expanded it will expand to a formatted
string based on the current date, while a 'source' default will expand to a table whose
name is based on the current date.
datestring() takes two named arguments, 'format' and 'offset'. The former is a
format string that is the same as for Python's time.strftime function. The latter
is a string containing a comma-separated list of expressions such as -1y, +2m,
etc; these are offsets from the time of expansion that are applied in order. The
suffix (y, m, d, h, M) correspond to units of years, months, days, hours and
minutes, while the +n or -n prefix is the number of units to add or subtract from
the time of expansion. Three special values 'now', 'today' and 'yesterday' are
also supported; 'today' and 'yesterday' will be midnight UTC on the current date
or previous days date.
source() can take a 'name' argument for a fixed table name, or 'format' and 'offset'
arguments similar to datestring(), but unlike datestring() will resolve to a Table
with the specified name.
""")
sql_parser.add_argument('-m',
'--module',
help='The name for this SQL module')
sql_parser.set_defaults(func=lambda args, cell: sql_cell(args, cell))
return sql_parser
def storage(line):
"""Implements the storage line magic for ipython notebooks.
Args:
line: the contents of the storage line.
Returns:
The results of executing the cell.
"""
parser = _commands.CommandParser(prog='storage', description="""
Execute various storage-related operations. Use "%storage <command> -h"
for help on a specific command.
""")
# TODO(gram): consider adding a move command too. I did try this already using the
# objects.patch API to change the object name but that fails with an error:
#
# Value 'newname' in content does not agree with value 'oldname'. This can happen when a value
# set through a parameter is inconsistent with a value set in the request.
#
# This is despite 'name' being identified as writable in the storage API docs.
# The alternative would be to use a copy/delete.
copy_parser = parser.subcommand('copy',
'Copy one or more GCS objects to a different location.')
copy_parser.add_argument('-s', '--source', help='The name of the object(s) to copy', nargs='+')
copy_parser.add_argument('-d', '--destination', required=True,
help='The copy destination. For multiple source items this must be a bucket.')
copy_parser.set_defaults(func=_storage_copy)
create_parser = parser.subcommand('create', 'Create one or more GCS buckets.')
create_parser.add_argument('-p', '--project', help='The project associated with the objects')
create_parser.add_argument('-b', '--bucket', help='The name of the bucket(s) to create',
nargs='+')
create_parser.set_defaults(func=_storage_create)
delete_parser = parser.subcommand('delete', 'Delete one or more GCS buckets or objects.')
delete_parser.add_argument('-b', '--bucket', nargs='*',
help='The name of the bucket(s) to remove')
delete_parser.add_argument('-o', '--object', nargs='*',
help='The name of the object(s) to remove')
delete_parser.set_defaults(func=_storage_delete)
list_parser = parser.subcommand('list', 'List buckets in a project, or contents of a bucket.')
list_parser.add_argument('-p', '--project', help='The project associated with the objects')
group = list_parser.add_mutually_exclusive_group()
group.add_argument('-o', '--object',
help='The name of the objects(s) to list; can include wildchars',
nargs='?')
group.add_argument('-b', '--bucket',
help='The name of the buckets(s) to list; can include wildchars',
nargs='?')
list_parser.set_defaults(func=_storage_list)
read_parser = parser.subcommand('read',
'Read the contents of a storage object into a Python variable.')
read_parser.add_argument('-o', '--object', help='The name of the object to read',
required=True)
read_parser.add_argument('-v', '--variable', required=True,
help='The name of the Python variable to set')
read_parser.set_defaults(func=_storage_read)
view_parser = parser.subcommand('view', 'View the contents of a storage object.')
view_parser.add_argument('-n', '--head', type=int, default=20,
help='The number of initial lines to view')
view_parser.add_argument('-t', '--tail', type=int, default=20,
help='The number of lines from end to view')
view_parser.add_argument('-o', '--object', help='The name of the object to view',
required=True)
view_parser.set_defaults(func=_storage_view)
write_parser = parser.subcommand('write',
'Write the value of a Python variable to a storage object.')
write_parser.add_argument('-v', '--variable', help='The name of the source Python variable',
required=True)
write_parser.add_argument('-o', '--object', required=True,
help='The name of the destination GCS object to write')
write_parser.set_defaults(func=_storage_write)
return _utils.handle_magic_line(line, None, parser)