def SQL_parser():
"""
defines the way to parse the magic command ``%%SQL``
"""
parser = MagicCommandParser(prog="SQL",
description='query the database')
parser.add_argument('--df',
type=str,
help='output dataframe',
default="temp_view",
no_eval=True)
parser.add_argument('-n',
'--n',
type=int,
help='number of first lines to display',
default=10,
eval_type=int)
parser.add_argument(
'-q',
'--query',
type=str,
help=
'when used in a single line (no cell), query is the SQL query, the command '
+ 'returns the full dataframe',
default="",
eval_type=str)
parser.add_argument(
'-v',
'--variable',
default="DB",
help='variable name used to store the database object')
return parser
def SQL_import_tsv_parser():
"""
defines the way to parse the magic command ``%SQL_import_tsv``
"""
parser = MagicCommandParser(
prog="SQL_import_tsv",
description='import a tsv file into the database')
parser.add_argument('filename',
type=str,
help='tsv file name',
eval_type=str)
parser.add_argument('-t',
'--table',
type=str,
help='table name',
default="-",
eval_type=str)
parser.add_argument('--verbose',
help='print progress',
action="store_true")
parser.add_argument(
'-v',
'--variable',
default="DB",
help='variable name used to store the database object')
return parser
def mpl_style_parser():
"""
defines the way to parse the magic command ``%mpl_style``
"""
parser = MagicCommandParser(description="changes matplotlib style", prog="mpl_style")
parser.add_argument("style", type=str, help="style, ggplot for exemple", default="ggplot")
return parser
def mlnet_parser():
"""
Defines the way to parse the magic command ``%%mlnet``.
"""
parser = MagicCommandParser(
prog="mlnet",
description=
'Compiles and wrap a C# function into a Python function.\n'
'Automatically adds ML.net dependencies.')
parser.add_argument('name', type=str, help='function name')
parser.add_argument(
'-i',
'--idep',
nargs='*',
action='append',
help='internal dependencies (like System, System.Linq)')
parser.add_argument(
'-d',
'--dep',
nargs='*',
action='append',
help='dependencies (assembly name without extension)')
parser.add_argument('-c',
'--catch',
action='store',
default=False,
help='catch exception')
return parser
def snap_parser():
"""
defines the way to parse the magic command ``%snap``
"""
parser = MagicCommandParser(prog="snap",
description='insert a snap window inside a notebook')
parser.add_argument(
'-f',
'--file',
type=str,
default="",
help='scratch or snap file to display')
parser.add_argument(
'-d',
'--div',
type=str,
default="scratch_div_id",
help='id for the HTML div')
parser.add_argument(
'-W',
'--width',
type=int,
default=1000,
help='window width')
parser.add_argument(
'-H',
'--height',
type=int,
default=600,
help='window height')
return parser
def dfs_mkdir_parser():
"""
defines the way to parse the magic command ``%dfs_mkdir``
"""
parser = MagicCommandParser(prog="dfs_mkdir",
description='create a folder')
parser.add_argument('path', type=str, help='path to remove')
return parser
def remote_ls_parser():
"""
defines the way to parse the magic command ``%remote_ls``
"""
parser = MagicCommandParser(
prog="remote_ls",
description='returns the content of a folder as a dataframe')
parser.add_argument('path', type=str, help='path to look into')
return parser
def lsrepo_parser():
"""
Defines the way to parse the magic command ``%lsrepo``.
"""
parser = MagicCommandParser(prog="lsrepo",
description='display the content of a repository (GIT or SVN)')
parser.add_argument('path', type=str, nargs="?",
help='path', default=".")
return parser
def HIVE_parser():
"""
defines the way to parse the magic command ``%%HIVE``
"""
parser = MagicCommandParser(
prog="HIVE",
description=
'The command store the content of the cell as a local file.')
parser.add_argument('file', type=str, help='file name')
return parser
def PYTHON_parser():
"""
Defines the way to parse the magic command ``%%PYTHON``.
"""
parser = MagicCommandParser(
prog="PYTHON",
description=
'the command stores the content of the cell as a local file.')
parser.add_argument('file', type=str, help='filename')
return parser
def test_eval(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
params = {"x": 3, "y": 4}
cl = MagicCommandParser(prog="test_command")
res = cl.eval("x+y", params, fLOG=fLOG)
fLOG(res)
assert res == 7
def SQL_parser():
"""
defines the way to parse the magic command ``%%SQL``
"""
parser = MagicCommandParser(
prog="SQL", description='query the database')
parser.add_argument(
'--df',
type=str,
help='output dataframe',
default="temp_view",
no_eval=True)
parser.add_argument(
'-n',
'--n',
type=int,
help='number of first lines to display',
default=10,
eval_type=int)
parser.add_argument(
'-q',
'--query',
type=str,
help=
'when used in a single line (no cell), query is the SQL query, the command returns the full dataframe',
default="",
eval_type=str)
parser.add_argument(
'-v',
'--variable',
default="DB",
help='variable name used to store the database object')
return parser
def test_parse(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
parser = MagicCommandParser(prog="test_command",
description='display the first lines of a text file')
typstr = str # unicode#
parser.add_argument('f', type=typstr, help='filename')
parser.add_argument(
'-n', '--n',
type=typstr, default=10,
help='number of lines to display')
parser.add_argument(
'-e',
'--encoding',
default="utf8",
help='file encoding')
params = {"x": 3, "y": 4}
res = parser.parse_cmd('this.py -n x+y', context=params, fLOG=fLOG)
fLOG(res.__dict__)
r = parser.format_help()
assert "usage: test_command [-h] [-n N] [-e ENCODING] f" in r
fLOG("###\n", r, "###\n")
fLOG(parser.usage)
self.assertEqual(res.n, 7)
def remote_py_parser():
"""
defines the way to parse the magic command ``%remote_py``
"""
parser = MagicCommandParser(prog="remote_py",
description='run a python script on the bridge')
parser.add_argument(
'file',
type=str,
help='file name')
parser.add_argument(
'args',
nargs='*',
type=str,
help='list of options for the job')
parser.add_argument(
'-i',
'--interpreter',
type=str,
default='python',
help='change the interpreter, python by default')
parser.add_argument(
'--raw-output',
default=False,
action='store_true',
help='display raw text instead of HTML')
return parser
def remote_down_cluster_parser():
"""
defines the way to parse the magic command ``%remote_down_cluster``
"""
parser = MagicCommandParser(prog="remote_down_cluster",
description='download a file from the cluster to the remote machine and then to your local machine')
parser.add_argument(
'remotepath',
type=str,
help='remote path (HDFS) of the uploaded file')
parser.add_argument(
'localfile',
type=str,
help='local file to upload')
parser.add_argument(
'-o',
'--overwrite',
action='store_true',
default=False,
help='overwrite the local file')
parser.add_argument(
'-m',
'--merge',
action='store_true',
default=False,
help='merges files in folder in a single file')
return parser
def tail_parser():
"""
defines the way to parse the magic command ``%tail``
"""
parser = MagicCommandParser(prog="tail",
description='display the last lines of a text file')
parser.add_argument('f', type=str, help='filename')
parser.add_argument(
'-n',
'--n',
type=int,
default=10,
help='number of lines to display')
parser.add_argument(
'-r',
'--raw',
default=False,
action='store_true',
help='display raw text instead of HTML')
parser.add_argument(
'-e',
'--encoding',
default="utf8",
help='file encoding')
return parser
def hd_pig_submit_parser():
"""
defines the way to parse the magic command ``%hd_pig_submit``
"""
parser = MagicCommandParser(prog="hd_pig_submit",
description='Submits a job to the cluster, the job is local, the job is first uploaded to the cluster. The magic command populates the local variable last_job with the submitted job id.')
parser.add_argument(
'file',
type=str,
help='file name')
parser.add_argument(
'-d',
'--dependency',
nargs="*",
type=list,
help='dependency of the job, the python script')
parser.add_argument(
'-s',
'--stop_on_failure',
action='store_true',
default=False,
help='if true, the job stops on failure right away')
parser.add_argument(
'-o',
'--options',
nargs='*',
type=list,
help='list of options for the job')
return parser
def blob_lsl_parser():
"""
defines the way to parse the magic command ``%blob_lsl``
"""
parser = MagicCommandParser(prog="blob_lsl",
description='describes the content of folder in a blob storage + metadata')
parser.add_argument(
'path',
type=str,
help='path to look into, </path> or <container>/<path>')
return parser
def PYTHON_parser():
"""
Defines the way to parse the magic command ``%%PYTHON``.
"""
parser = MagicCommandParser(prog="PYTHON",
description='the command stores the content of the cell as a local file.')
parser.add_argument(
'file',
type=str,
help='filename')
return parser
def blob_rmr_parser():
"""
defines the way to parse the magic command ``%blob_rmr``
"""
parser = MagicCommandParser(prog="blob_rmr",
description='remove a remote folder')
parser.add_argument(
'remotepath',
type=str,
help='remote path to remove')
return parser
def hd_job_status_parser():
"""
defines the way to parse the magic command ``%hd_job_status``
"""
parser = MagicCommandParser(prog="hd_job_status",
description='get the status of the job')
parser.add_argument(
'jobid',
type=str,
help='job id')
return parser
def hd_job_kill_parser():
"""
defines the way to parse the magic command ``%hd_job_kill``
"""
parser = MagicCommandParser(prog="hd_job_kill",
description='kill a job')
parser.add_argument(
'jobid',
type=str,
help='job id')
return parser
def HIVE_parser():
"""
defines the way to parse the magic command ``%%HIVE``
"""
parser = MagicCommandParser(prog="HIVE",
description='The command store the content of the cell as a local file.')
parser.add_argument(
'file',
type=str,
help='file name')
return parser
def dfs_ls_parser():
"""
defines the way to parse the magic command ``%dfs_ls``
"""
parser = MagicCommandParser(prog="dfs_ls",
description='returns the content of a folder from the cluster as a dataframe')
parser.add_argument(
'path',
type=str,
help='path to look into')
return parser
def mpl_style_parser():
"""
defines the way to parse the magic command ``%mpl_style``
"""
parser = MagicCommandParser(description='changes matplotlib style',
prog="mpl_style")
parser.add_argument('style',
type=str,
help='style, ggplot for exemple',
default="ggplot")
return parser
def job_syntax_parser():
"""
defines the way to parse the magic command ``%job_syntax``
"""
parser = MagicCommandParser(prog="remote_py",
description='check syntax of a pig job')
parser.add_argument(
'file',
type=str,
help='file name')
return parser
def dfs_mkdir_parser():
"""
defines the way to parse the magic command ``%dfs_mkdir``
"""
parser = MagicCommandParser(prog="dfs_mkdir",
description='create a folder')
parser.add_argument(
'path',
type=str,
help='path to remove')
return parser
def SQL_tables_parser():
"""
defines the way to parse the magic command ``%SQL_tables``
"""
parser = MagicCommandParser(prog="SQL_tables",
description='list the tables of a database')
parser.add_argument(
'-v',
'--variable',
default="DB",
help='variable name used to store the database object')
return parser
def SQL_tables_parser():
"""
defines the way to parse the magic command ``%SQL_tables``
"""
parser = MagicCommandParser(
prog="SQL_tables", description='list the tables of a database')
parser.add_argument(
'-v',
'--variable',
default="DB",
help='variable name used to store the database object')
return parser
def SQL_close_parser():
"""
defines the way to parse the magic command ``%SQL_close``
"""
parser = MagicCommandParser(prog="SQL_close",
description='connect to a SQL database')
parser.add_argument(
'-v',
'--variable',
default="DB",
help='variable name used to store the database object (and to close)')
return parser
def SQL_refresh_completion_parser():
"""
defines the way to parse the magic command ``%SQL_refresh_completion``
"""
parser = MagicCommandParser(prog="SQL_refresh_completion",
description='refresh completion (tables names, ...)')
parser.add_argument(
'-v',
'--variable',
default="DB",
help='variable name used to store the database object')
return parser
def maml_parser():
"""
Defines the way to parse the magic command ``%%maml``.
"""
parser = MagicCommandParser(prog="maml",
description='Runs a maml script.')
parser.add_argument('-q',
'--quiet',
action='store_true',
default=False,
help='hide output')
return parser
def SQL_refresh_completion_parser():
"""
defines the way to parse the magic command ``%SQL_refresh_completion``
"""
parser = MagicCommandParser(
prog="SQL_refresh_completion",
description='refresh completion (tables names, ...)')
parser.add_argument(
'-v',
'--variable',
default="DB",
help='variable name used to store the database object')
return parser
def SQL_drop_table_parser():
"""
defines the way to parse the magic command ``%SQL_drop_table``
"""
parser = MagicCommandParser(prog="SQL_drop_table",
description='drop a table from a database')
parser.add_argument('table', type=str, help='table', eval_type=str)
parser.add_argument(
'-v',
'--variable',
default="DB",
help='variable name used to store the database object')
return parser
def open_remote_shell_parser():
"""
defines the way to parse the magic command ``%open_remote_shell``
"""
parser = MagicCommandParser(
prog="open_remote_shell",
description='command will execute as if they were in a shell')
parser.add_argument('-f',
'--format',
type=str,
default='html',
help='formart of this output, html or plain')
return parser
def lsrepo_parser():
"""
defines the way to parse the magic command ``%lsrepo``
"""
parser = MagicCommandParser(
prog="lsrepo",
description='display the content of a repository (GIT or SVN)')
parser.add_argument('path',
type=str,
nargs="?",
help='path',
default=".")
return parser
def SQL_close_parser():
"""
defines the way to parse the magic command ``%SQL_close``
"""
parser = MagicCommandParser(prog="SQL_close",
description='connect to a SQL database')
parser.add_argument(
'-v',
'--variable',
default="DB",
help=
'variable name used to store the database object (and to close)')
return parser
def SQL_add_function_parser():
"""
defines the way to parse the magic command ``%SQL_add_function``
"""
parser = MagicCommandParser(prog="SQL_add_function",
description='add a custom function to the database')
parser.add_argument('funct', type=str, help='function name')
parser.add_argument(
'-v',
'--variable',
default="DB",
help='variable name used to store the database object')
return parser
def hd_queue_parser():
"""
defines the way to parse the magic command ``%hd_queue``
"""
parser = MagicCommandParser(prog="hd_queue",
description='displays the job queue')
parser.add_argument(
'-s',
'--showall',
action="store_true",
default=False,
help="show all jobs, only users'")
return parser
def open_remote_shell_parser():
"""
defines the way to parse the magic command ``%open_remote_shell``
"""
parser = MagicCommandParser(prog="open_remote_shell",
description='command will execute as if they were in a shell')
parser.add_argument(
'-f',
'--format',
type=str,
default='html',
help='formart of this output, html or plain')
return parser
def SQL_connect_parser():
"""
defines the way to parse the magic command ``%SQL_connect``
"""
parser = MagicCommandParser(prog="SQL_connect",
description='connect to a SQL database')
parser.add_argument('filename', type=str,
help='database filename', eval_type=str)
parser.add_argument(
'-v',
'--variable',
default="DB",
help='variable name used to store the database object')
return parser
def encoding_parser():
"""
Defines the way to parse the magic command ``%encoding``.
"""
parser = MagicCommandParser(prog="encoding",
description='guess the encoding of a file')
parser.add_argument('f', type=str, help='filename')
parser.add_argument(
'-n',
'--n',
type=int,
default=2**20,
help='maximum number of lines to use to guess the encoding')
return parser
def blob_copy_parser():
"""
defines the way to parse the magic command ``%blob_copy``
"""
parser = MagicCommandParser(prog="blob_copy",
description='copy a blob folder')
parser.add_argument(
'remotepath',
type=str,
help='remote path to remove')
parser.add_argument(
'remotedest',
type=str,
help='remote destination')
return parser
def remote_up_cluster_parser():
"""
defines the way to parse the magic command ``%remote_up_cluster``
"""
parser = MagicCommandParser(prog="remote_up_cluster",
description='upload a file to the remote machine and then to the cluster')
parser.add_argument(
'localfile',
type=str,
help='local file to upload')
parser.add_argument(
'remotepath',
type=str,
help='remote path (HDFS) of the uploaded file')
return parser
def blob_up_parser():
"""
defines the way to parse the magic command ``%blob_up``
"""
parser = MagicCommandParser(prog="blob_up",
description='upload a file on a blob storage, we assume the container is the first element to the remote path')
parser.add_argument(
'localfile',
type=str,
help='local file to upload')
parser.add_argument(
'remotepath',
type=str,
help='remote path of the uploaded file')
return parser
def test_parse(self):
parser = MagicCommandParser(
prog="test_command",
description='display the first lines of a text file')
typstr = str # unicode#
parser.add_argument('f', type=typstr, help='filename')
parser.add_argument('-n',
'--n',
type=typstr,
default=10,
help='number of lines to display')
parser.add_argument('-e',
'--encoding',
default="utf8",
help='file encoding')
params = {"x": 3, "y": 4}
res = parser.parse_cmd('this.py -n x+y', context=params)
self.assertNotEmpty(res)
r = parser.format_help()
self.assertIn("usage: test_command [-h] [-n N] [-e ENCODING] f", r)
self.assertEqual(res.n, 7)
def remote_py_parser():
"""
defines the way to parse the magic command ``%remote_py``
"""
parser = MagicCommandParser(
prog="remote_py", description='run a python script on the bridge')
parser.add_argument('file', type=str, help='file name')
parser.add_argument('args',
nargs='*',
type=str,
help='list of options for the job')
parser.add_argument('-i',
'--interpreter',
type=str,
default='python',
help='change the interpreter, python by default')
parser.add_argument('--raw-output',
default=False,
action='store_true',
help='display raw text instead of HTML')
return parser
def head_parser():
"""
defines the way to parse the magic command ``%head``
"""
parser = MagicCommandParser(
prog="head", description='display the first lines of a text file')
parser.add_argument('f', type=str, help='filename')
parser.add_argument('-n',
'--n',
type=int,
default=10,
help='number of lines to display')
parser.add_argument('-r',
'--raw',
default=False,
action='store_true',
help='display raw text instead of HTML')
parser.add_argument('-e',
'--encoding',
default="utf8",
help='file encoding')
return parser
def SQL_schema_parser():
"""
defines the way to parse the magic command ``%SQL_schema``
"""
parser = MagicCommandParser(prog="SQL_schema",
description='schema of a table')
parser.add_argument('table', type=str, help='table', eval_type=str)
parser.add_argument('--as_list',
help='as a dictionary (False) or as a list (True)',
action="store_true")
parser.add_argument(
'-v',
'--variable',
default="DB",
help='variable name used to store the database object')
return parser
def remote_down_cluster_parser():
"""
defines the way to parse the magic command ``%remote_down_cluster``
"""
parser = MagicCommandParser(
prog="remote_down_cluster",
description=
'download a file from the cluster to the remote machine and then to your local machine'
)
parser.add_argument('remotepath',
type=str,
help='remote path (HDFS) of the uploaded file')
parser.add_argument('localfile', type=str, help='local file to upload')
parser.add_argument('-o',
'--overwrite',
action='store_true',
default=False,
help='overwrite the local file')
parser.add_argument('-m',
'--merge',
action='store_true',
default=False,
help='merges files in folder in a single file')
return parser
def remote_down_parser():
"""
defines the way to parse the magic command ``%remote_down``
"""
parser = MagicCommandParser(
prog="remote_down",
description='download a file from the remote machine')
parser.add_argument('remotepath',
type=str,
help='remote path of the uploaded file')
parser.add_argument('localfile', type=str, help='local file to upload')
parser.add_argument('-o',
'--overwrite',
action='store_true',
default=False,
help='overwrite the local file')
return parser
def remote_up_parser():
"""
defines the way to parse the magic command ``%remote_up``
"""
parser = MagicCommandParser(
prog="remote_up",
description='upload a file to the remote machine')
parser.add_argument('localfile', type=str, help='local file to upload')
parser.add_argument('remotepath',
type=str,
help='remote path of the uploaded file')
return parser
def job_syntax_parser():
"""
defines the way to parse the magic command ``%job_syntax``
"""
parser = MagicCommandParser(prog="remote_py",
description='check syntax of a pig job')
parser.add_argument('file', type=str, help='file name')
parser.add_argument('--raw-output',
default=False,
action='store_true',
help='display raw text instead of HTML')
return parser
def SQL_drop_table_parser():
"""
defines the way to parse the magic command ``%SQL_drop_table``
"""
parser = MagicCommandParser(prog="SQL_drop_table",
description='drop a table from a database')
parser.add_argument('table', type=str, help='table', eval_type=str)
parser.add_argument(
'-v',
'--variable',
default="DB",
help='variable name used to store the database object')
return parser
def SQL_import_df_parser():
"""
defines the way to parse the magic command ``%SQL_import_df``
"""
parser = MagicCommandParser(
prog="SQL_import_df",
description='import a dataframe into the database')
parser.add_argument('df', type=str, help='dataframe', no_eval=True)
parser.add_argument('-t',
'--table',
type=str,
help='table name',
default="-",
eval_type=str)
parser.add_argument(
'-v',
'--variable',
default="DB",
help='variable name used to store the database object')
return parser
def dfs_rm_parser():
"""
defines the way to parse the magic command ``%dfs_rm``
"""
parser = MagicCommandParser(prog="dfs_rm",
description='remove a file on the cluster')
parser.add_argument('path', type=str, help='path to remove')
parser.add_argument('-r',
'--recursive',
action='store_true',
default=False,
help='to remove subfolders too')
return parser
def hive_submit_parser():
"""
defines the way to parse the magic command ``%hive_submit``
"""
parser = MagicCommandParser(
prog="hive_submit",
description='Submits a job to the cluster, the job is local, ' +
'the job is first uploaded to the cluster. The magic ' +
'command populates the local variable last_job with the submitted job id.'
)
parser.add_argument('file', type=str, help='file name')
parser.add_argument('-r',
'--redirection',
type=str,
default="redirection",
help='list of options for the job')
parser.add_argument('--raw-output',
default=False,
action='store_true',
help='display raw text instead of HTML')
return parser
def encoding_parser():
"""
defines the way to parse the magic command ``%encoding``
"""
parser = MagicCommandParser(prog="encoding",
description='guess the encoding of a file')
parser.add_argument('f', type=str, help='filename')
parser.add_argument(
'-n',
'--n',
type=int,
default=2**20,
help='maximum number of lines to use to guess the encoding')
return parser
def SQL_add_function_parser():
"""
defines the way to parse the magic command ``%SQL_add_function``
"""
parser = MagicCommandParser(
prog="SQL_add_function",
description='add a custom function to the database')
parser.add_argument('funct', type=str, help='function name')
parser.add_argument(
'-v',
'--variable',
default="DB",
help='variable name used to store the database object')
return parser
def runpy_parser():
"""
Defines the way to parse the magic command ``%%runpy``.
"""
parser = MagicCommandParser(
prog="runpy",
description='run a python script which accepts standards input and '
+ 'produces standard outputs, a timeout is set up at 10s')
parser.add_argument('file', type=str, help='python file')
parser.add_argument('args',
type=str,
nargs="*",
help='arguments for the scripts',
default=".")
return parser
