def parse(cls, args):
"""
Parse command line arguments to construct a dictionary of command
parameters that can be used to create a command
Args:
`args`: sequence of arguments
Returns:
Dictionary that can be used in create method
Raises:
ParseError: when the arguments are not correct
"""
try:
(options, args) = cls.optparser.parse_args(args)
if (options.db_tap_id is None):
raise ParseError("db_tap_id is required",
cls.optparser.format_help())
if (options.query is None):
raise ParseError("query is required",
cls.optparser.format_help())
except OptionParsingError as e:
raise ParseError(e.msg, cls.optparser.format_help())
except OptionParsingExit as e:
return None
if options.macros is not None:
options.macros = json.loads(options.macros)
v = vars(options)
v["command_type"] = "DbTapQueryCommand"
return v
def parse(cls, args):
"""
Parse command line arguments to construct a dictionary of command
parameters that can be used to create a command
Args:
`args`: sequence of arguments
Returns:
Dictionary that can be used in create method
Raises:
ParseError: when the arguments are not correct
"""
try:
(options, args) = cls.optparser.parse_args(args)
if options.mode not in ["1", "2"]:
raise ParseError("mode must be either '1' or '2'",
cls.optparser.format_help())
if (options.dbtap_id is None) or (options.db_table is None):
raise ParseError("dbtap_id and db_table are required",
cls.optparser.format_help())
# TODO: Semantic checks for parameters in mode 1 and 2
except OptionParsingError as e:
raise ParseError(e.msg, cls.optparser.format_help())
except OptionParsingExit as e:
return None
v = vars(options)
v["command_type"] = "DbImportCommand"
return v
def validate_cmdline(cls, options):
bool_cmdline = options.cmdline is not None
bool_other_options = options.script_location is not None or options.program is not None
# if both are false then no option is specified ==> raise ParseError
# if both are true then atleast two option specified ==> raise ParseError
if bool_cmdline == bool_other_options:
raise ParseError("Exactly One of script location or program or cmdline should be specified", cls.optparser.format_help())
if bool_cmdline:
if options.language is not None:
raise ParseError("Language cannot be specified with the commandline option", cls.optparser.format_help())
def validate_program(cls, options):
bool_program = options.program is not None
bool_other_options = options.script_location is not None or options.cmdline is not None or options.sql is not None
# if both are false then no option is specified ==> raise ParseError
# if both are true then atleast two option specified ==> raise ParseError
if bool_program == bool_other_options:
raise ParseError("Exactly One of script location or program or cmdline or sql should be specified", cls.optparser.format_help())
if bool_program:
if options.language is None:
raise ParseError("Unspecified language for Program", cls.optparser.format_help())
def parse(cls, args):
"""
Parse command line arguments to construct a dictionary of command
parameters that can be used to create a command
Args:
`args`: sequence of arguments
Returns:
Dictionary that can be used in create method
Raises:
ParseError: when the arguments are not correct
"""
try:
(options, args) = cls.optparser.parse_args(args)
if options.query is None and options.script_location is None:
raise ParseError("One of query or script location"
" must be specified",
cls.optparser.format_help())
except OptionParsingError as e:
raise ParseError(e.msg, cls.optparser.format_help())
except OptionParsingExit as e:
return None
if options.script_location is not None:
if options.query is not None:
raise ParseError(
"Both query and script_location cannot be specified",
cls.optparser.format_help())
if ((options.script_location.find("s3://") != 0) and
(options.script_location.find("s3n://") != 0)):
# script location is local file
try:
q = open(options.script_location).read()
except IOError as e:
raise ParseError("Unable to open script location: %s" %
str(e),
cls.optparser.format_help())
options.script_location = None
options.query = q
if options.macros is not None:
options.macros = json.loads(options.macros)
v = vars(options)
v["command_type"] = "HiveCommand"
return v
def parse(cls, args):
"""
Parse command line arguments to construct a dictionary of command
parameters that can be used to create a command
Args:
`args`: sequence of arguments
Returns:
Dictionary that can be used in create method
Raises:
ParseError: when the arguments are not correct
"""
try:
(options, args) = cls.optparser.parse_args(args)
except OptionParsingError as e:
raise ParseError(e.msg, cls.optparser.format_help())
except OptionParsingExit as e:
return None
SparkCommand.validate_program(options)
SparkCommand.validate_script_location(options)
SparkCommand.validate_cmdline(options)
SparkCommand.validate_sql(options)
if options.macros is not None:
options.macros = json.loads(options.macros)
v = vars(options)
v["command_type"] = "SparkCommand"
return v
def validate_script_location(cls, options):
bool_script_location = options.script_location is not None
bool_other_options = options.program is not None or options.cmdline is not None or options.sql is not None
# if both are false then no option is specified ==> raise ParseError
# if both are true then atleast two option specified ==> raise ParseError
if bool_script_location == bool_other_options:
raise ParseError(
"Exactly One of script location or program or cmdline or sql should be specified",
cls.optparser.format_help())
if bool_script_location:
if options.language is not None:
raise ParseError(
"Both script location and language cannot be specified together",
cls.optparser.format_help())
# for now, aws script_location is not supported and throws an error
if ((options.script_location.find("s3://") != 0)
and (options.script_location.find("s3n://") != 0)):
# script location is local file so set the program as the text from the file
try:
q = open(options.script_location).read()
except IOError as e:
raise ParseError(
"Unable to open script location: %s" % str(e),
cls.optparser.format_help())
fileName, fileExtension = os.path.splitext(
options.script_location)
# getting the language of the program from the file extension
if fileExtension == ".py":
options.language = "python"
elif fileExtension == ".scala":
options.language = "scala"
else:
raise ParseError(
"Invalid program type, Please choose one from python or scala %s"
% str(fileExtension), cls.optparser.format_help())
else:
raise ParseError(
"Invalid location, Please choose a local file location",
cls.optparser.format_help())
options.script_location = None
options.program = q
def parse(cls, args):
"""
Parse command line arguments to construct a dictionary of command
parameters that can be used to create a command
Args:
`args`: sequence of arguments
Returns:
Dictionary that can be used in create method
Raises:
ParseError: when the arguments are not correct
"""
try:
(options, args) = cls.optparser.parse_args(args)
if options.mode not in ["1", "2"]:
raise ParseError("mode must be either '1' or '2'",
cls.optparser.format_help())
if (options.dbtap_id is None) or (options.db_table is None):
raise ParseError("dbtap_id and db_table are required",
cls.optparser.format_help())
if options.mode is "1":
if options.hive_table is None:
raise ParseError("hive_table is required for mode 1",
cls.optparser.format_help())
elif options.export_dir is None: # mode 2
raise ParseError("export_dir is required for mode 2",
cls.optparser.format_help())
if options.db_update_mode is not None:
if options.db_update_mode not in ["allowinsert", "updateonly"]:
raise ParseError(
"db_update_mode should either be left blank for append "
"mode or be 'updateonly' or 'allowinsert'",
cls.optparser.format_help())
if options.db_update_mode is "updateonly":
if options.db_update_keys is None:
raise ParseError(
"db_update_keys is required when db_update_mode "
"is 'updateonly'", cls.optparser.format_help())
elif options.db_update_keys is not None:
raise ParseError(
"db_update_keys is used only when db_update_mode "
"is 'updateonly'", cls.optparser.format_help())
except OptionParsingError as e:
raise ParseError(e.msg, cls.optparser.format_help())
except OptionParsingExit as e:
return None
v = vars(options)
v["command_type"] = "DbExportCommand"
return v
def find(cls, name="default", **kwargs):
if (name is None) or (name == "default"):
conn = Qubole.agent()
return cls(conn.get(cls.rest_entity_path))
else:
raise ParseError(
"Bad name %s" % name,
"Hadoop Clusters can only be named 'default' currently")
def parse(cls, args):
"""
Parse command line arguments to construct a dictionary of command
parameters that can be used to create a command
Args:
`args`: sequence of arguments
Returns:
Dictionary that can be used in create method
Raises:
ParseError: when the arguments are not correct
"""
parsed = {}
try:
(options, args) = cls.optparser.parse_args(args)
except OptionParsingError as e:
raise ParseError(e.msg, cls.optparser.format_help())
except OptionParsingExit as e:
return None
parsed['label'] = options.label
parsed['can_notify'] = options.can_notify
parsed['name'] = options.name
parsed['tags'] = options.tags
parsed["command_type"] = "HadoopCommand"
parsed['print_logs'] = options.print_logs
if len(args) < 2:
raise ParseError("Need at least two arguments", cls.usage)
subcmd = args.pop(0)
if subcmd not in cls.subcmdlist:
raise ParseError("First argument must be one of <%s>" %
"|".join(cls.subcmdlist))
parsed["sub_command"] = subcmd
parsed["sub_command_args"] = " ".join("'" + a + "'" for a in args)
return parsed
def parse(cls, args):
"""
Parse command line arguments to construct a dictionary of command
parameters that can be used to create a command
Args:
`args`: sequence of arguments
Returns:
Dictionary that can be used in create method
Raises:
ParseError: when the arguments are not correct
"""
try:
(options, args) = cls.optparser.parse_args(args)
if options.latin_statements is None and options.script_location is None:
raise ParseError("One of script or it's location"
" must be specified",
cls.optparser.format_help())
except OptionParsingError as e:
raise ParseError(e.msg, cls.optparser.format_help())
except OptionParsingExit as e:
return None
if options.script_location is not None:
if options.latin_statements is not None:
raise ParseError(
"Both script and script_location cannot be specified",
cls.optparser.format_help())
if ((options.script_location.find("s3://") != 0) and
(options.script_location.find("s3n://") != 0)):
# script location is local file
try:
s = open(options.script_location).read()
except IOError as e:
raise ParseError("Unable to open script location: %s" %
str(e),
cls.optparser.format_help())
options.script_location = None
options.latin_statements = s
if (args is not None) and (len(args) > 0):
if options.latin_statements is not None:
raise ParseError(
"Extra arguments can only be "
"supplied with a script_location in S3 right now",
cls.optparser.format_help())
p = {}
for a in args:
kv = a.split('=')
if len(kv) != 2:
raise ParseError("Arguments to pig script must be of this format k1=v1 k2=v2 k3=v3...")
p[kv[0]] = kv[1]
setattr(options, 'parameters', p)
else:
if (args is not None) and (len(args) > 0):
raise ParseError(
"Extra arguments can only be supplied with a script_location",
cls.optparser.format_help())
v = vars(options)
v["command_type"] = "PigCommand"
return v
def parse(cls, args):
"""
Parse command line arguments to construct a dictionary of command
parameters that can be used to create a command
Args:
`args`: sequence of arguments
Returns:
Dictionary that can be used in create method
Raises:
ParseError: when the arguments are not correct
"""
try:
(options, args) = cls.optparser.parse_args(args)
if options.inline is None and options.script_location is None:
raise ParseError("One of script or it's location"
" must be specified",
cls.optparser.format_help())
except OptionParsingError as e:
raise ParseError(e.msg, cls.optparser.format_help())
except OptionParsingExit as e:
return None
if options.script_location is not None:
if options.inline is not None:
raise ParseError(
"Both script and script_location cannot be specified",
cls.optparser.format_help())
if ((options.script_location.find("s3://") != 0) and
(options.script_location.find("s3n://") != 0)):
# script location is local file
try:
s = open(options.script_location).read()
except IOError as e:
raise ParseError("Unable to open script location: %s" %
str(e),
cls.optparser.format_help())
options.script_location = None
options.inline = s
if (args is not None) and (len(args) > 0):
if options.inline is not None:
raise ParseError(
"Extra arguments can only be "
"supplied with a script_location in S3 right now",
cls.optparser.format_help())
setattr(options, 'parameters',
" ".join([pipes.quote(a) for a in args]))
else:
if (args is not None) and (len(args) > 0):
raise ParseError(
"Extra arguments can only be supplied with a script_location",
cls.optparser.format_help())
v = vars(options)
v["command_type"] = "ShellCommand"
return v