def parse(cls, args):
"""
Parse command line arguments to construct a dictionary of command
parameters that can be used to create a command
Args:
`args`: sequence of arguments
Returns:
Dictionary that can be used in create method
Raises:
ParseError: when the arguments are not correct
"""
try:
(options, args) = cls.optparser.parse_args(args)
if options.mode not in ["1", "2"]:
raise ParseError("mode must be either '1' or '2'",
cls.optparser.format_help())
if (options.dbtap_id is None) or (options.db_table is None):
raise ParseError("dbtap_id and db_table are required",
cls.optparser.format_help())
# TODO: Semantic checks for parameters in mode 1 and 2
except OptionParsingError as e:
raise ParseError(e.msg, cls.optparser.format_help())
except OptionParsingExit as e:
return None
v = vars(options)
v["command_type"] = "DbImportCommand"
return v
def parse(cls, args):
"""
Parse command line arguments to construct a dictionary of command
parameters that can be used to create a command
Args:
`args`: sequence of arguments
Returns:
Dictionary that can be used in create method
Raises:
ParseError: when the arguments are not correct
"""
try:
(options, args) = cls.optparser.parse_args(args)
if (options.db_tap_id is None):
raise ParseError("db_tap_id is required",
cls.optparser.format_help())
if (options.query is None):
raise ParseError("query is required",
cls.optparser.format_help())
except OptionParsingError as e:
raise ParseError(e.msg, cls.optparser.format_help())
except OptionParsingExit as e:
return None
if options.macros is not None:
options.macros = json.loads(options.macros)
v = vars(options)
v["command_type"] = "DbTapQueryCommand"
return v
def parse(cls, args):
"""
Parse command line arguments to construct a dictionary of command
parameters that can be used to create a command
Args:
`args` - sequence of arguments
Returns:
Dictionary that can be used in create method
Raises:
ParseError: when the arguments are not correct
"""
parsed = {}
if len(args) >= 1 and args[0] == "-h":
sys.stderr.write(cls.usage + "\n")
return None
if len(args) < 2:
raise ParseError("Need at least two arguments", cls.usage)
subcmd = args.pop(0)
if subcmd not in cls.subcmdlist:
raise ParseError("First argument must be one of <%s>" %
"|".join(cls.subcmdlist))
parsed["sub_command"] = subcmd
parsed["sub_command_args"] = " ".join("'" + a + "'" for a in args)
return parsed
def parse(cls, args):
"""
Parse command line arguments to construct a dictionary of command
parameters that can be used to create a command
Args:
`args` - sequence of arguments
Returns:
Dictionary that can be used in create method
Raises:
ParseError: when the arguments are not correct
"""
try:
(options, args) = cls.optparser.parse_args(args)
if options.query is None and options.script_location is None:
raise ParseError(
"One of query or script location"
" must be specified", cls.usage)
except OptionParsingError as e:
raise ParseError(e.msg, cls.usage)
except OptionParsingExit as e:
return None
return vars(options)
def parse(cls, args):
"""
Parse command line arguments to construct a dictionary of command
parameters that can be used to create a command
Args:
`args`: sequence of arguments
Returns:
Dictionary that can be used in create method
Raises:
ParseError: when the arguments are not correct
"""
try:
(options, args) = cls.optparser.parse_args(args)
if options.inline is None and options.script_location is None:
raise ParseError(
"One of script or it's location"
" must be specified", cls.optparser.format_help())
except OptionParsingError as e:
raise ParseError(e.msg, cls.optparser.format_help())
except OptionParsingExit as e:
return None
if options.script_location is not None:
if options.inline is not None:
raise ParseError(
"Both script and script_location cannot be specified",
cls.optparser.format_help())
if ((options.script_location.find("s3://") != 0)
and (options.script_location.find("s3n://") != 0)):
# script location is local file
try:
s = open(options.script_location).read()
except IOError, e:
raise ParseError(
"Unable to open script location: %s" % str(e),
cls.optparser.format_help())
options.script_location = None
options.inline = s
if (args is not None) and (len(args) > 0):
if options.inline is not None:
raise ParseError(
"Extra arguments can only be "
"supplied with a script_location in S3 right now",
cls.optparser.format_help())
setattr(options, 'parameters',
" ".join([pipes.quote(a) for a in args]))
def parse(cls, args):
"""
Parse command line arguments to construct a dictionary of command
parameters that can be used to create a command
Args:
`args` - sequence of arguments
Returns:
Dictionary that can be used in create method
Raises:
ParseError: when the arguments are not correct
"""
try:
(options, args) = cls.optparser.parse_args(args)
if options.query is None and options.script_location is None:
raise ParseError("One of query or script location"
" must be specified",
cls.optparser.format_help())
except OptionParsingError as e:
raise ParseError(e.msg, cls.optparser.format_help())
except OptionParsingExit as e:
return None
if options.script_location is not None:
if options.query is not None:
raise ParseError(
"Both query and script_location cannot be specified",
cls.optparser.format_help())
if ((options.script_location.find("s3://") != 0) and
(options.script_location.find("s3n://") != 0)):
# script location is local file
try:
q = open(options.script_location).read()
except:
raise ParseError("Unable to open script location: %s" %
options.script_location,
cls.optparser.format_help())
options.script_location = None
options.query = q
if options.macros is not None:
options.macros = json.loads(options.macros)
return vars(options)
def parse(cls, args):
"""
Parse command line arguments to construct a dictionary of command
parameters that can be used to create a command
Args:
`args`: sequence of arguments
Returns:
Dictionary that can be used in create method
Raises:
ParseError: when the arguments are not correct
"""
try:
(options, args) = cls.optparser.parse_args(args)
if options.mode not in ["1", "2"]:
raise ParseError("mode must be either '1' or '2'",
cls.optparser.format_help())
if (options.dbtap_id is None) or (options.db_table is None):
raise ParseError("dbtap_id and db_table are required",
cls.optparser.format_help())
if options.mode is "1":
if options.hive_table is None:
raise ParseError("hive_table is required for mode 1",
cls.optparser.format_help())
elif options.export_dir is None: # mode 2
raise ParseError("export_dir is required for mode 2",
cls.optparser.format_help())
if options.db_update_mode is not None:
if options.db_update_mode not in ["allowinsert", "updateonly"]:
raise ParseError(
"db_update_mode should either be left blank for append "
"mode or be 'updateonly' or 'allowinsert'",
cls.optparser.format_help())
if options.db_update_mode is "updateonly":
if options.db_update_keys is None:
raise ParseError(
"db_update_keys is required when db_update_mode "
"is 'updateonly'", cls.optparser.format_help())
elif options.db_update_keys is not None:
raise ParseError(
"db_update_keys is used only when db_update_mode "
"is 'updateonly'", cls.optparser.format_help())
except OptionParsingError as e:
raise ParseError(e.msg, cls.optparser.format_help())
except OptionParsingExit as e:
return None
v = vars(options)
v["command_type"] = "DbExportCommand"
return v
def f(item):
item1 = self.process(item)
try:
item2 = tokenlib.update(treatment(item1.acc),item1)
return item2
except ParseError as pe:
nonterminal=msg.setdefault(name,'treat')
raise ParseError(ErrorItem(item=item,nonterminal=nonterminal)) from pe
def find(cls, name="default", **kwargs):
if ((name is None) or (name == "default")):
conn = Qubole.agent()
return cls(conn.get(cls.rest_entity_path))
else:
raise ParseError(
"Bad name 'default'",
"Hadoop Clusters can only be named 'default' currently")
def f(item):
try:
item1 = self.process(item)
acc = item1.acc
its1 = [tokenlib.Item(stream=a,pos=0,acc=None) for a in acc]
acc2 = [(p2 + Parse.finished()).treat(lib.fst).process(it).acc for it in its1]
item3 = tokenlib.update(acc2,item1)
return item3
except ParseError as pe:
raise ParseError(ErrorItem(item=item,nonterminal='reparse_list-'+p2.nonterminal)) from pe
def f(item):
try:
item0 = self.process(item)
acc = item0.acc
if not acc:
return item
item1 = tokenlib.Item(stream=acc,pos=0,acc=None)
item2 = (self + Parse.finished()).treat(lib.fst).process(item1)
item3 = tokenlib.update(item2.acc,item0)
return item3
except ParseError as pe:
raise ParseError(ErrorItem(item=item1,nonterminal='reparse-'+p2.nonterminal)) from pe
def next_item(item: Item) -> Item:
"""Advance to the next item of the stream.
The stream is left unchanged.
NoCatch Exception if accumulated nonempty. Nonempty would be a bug in this program.
"""
if item.pos >= len(item.stream):
raise ParseError(ErrorItem(item=item,
nonterminal='eof')) #StopIteration
if item.acc:
raise ParseNoCatch([ErrorItem(item=item, nonterminal='.')])
return Item(pos=item.pos + 1,
stream=item.stream,
acc=item.stream[item.pos])
def parse(cls, args):
"""
Parse command line arguments to construct a dictionary of command
parameters that can be used to create a command
Args:
`args`: sequence of arguments
Returns:
Dictionary that can be used in create method
Raises:
ParseError: when the arguments are not correct
"""
parsed = {}
try:
(options, args) = cls.optparser.parse_args(args)
except OptionParsingError as e:
raise ParseError(e.msg, cls.optparser.format_help())
except OptionParsingExit as e:
return None
parsed['label'] = options.label
parsed['can_notify'] = options.can_notify
parsed["command_type"] = "HadoopCommand"
if len(args) < 2:
raise ParseError("Need at least two arguments", cls.usage)
subcmd = args.pop(0)
if subcmd not in cls.subcmdlist:
raise ParseError("First argument must be one of <%s>" %
"|".join(cls.subcmdlist))
parsed["sub_command"] = subcmd
parsed["sub_command_args"] = " ".join("'" + a + "'" for a in args)
return parsed
def process(self,item):
# sample mode
if state.mk_sample:
acc = self.sample()
return tokenlib.update(acc,item)
# process mode, the front door.
try:
return self._process(item)
except ParseError as pe:
ei = pe.error_stack
nt = ei.nonterminal
if self.nonterminal or self.production:
nt = f'({nt}; {self})'
#As a last resort try a backdoor, nonterminal as terminal...
#the backdoor should never raise an exception.
if not(tokenlib.eof(item)):
item1 = tokenlib.next_item(item)
if item1.acc.value == self.nonterminal:
acc = Etok.etok(item1.acc)
return tokenlib.update(acc,item1)
raise ParseError(ErrorItem(item=ei.item, nonterminal=nt))
def parse(cls, args):
raise ParseError("dbimport command not implemented yet", "")
def parse(cls, args):
"""
Parse command line arguments to construct a dictionary of command
parameters that can be used to create a command
Args:
`args` - sequence of arguments
Returns:
Dictionary that can be used in create method
Raises:
ParseError: when the arguments are not correct
"""
try:
(options, args) = cls.optparser.parse_args(args)
if options.latin_statements is None and options.script_location is None:
raise ParseError("One of script or it's location"
" must be specified",
cls.optparser.format_help())
except OptionParsingError as e:
raise ParseError(e.msg, cls.optparser.format_help())
except OptionParsingExit as e:
return None
if options.script_location is not None:
if options.latin_statements is not None:
raise ParseError(
"Both script and script_location cannot be specified",
cls.optparser.format_help())
if ((options.script_location.find("s3://") != 0) and
(options.script_location.find("s3n://") != 0)):
# script location is local file
try:
s = open(options.script_location).read()
except:
raise ParseError("Unable to open script location: %s" %
options.script_location,
cls.optparser.format_help())
options.script_location = None
options.latin_statements = s
if ((args is not None) and (len(args) > 0)):
if options.latin_statements is not None:
raise ParseError(
"This sucks - but extra arguments can only be "
"supplied with a script_location in S3 right now",
cls.optparser.format_help())
p = {}
for a in args:
kv = a.split('=')
if len(kv)!=2:
raise ParseError("Arguments to pig script must be of this format k1=v1 k2=v2 k3=v3...")
p[kv[0]] = kv[1]
setattr(options, 'parameters',p)
else:
if ((args is not None) and (len(args) > 0)):
raise ParseError(
"Extra arguments can only be supplied with a script_location",
cls.optparser.format_help())
return vars(options)
def f(item):
if item.pos < len(item.stream):
raise ParseError(ErrorItem(item=item,nonterminal='$'))
return item
def f(item):
raise ParseError(ErrorItem(item=item,nonterminal='fail'))
def f(item):
try:
self.process(item)
return item
except ParseError as pe:
raise ParseError(ErrorItem(item=item,nonterminal='-probe')) from pe
options.inline = s
if (args is not None) and (len(args) > 0):
if options.inline is not None:
raise ParseError(
"Extra arguments can only be "
"supplied with a script_location in S3 right now",
cls.optparser.format_help())
setattr(options, 'parameters',
" ".join([pipes.quote(a) for a in args]))
else:
if (args is not None) and (len(args) > 0):
raise ParseError(
"Extra arguments can only be supplied with a script_location",
cls.optparser.format_help())
v = vars(options)
v["command_type"] = "ShellCommand"
return v
class PigCommand(Command):
usage = ("pigcmd <submit|run> [options] [key1=value1] [key2=value2] ...")
optparser = GentleOptionParser(usage=usage)
optparser.add_option("-s",
"--script",
dest="latin_statements",
help="latin statements that has to be executed")
def parse(cls, args):
raise ParseError("pigcmd not implemented yet", "")
def sample(self):
if not(self._sample):
raise ParseError(ErrorItem(item=None,nonterminal=f'sample not installed: {self.nonterminal}-{self.production}'))
return self._sample()
def f(item):
raise ParseError(ErrorItem(item=item,nonterminal='first-empty'))
def f(item):
item1 = self.process(item)
if p(item1.acc):
return item1
else:
raise ParseError(ErrorItem(item=item,nonterminal=f'{self.nonterminal}-if-{self.production}'))