def main():
parser = optparse.OptionParser('usage: %prog DATABASE')
parser.add_option("-m", "--make",
action="store_true", dest="make", default=False,
help="Remake the shared object files (calls make).")
parser.add_option("-p", "--put",
action="store_true", dest="put", default=False,
help="Put the the shared objects into HDFS")
parser.add_option("-n", "--noact",
action="store_true", dest="noact", default=False,
help="just print queries, don't execute over impala")
(options, args) = parser.parse_args()
if len(args) < 1:
parser.print_usage()
return
# compile the lib*.so files
if options.make:
doit("make -B all")
# put them into HDFS so impala can load them
if options.put:
for lb, tar in libs:
doit('hadoop fs -rm /user/cloudera/%s' % tar, mayfail=True)
doit('hadoop fs -mkdir -p /user/cloudera')
doit('hadoop fs -put %s /user/cloudera/%s' % (lb, tar))
# register the functions with impala
for q in queries:
print q
if not options.noact:
iutil.impala_shell_exec(queries, args[0])
def main():
parser = optparse.OptionParser('usage: %prog DATABASE')
parser.add_option("-m",
"--make",
action="store_true",
dest="make",
default=False,
help="Remake the shared object files (calls make).")
parser.add_option("-p",
"--put",
action="store_true",
dest="put",
default=False,
help="Put the the shared objects into HDFS")
parser.add_option("-n",
"--noact",
action="store_true",
dest="noact",
default=False,
help="just print queries, don't execute over impala")
parser.add_option("-o",
"--path",
default='/user/cloudera/lib',
help="abs path (dir) on HDFS to put the shared objects")
(options, args) = parser.parse_args()
if len(args) < 1:
parser.print_usage()
return
# compile the lib*.so files
if options.make:
doit("make -B all")
# put them into HDFS so impala can load them
if options.put:
for lb, tar in libs:
doit('hadoop fs -rm %s' % os.path.join(options.path, tar),
mayfail=True)
doit('hadoop fs -mkdir -p %s' % options.path)
doit('hadoop fs -put %s %s' %
(lb, os.path.join(options.path, tar)))
# register the functions with impala
bound_queries = []
for q in queries:
try:
bound_query = q % options.path
bound_queries.append(bound_query)
print bound_query
except TypeError:
bound_queries.append(q)
print q
if not options.noact:
iutil.impala_shell_exec(bound_queries, args[0])
def main():
parser = optparse.OptionParser('usage: %prog DATABASE')
parser.add_option("-m", "--make",
action="store_true", dest="make", default=False,
help="Remake the shared object files (calls make).")
parser.add_option("-p", "--put",
action="store_true", dest="put", default=False,
help="Put the the shared objects into HDFS")
parser.add_option("-n", "--noact",
action="store_true", dest="noact", default=False,
help="just print queries, don't execute over impala")
parser.add_option("-o", "--path", default='/user/cloudera/lib',
help="abs path (dir) on HDFS to put the shared objects")
(options, args) = parser.parse_args()
if len(args) < 1:
parser.print_usage()
return
# compile the lib*.so files
if options.make:
doit("make -B all")
# put them into HDFS so impala can load them
if options.put:
for lb, tar in libs:
doit('hadoop fs -rm %s' % os.path.join(options.path, tar), mayfail=True)
doit('hadoop fs -mkdir -p %s' % options.path)
doit('hadoop fs -put %s %s' % (lb, os.path.join(options.path, tar)))
# register the functions with impala
bound_queries = []
for q in queries:
try:
bound_query = q % options.path
bound_queries.append(bound_query)
print bound_query
except TypeError:
bound_queries.append(q)
print q
if not options.noact:
iutil.impala_shell_exec(bound_queries, args[0])