def __init__(self, sc=None, appName="Hail", master=None, local='local[*]',
log='hail.log', quiet=False, append=False, parquet_compression='uncompressed',
block_size=1, branching_factor=50, tmp_dir='/tmp'):
from pyspark import SparkContext
SparkContext._ensure_initialized()
self._gateway = SparkContext._gateway
self._jvm = SparkContext._jvm
Env._jvm = self._jvm
Env._gateway = self._gateway
# hail package
self._hail = getattr(self._jvm, 'is').hail
driver = scala_package_object(self._hail.driver)
if not sc:
self._jsc = driver.configureAndCreateSparkContext(
appName, joption(master), local, parquet_compression, block_size)
self.sc = SparkContext(gateway=self._gateway, jsc=self._jvm.JavaSparkContext(self._jsc))
else:
self.sc = sc
# sc._jsc is a JavaSparkContext
self._jsc = sc._jsc.sc()
driver.configureHail(branching_factor, tmp_dir)
driver.configureLogging(log, quiet, append)
self._jsql_context = driver.createSQLContext(self._jsc)
self._sql_context = SQLContext(self.sc, self._jsql_context)
def export_mongodb(self, mode='append'):
"""Export to MongoDB"""
(scala_package_object(self.hc._hail.driver).exportMongoDB(
self.hc._jsql_context, self._jkt, mode))