def create_smv_app(self, smv_args, driver_args):
"""Override this to define how this driver's SmvApp is created
Default is just SmvApp.createInstance(smv_args). Note that it's important to use `createInstance` to ensure that
the singleton app is set.
SmvDriver will parse the full CLI args to distinguish the SMV args from from the args to your driver.
Args:
smv_args (list(str)): CLI args for SMV - should be passed to `SmvApp`)
driver_args (list(str)): CLI args for the driver
"""
spark_builder = SparkSession.builder.enableHiveSupport()
# read the props from kernel config file and use them as spark conf
smvconf = SmvConfig(smv_args)
kernel_conf = smvconf.read_props_from_kernel_config_file()
for key in kernel_conf:
# use the master setting in the config file if exists
if key == 'master':
spark_builder = spark_builder.master(kernel_conf.get(key))
else:
spark_builder = spark_builder.config(key, kernel_conf.get(key))
sparkSession = spark_builder.getOrCreate()
# When SmvDriver is in use, user will call smv-run and interact
# through command-line, so no need to do py module hotload
return SmvApp.createInstance(smv_args,
sparkSession,
py_module_hotload=False)
def create_smv_app(self, smv_args, driver_args):
"""Override this to define how this driver's SmvApp is created
Default is just SmvApp.createInstance(smv_args). Note that it's important to use `createInstance` to ensure that
the singleton app is set.
SmvDriver will parse the full CLI args to distinguish the SMV args from from the args to your driver.
Args:
smv_args (list(str)): CLI args for SMV - should be passed to `SmvApp`)
driver_args (list(str)): CLI args for the driver
"""
spark_builder = SparkSession.builder.enableHiveSupport()
# read the props from kernel config file and use them as spark conf
smvconf = SmvConfig(smv_args)
kernel_conf = smvconf.read_props_from_kernel_config_file()
for key in kernel_conf:
# use the master setting in the config file if exists
if key == 'master':
spark_builder = spark_builder.master(kernel_conf.get(key))
else:
spark_builder = spark_builder.config(key, kernel_conf.get(key))
sparkSession = spark_builder.getOrCreate()
# When SmvDriver is in use, user will call smv-run and interact
# through command-line, so no need to do py module hotload
return SmvApp.createInstance(smv_args, sparkSession, py_module_hotload=False)
def setUpClass(cls):
cls.sparkContext = TestConfig.sparkContext()
cls.sqlContext = TestConfig.sqlContext()
cls.sparkContext.setLogLevel("ERROR")
import random;
callback_server_port = random.randint(20000, 65535)
args = TestConfig.smv_args() + cls.smvAppInitArgs() + ['--cbs-port', str(callback_server_port), '--data-dir', cls.DataDir]
cls.smvApp = SmvApp.createInstance(args, cls.sparkContext, cls.sqlContext)
def __init__(self):
options = self.parseArgs()
# init Smv context
smvApp = SmvApp.createInstance([])
# to reduce complexity in SmvApp, keep the rest server single-threaded
app.run(host=options.ip,
port=int(options.port),
threaded=False,
processes=1)
def __init__(self):
options = self.parseArgs()
# init Smv context
sparkSession = SparkSession.builder.\
enableHiveSupport().\
getOrCreate()
smvApp = SmvApp.createInstance([], sparkSession)
# to reduce complexity in SmvApp, keep the rest server single-threaded
app.run(host=options.ip, port=int(options.port), threaded=False, processes=1)
def create_smv_app(self, smv_args, driver_args):
"""Override this to define how this driver's SmvApp is created
Default is just SmvApp.createInstance(smv_args). Note that it's important to use `createInstance` to ensure that
the singleton app is set.
SmvDriver will parse the full CLI args to distinguish the SMV args from from the args to your driver.
Args:
smv_args (list(str)): CLI args for SMV - should be passed to `SmvApp`)
driver_args (list(str)): CLI args for the driver
"""
return SmvApp.createInstance(smv_args)
def setUp(self):
"""Patch for Python 2.6 without using unittest
"""
cls = self.__class__
if not hasattr(cls, 'smvApp'):
cls.sparkContext = TestConfig.sparkContext()
cls.sqlContext = TestConfig.sqlContext()
cls.sparkContext.setLogLevel("ERROR")
import random;
callback_server_port = random.randint(20000, 65535)
args = TestConfig.smv_args() + cls.smvAppInitArgs() + ['--cbs-port', str(callback_server_port)]
cls.smvApp = SmvApp.createInstance(args, cls.sparkContext, cls.sqlContext)
def __init__(self):
# TODO: should be done by SmvApp (python) automatically.
codePath = os.path.abspath("src/main/python")
sys.path.insert(1, codePath)
# init Smv context
smvApp = SmvApp.createInstance([])
# start server
host = os.environ.get('SMV_HOST', '0.0.0.0')
port = os.environ.get('SMV_PORT', '5000')
project_dir = os.environ.get('PROJECT_DIR', './')
# to reduce complexity in SmvApp, keep the rest server single-threaded
app.run(host=host, port=int(port), threaded=False, processes=1)
def __init__(self):
options = self.parseArgs()
# init Smv context
sparkSession = SparkSession.builder.\
enableHiveSupport().\
getOrCreate()
smvApp = SmvApp.createInstance([], sparkSession)
# to reduce complexity in SmvApp, keep the rest server single-threaded
app.run(host=options.ip,
port=int(options.port),
threaded=False,
processes=1)
def create_smv_app(self, smv_args, driver_args):
"""Override this to define how this driver's SmvApp is created
Default is just SmvApp.createInstance(smv_args). Note that it's important to use `createInstance` to ensure that
the singleton app is set.
SmvDriver will parse the full CLI args to distinguish the SMV args from from the args to your driver.
Args:
smv_args (list(str)): CLI args for SMV - should be passed to `SmvApp`)
driver_args (list(str)): CLI args for the driver
"""
sparkSession = SparkSession.builder.\
enableHiveSupport().\
getOrCreate()
# When SmvDriver is in use, user will call smv-run and interact
# through command-line, so no need to do py module hotload
return SmvApp.createInstance(smv_args,
sparkSession,
py_module_hotload=False)
readline.write_history_file(historyPath)
if os.path.exists(historyPath):
readline.read_history_file(historyPath)
atexit.register(save_history)
# Get a random port for callback server
import random
callback_server_port = random.randint(20000, 65535)
# Import commonly used pyspark lib
from pyspark.sql.functions import *
from pyspark.sql.types import *
sc.setLogLevel("ERROR")
user_args = "".split()
all_args = ['--cbs-port', str(callback_server_port)] + user_args
app = SmvApp.createInstance(all_args, spark)
from smv.smvshell import *
# Import user-defined helpers
app.prepend_source("conf/")
if os.path.exists("conf/smv_shell_app_init.py"):
from smv_shell_app_init import *
del os, atexit, readline, rlcompleter, save_history, historyPath
if os.path.exists(historyPath):
try:
readline.read_history_file(historyPath)
except:
os.remove(historyPath)
print("Unable to read history, deleting history file.")
atexit.register(save_history)
# Import commonly used pyspark lib
import pyspark.sql.functions as F
import pyspark.sql.types as T
sc.setLogLevel("ERROR")
with open(".smv_shell_all_args") as fp:
args = fp.readline()
user_args = args.split()
app = SmvApp.createInstance(user_args, spark)
from smv.smvshell import *
# Import user-defined helpers
app.prepend_source("conf/")
if os.path.exists("conf/smv_shell_app_init.py"):
from smv_shell_app_init import *
del os, atexit, readline, rlcompleter, save_history, historyPath
# # This file is licensed under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from smv import SmvApp import sys app = SmvApp.createInstance(sys.argv[1:]) # skip the first argument, which is this program app.run()
if os.path.exists(historyPath):
try:
readline.read_history_file(historyPath)
except:
os.remove(historyPath)
print("Unable to read history, deleting history file.")
atexit.register(save_history)
# Import commonly used pyspark lib
import pyspark.sql.functions as F
import pyspark.sql.types as T
sc.setLogLevel("ERROR")
with open(".smv_shell_all_args") as fp:
args = fp.readline()
user_args = args.split()
app = SmvApp.createInstance(user_args, spark)
from smv.smvshell import *
# Import user-defined helpers
app.prepend_source("conf/")
if os.path.exists("conf/smv_shell_app_init.py"):
from smv_shell_app_init import *
del os, atexit, readline, rlcompleter, save_history, historyPath
