def create_smv_app(self, smv_args, driver_args): """Override this to define how this driver's SmvApp is created Default is just SmvApp.createInstance(smv_args). Note that it's important to use `createInstance` to ensure that the singleton app is set. SmvDriver will parse the full CLI args to distinguish the SMV args from from the args to your driver. Args: smv_args (list(str)): CLI args for SMV - should be passed to `SmvApp`) driver_args (list(str)): CLI args for the driver """ spark_builder = SparkSession.builder.enableHiveSupport() # read the props from kernel config file and use them as spark conf smvconf = SmvConfig(smv_args) kernel_conf = smvconf.read_props_from_kernel_config_file() for key in kernel_conf: # use the master setting in the config file if exists if key == 'master': spark_builder = spark_builder.master(kernel_conf.get(key)) else: spark_builder = spark_builder.config(key, kernel_conf.get(key)) sparkSession = spark_builder.getOrCreate() # When SmvDriver is in use, user will call smv-run and interact # through command-line, so no need to do py module hotload return SmvApp.createInstance(smv_args, sparkSession, py_module_hotload=False)
def setUpClass(cls): cls.sparkContext = TestConfig.sparkContext() cls.sqlContext = TestConfig.sqlContext() cls.sparkContext.setLogLevel("ERROR") import random; callback_server_port = random.randint(20000, 65535) args = TestConfig.smv_args() + cls.smvAppInitArgs() + ['--cbs-port', str(callback_server_port), '--data-dir', cls.DataDir] cls.smvApp = SmvApp.createInstance(args, cls.sparkContext, cls.sqlContext)
def __init__(self): options = self.parseArgs() # init Smv context smvApp = SmvApp.createInstance([]) # to reduce complexity in SmvApp, keep the rest server single-threaded app.run(host=options.ip, port=int(options.port), threaded=False, processes=1)
def __init__(self): options = self.parseArgs() # init Smv context sparkSession = SparkSession.builder.\ enableHiveSupport().\ getOrCreate() smvApp = SmvApp.createInstance([], sparkSession) # to reduce complexity in SmvApp, keep the rest server single-threaded app.run(host=options.ip, port=int(options.port), threaded=False, processes=1)
def create_smv_app(self, smv_args, driver_args): """Override this to define how this driver's SmvApp is created Default is just SmvApp.createInstance(smv_args). Note that it's important to use `createInstance` to ensure that the singleton app is set. SmvDriver will parse the full CLI args to distinguish the SMV args from from the args to your driver. Args: smv_args (list(str)): CLI args for SMV - should be passed to `SmvApp`) driver_args (list(str)): CLI args for the driver """ return SmvApp.createInstance(smv_args)
def setUp(self): """Patch for Python 2.6 without using unittest """ cls = self.__class__ if not hasattr(cls, 'smvApp'): cls.sparkContext = TestConfig.sparkContext() cls.sqlContext = TestConfig.sqlContext() cls.sparkContext.setLogLevel("ERROR") import random; callback_server_port = random.randint(20000, 65535) args = TestConfig.smv_args() + cls.smvAppInitArgs() + ['--cbs-port', str(callback_server_port)] cls.smvApp = SmvApp.createInstance(args, cls.sparkContext, cls.sqlContext)
def __init__(self): # TODO: should be done by SmvApp (python) automatically. codePath = os.path.abspath("src/main/python") sys.path.insert(1, codePath) # init Smv context smvApp = SmvApp.createInstance([]) # start server host = os.environ.get('SMV_HOST', '0.0.0.0') port = os.environ.get('SMV_PORT', '5000') project_dir = os.environ.get('PROJECT_DIR', './') # to reduce complexity in SmvApp, keep the rest server single-threaded app.run(host=host, port=int(port), threaded=False, processes=1)
def create_smv_app(self, smv_args, driver_args): """Override this to define how this driver's SmvApp is created Default is just SmvApp.createInstance(smv_args). Note that it's important to use `createInstance` to ensure that the singleton app is set. SmvDriver will parse the full CLI args to distinguish the SMV args from from the args to your driver. Args: smv_args (list(str)): CLI args for SMV - should be passed to `SmvApp`) driver_args (list(str)): CLI args for the driver """ sparkSession = SparkSession.builder.\ enableHiveSupport().\ getOrCreate() # When SmvDriver is in use, user will call smv-run and interact # through command-line, so no need to do py module hotload return SmvApp.createInstance(smv_args, sparkSession, py_module_hotload=False)
readline.write_history_file(historyPath) if os.path.exists(historyPath): readline.read_history_file(historyPath) atexit.register(save_history) # Get a random port for callback server import random callback_server_port = random.randint(20000, 65535) # Import commonly used pyspark lib from pyspark.sql.functions import * from pyspark.sql.types import * sc.setLogLevel("ERROR") user_args = "".split() all_args = ['--cbs-port', str(callback_server_port)] + user_args app = SmvApp.createInstance(all_args, spark) from smv.smvshell import * # Import user-defined helpers app.prepend_source("conf/") if os.path.exists("conf/smv_shell_app_init.py"): from smv_shell_app_init import * del os, atexit, readline, rlcompleter, save_history, historyPath
if os.path.exists(historyPath): try: readline.read_history_file(historyPath) except: os.remove(historyPath) print("Unable to read history, deleting history file.") atexit.register(save_history) # Import commonly used pyspark lib import pyspark.sql.functions as F import pyspark.sql.types as T sc.setLogLevel("ERROR") with open(".smv_shell_all_args") as fp: args = fp.readline() user_args = args.split() app = SmvApp.createInstance(user_args, spark) from smv.smvshell import * # Import user-defined helpers app.prepend_source("conf/") if os.path.exists("conf/smv_shell_app_init.py"): from smv_shell_app_init import * del os, atexit, readline, rlcompleter, save_history, historyPath
# # This file is licensed under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from smv import SmvApp import sys app = SmvApp.createInstance(sys.argv[1:]) # skip the first argument, which is this program app.run()