def main(): #spark_conf = SparkConf().setAppName("Text Preprocesser").set("spark.cores.max", "30") global sc #sc = SparkContext(conf=spark_conf) sc_conf = SparkConf() sc_conf.set("spark.redis.host", config.REDIS_SERVER) sc_conf.set("spark.redis.port", "6379") sc = SparkContext(conf=sc_conf) sc.setLogLevel("ERROR") sc.addFile( os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + "/lib/util.py") sc.addFile( os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + "/config/config.py") global sql_context sql_context = SQLContext(sc) start_time = time.time() preprocess_files(config.S3_BUCKET, config.S3_FOLDER_EXTRACTED) end_time = time.time() print( colored( "Preprocessing run time (seconds): {0}".format(end_time - start_time), "magenta"))
def test_from_conf_with_settings(self): conf = SparkConf() conf.set("spark.cleaner.ttl", "10") conf.setMaster(self.master) conf.setAppName(self.appName) self.ssc = StreamingContext(conf=conf, duration=self.batachDuration) self.assertEqual(int(self.ssc.sparkContext._conf.get("spark.cleaner.ttl")), 10)
def init_spark_session(self, application_name, spark_master=None): """Setup a spark session. :param spark_master: A master parameter used by spark session builder. Use default value (None) to use system environment configured spark cluster. Use 'local[*]' to run on a local box. :return: spark_session: A spark session """ eva_spark_conf = SparkConf() eva_spark_conf.set('spark.logConf', 'true') # enable Arrow optimization for spark Session # This is added to help with to and fro conversion # between pandas and spark dataframe # https://docs.databricks.com/spark/latest/spark-sql/spark-pandas.html eva_spark_conf.set('spark.sql.execution.arrow.pyspark.enabled', 'true') session_builder = SparkSession \ .builder \ .appName(application_name) \ .config(conf=eva_spark_conf) if spark_master: session_builder.master(spark_master) # Gets an existing SparkSession or, # if there is no existing one, creates a new one based # on the options set in this builder. self._session = session_builder.getOrCreate() # Configure logging log4j_level = LoggingManager().getLog4JLevel() spark_context = self._session.sparkContext spark_context.setLogLevel(log4j_level)
def __init__(self): # Setup PySpark. This is needed until PySpark becomes available on PyPI, # after which we can simply add it to requirements.txt. _setup_pyspark() from pyspark.conf import SparkConf from pyspark.context import SparkContext from pyspark.serializers import MarshalSerializer # Create a temporary .zip lib file for Metis, which will be copied over to # Spark workers so they can unpickle Metis functions and objects. metis_lib_file = tempfile.NamedTemporaryFile(suffix='.zip', delete=False) metis_lib_file.close() _copy_lib_for_spark_workers(metis_lib_file.name) # Also ship the Metis lib file so worker nodes can deserialize Metis # internal data structures. conf = SparkConf() conf.setMaster(app.config['SPARK_MASTER']) conf.setAppName('chronology:metis') parallelism = int(app.config.get('SPARK_PARALLELISM', 0)) if parallelism: conf.set('spark.default.parallelism', parallelism) self.context = SparkContext(conf=conf, pyFiles=[metis_lib_file.name], serializer=MarshalSerializer()) # Delete temporary Metis lib file. os.unlink(metis_lib_file.name) # We'll use this to parallelize fetching events in KronosSource. # The default of 8 is from: # https://spark.apache.org/docs/latest/configuration.html self.parallelism = parallelism or 8
def main(): #spark_conf = SparkConf().setAppName("Text Preprocesser").set("spark.cores.max", "30") global sc #sc = SparkContext(conf=spark_conf) sc_conf = SparkConf() sc_conf.set("spark.redis.host", "ec2-52-73-233-196.compute-1.amazonaws.com") sc_conf.set("spark.redis.port", "6379") sc = SparkContext(conf=sc_conf) sc.setLogLevel("ERROR") # sc.addFile(os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + "/lib/util.py") # sc.addFile(os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + "/config/config.py") global sql_context sql_context = SQLContext(sc) start_time = time.time() preprocess_files("dataignition-tech-xml-parq", "posts.parquet") end_time = time.time() print( colored( "Preprocessing run time (seconds): {0}".format(end_time - start_time), "magenta"))
async def get_config(self) -> SparkConf: conf = SparkConf() \ .setAppName(self.app_name) \ .setMaster(self.master_uri) for option, value in SPARK_DEFAULTS.items(): conf.set(option, value) return conf
def __init__(self): # Setup PySpark. This is needed until PySpark becomes available on PyPI, # after which we can simply add it to requirements.txt. _setup_pyspark() from pyspark.conf import SparkConf from pyspark.context import SparkContext from pyspark.serializers import MarshalSerializer # Create a temporary .zip lib file for Metis, which will be copied over to # Spark workers so they can unpickle Metis functions and objects. metis_lib_file = tempfile.NamedTemporaryFile(suffix='.zip', delete=False) metis_lib_file.close() _copy_lib_for_spark_workers(metis_lib_file.name) # Also ship the Metis lib file so worker nodes can deserialize Metis # internal data structures. conf = SparkConf() conf.setMaster(app.config['SPARK_MASTER']) conf.setAppName('chronology:metis') parallelism = int(app.config.get('SPARK_PARALLELISM', 0)) if parallelism: conf.set('spark.default.parallelism', parallelism) self.context = SparkContext(conf=conf, pyFiles=[metis_lib_file.name], serializer=MarshalSerializer()) # Delete temporary Metis lib file. os.unlink(metis_lib_file.name) # We'll use this to parallelize fetching events in KronosSource. # The default of 8 is from: # https://spark.apache.org/docs/latest/configuration.html self.parallelism = parallelism or 8
def init_spark_session(self, application_name, spark_master=None): """Setup a spark session. :param spark_master: A master parameter used by spark session builder. Use default value (None) to use system environment configured spark cluster. Use 'local[*]' to run on a local box. :return: spark_session: A spark session """ eva_spark_conf = SparkConf() pyspark_config = self._config.get_value('pyspark', 'property') for key, value in pyspark_config.items(): eva_spark_conf.set(key, value) session_builder = SparkSession \ .builder \ .appName(application_name) \ .config(conf=eva_spark_conf) if spark_master: session_builder.master(spark_master) # Gets an existing SparkSession or, # if there is no existing one, creates a new one based # on the options set in this builder. self._session = session_builder.getOrCreate() # Configure logging log4j_level = LoggingManager().getLog4JLevel() spark_context = self._session.sparkContext spark_context.setLogLevel(log4j_level)
def StartSpark(name, master, config): def set_environ(): import os, sys os.environ["SPARK_MAJOR_VERSION"] = "2" os.environ["SPARK_HOME"] = "/usr/hdp/current/spark2-client" os.environ["PYLIB"] = os.environ["SPARK_HOME"] + "/python/lib" os.environ["PYSPARK_PYTHON"] = "/opt/miniconda/bin/python" sys.path.insert(0, os.environ["PYLIB"] + "/py4j-0.10.4-src.zip") sys.path.insert(0, os.environ["PYLIB"] + "/pyspark.zip") os.environ[ "PYSPARK_SUBMIT_ARGS"] = "--master yarn-client --driver-memory 16G --jars \ /u/pesoft/hadoop/repos/scala/release/com/micron/pesoft/tteq/2.1.0/tteq-2.1.0.jar \ pyspark-shell" from pyspark.sql import SparkSession from pyspark.conf import SparkConf from pyspark.context import SparkContext set_environ() conf = SparkConf() conf.setMaster(master) for option in config.keys(): conf.set(option, config[option].replace(" ", "")) spark = SparkSession.builder\ .enableHiveSupport()\ .appName(name)\ .config(conf=conf)\ .getOrCreate() return spark
def getOrCreate(self): """Gets an existing :class:`SparkSession` or, if there is no existing one, creates a new one based on the options set in this builder. This method first checks whether there is a valid global default SparkSession, and if yes, return that one. If no valid global default SparkSession exists, the method creates a new SparkSession and assigns the newly created SparkSession as the global default. >>> s1 = SparkSession.builder.config("k1", "v1").getOrCreate() >>> s1.conf.get("k1") == "v1" True In case an existing SparkSession is returned, the config options specified in this builder will be applied to the existing SparkSession. >>> s2 = SparkSession.builder.config("k2", "v2").getOrCreate() >>> s1.conf.get("k1") == s2.conf.get("k1") True >>> s1.conf.get("k2") == s2.conf.get("k2") True """ with self._lock: from pyspark.context import SparkContext from pyspark.conf import SparkConf session = SparkSession._instantiatedContext if session is None: sparkConf = SparkConf() for key, value in self._options.items(): sparkConf.set(key, value) sc = SparkContext.getOrCreate(sparkConf) session = SparkSession(sc) for key, value in self._options.items(): session.conf.set(key, value) return session
def _test_broadcast_on_driver(self, *extra_confs): conf = SparkConf() for key, value in extra_confs: conf.set(key, value) conf.setMaster("local-cluster[2,1,1024]") self.sc = SparkContext(conf=conf) bs = self.sc.broadcast(value=5) self.assertEqual(5, bs.value)
def deserialize(self, config: dict): conf = SparkConf() for option, value in config.items(): conf.set(option, value) # set driver host to our local ip conf.set('spark.driver.host', get_local_ip()) return conf
def bootstrap(): conf = SparkConf() conf.set('spark.sql.shuffle.partitions', NUM_PARTITION) conf.set("spark.hadoop.mapred.output.compress", "true") conf.set("spark.hadoop.mapred.output.compression.codec", "true") conf.set("spark.hadoop.mapred.output.compression.codec", "org.apache.hadoop.io.compress.GzipCodec") conf.set("spark.hadoop.mapred.output.compression.type", "BLOCK") conf.set("spark.sql.broadcastTimeout", "1200") return conf
def _test_multiple_broadcasts(self, *extra_confs): """ Test broadcast variables make it OK to the executors. Tests multiple broadcast variables, and also multiple jobs. """ conf = SparkConf() for key, value in extra_confs: conf.set(key, value) conf.setMaster("local-cluster[2,1,1024]") self.sc = SparkContext(conf=conf) self._test_encryption_helper([5]) self._test_encryption_helper([5, 10, 20])
def spark_init(): conf = SparkConf() conf.set("spark.Master", "local[*]") sc = SparkContext(appName='SearchMusic', conf=conf) ss = SparkSession.builder\ .appName('SearchMusic')\ .master('local[*]')\ .getOrCreate() print(sc._conf.getAll()) return sc, ss
def getOrCreate(self): """Gets an existing :class:`SparkSession` or, if there is no existing one, creates a new one based on the options set in this builder. """ with self._lock: from pyspark.conf import SparkConf from pyspark.context import SparkContext from pyspark.sql.context import SQLContext sparkConf = SparkConf() for key, value in self._options.items(): sparkConf.set(key, value) sparkContext = SparkContext.getOrCreate(sparkConf) return SQLContext.getOrCreate(sparkContext).sparkSession
def getOrCreate(self): """Gets an existing :class:`SparkSession` or, if there is no existing one, creates a new one based on the options set in this builder. """ with self._lock: from pyspark.conf import SparkConf from pyspark.context import SparkContext from pyspark.sql.context import SQLContext sparkConf = SparkConf() for key, value in self._options.items(): sparkConf.set(key, value) sparkContext = SparkContext.getOrCreate(sparkConf) return SQLContext.getOrCreate(sparkContext).sparkSession
def data_to_res(list_job): #blob connection storage_account_name = "*******" storage_account_access_key = "**********" file_location = "**********" file_type = "csv" conf = SparkConf() conf.set( "fs.azure.account.key." + storage_account_name + ".blob.core.windows.net", storage_account_access_key) print('connected to blob, following files will checked for processing: ', list_job) #spark initialization sc = SparkContext(conf=conf) sql_sc = SQLContext(sc) for new in list_job: try: print('dealing with {}'.format(new)) spark = SparkSession.builder.master('local').appName( 'test').getOrCreate() df_name = '*********/{}.csv'.format(new) df = spark.read.csv(df_name, header='true') df.createOrReplaceTempView("table1") mean_amount = df.select(avg("amount")) mean_name = "********".format(new) mean_amount.coalesce(1).write.option('header', 'true').csv(mean_name) print('mean calculated') country_mean = df.groupBy('country_sender').agg({'amount': 'mean'}) country_m_name = "*********".format(new) country_mean.coalesce(1).write.option('header', 'true').csv(country_m_name) print('mean by country calculated') country_sum = df.groupBy('country_sender').agg({'amount': 'sum'}) country_s_name = "*********".format(new) country_sum.coalesce(1).write.option('header', 'true').csv(country_s_name) print('sum by country calculated') except: print('not able to process', new) return ('Finished to process new data')
def spark_session(): conf = SparkConf() conf.set("spark.master", "local[1]") conf.set("spark.app.name", "tests_helper") conf.set("spark.sql.shuffle.partitions", "3") conf.set("spark.sql.streaming.schemaInference ", "true") yield SparkSession.builder.config(conf=conf).getOrCreate() shutil.rmtree(TEST_DIR, ignore_errors=True)
def default_spark_config(cores: int, memory_per_executor: int, driver_overhead: int = 2000, tmp_dir: str = '', extra_java_options: str = '', enable_arrow: bool = True, use_utc: bool = False) -> SparkConf: ''' Constructs SparkConf object with sensible defaults for the ratschlab environment See also https://spark.apache.org/docs/latest/configuration.html for more information about the semantics of the configurations :param cores: number of executors (workers) :param memory_per_executor: memory per executor [MB] :param driver_overhead: Memory to allocate for the driver [MB], excluding exector memory :param tmp_dir: "scratch" space for spark, typically /tmp by default :param extra_java_options: extra parameters for the JVM :param enable_arrow: see https://spark.apache.org/docs/2.3.0/sql-programming-guide.html#pyspark-usage-guide-for-pandas-with-apache-arrow , requires pyarrow to be installed :return: SparkConf instance ''' driver_mem = cores * memory_per_executor + driver_overhead jars = [ str(p) for p in Path(Path(__file__).parent.parent, "jars").glob('*.jar') ] jar_paths = ':'.join(jars) cfg = SparkConf() if tmp_dir: cfg.set("spark.local.dir", tmp_dir) java_options = str(extra_java_options) if use_utc: # avoiding trouble with JDBC and timestamps java_options = "-Duser.timezone=UTC " + java_options return (cfg.set("spark.driver.memory", "{}m".format(driver_mem)).set( "spark.executor.memory", "{}m".format(memory_per_executor)).set( "spark.driver.extraJavaOptions", java_options).set("spark.master", "local[{}]".format(cores)).set( "spark.jars", jar_paths).set("spark.sql.execution.arrow.enabled", str(enable_arrow)))
def getOrCreate(self): """Gets an existing :class:`SparkSession` or, if there is no existing one, creates a new one based on the options set in this builder. This method first checks whether there is a valid global default SparkSession, and if yes, return that one. If no valid global default SparkSession exists, the method creates a new SparkSession and assigns the newly created SparkSession as the global default. >>> s1 = SparkSession.builder.config("k1", "v1").getOrCreate() >>> s1.conf.get("k1") == s1.sparkContext.getConf().get("k1") == "v1" True In case an existing SparkSession is returned, the config options specified in this builder will be applied to the existing SparkSession. >>> s2 = SparkSession.builder.config("k2", "v2").getOrCreate() >>> s1.conf.get("k1") == s2.conf.get("k1") True >>> s1.conf.get("k2") == s2.conf.get("k2") True """ with self._lock: from pyspark.context import SparkContext from pyspark.conf import SparkConf session = SparkSession._instantiatedSession if session is None or session._sc._jsc is None: if self._sc is not None: sc = self._sc else: sparkConf = SparkConf() for key, value in self._options.items(): sparkConf.set(key, value) sc = SparkContext.getOrCreate(sparkConf) # This SparkContext may be an existing one. for key, value in self._options.items(): # we need to propagate the confs # before we create the SparkSession. Otherwise, confs like # warehouse path and metastore url will not be set correctly ( # these confs cannot be changed once the SparkSession is created). sc._conf.set(key, value) session = SparkSession(sc) for key, value in self._options.items(): session._jsparkSession.sessionState().conf().setConfString(key, value) for key, value in self._options.items(): session.sparkContext._conf.set(key, value) return session
def getOrCreate(self): """Gets an existing :class:`SparkSession` or, if there is no existing one, creates a new one based on the options set in this builder. .. versionadded:: 2.0.0 Examples -------- This method first checks whether there is a valid global default SparkSession, and if yes, return that one. If no valid global default SparkSession exists, the method creates a new SparkSession and assigns the newly created SparkSession as the global default. >>> s1 = SparkSession.builder.config("k1", "v1").getOrCreate() >>> s1.conf.get("k1") == "v1" True In case an existing SparkSession is returned, the config options specified in this builder will be applied to the existing SparkSession. >>> s2 = SparkSession.builder.config("k2", "v2").getOrCreate() >>> s1.conf.get("k1") == s2.conf.get("k1") True >>> s1.conf.get("k2") == s2.conf.get("k2") True """ with self._lock: from pyspark.context import SparkContext from pyspark.conf import SparkConf session = SparkSession._instantiatedSession if session is None or session._sc._jsc is None: if self._sc is not None: sc = self._sc else: sparkConf = SparkConf() for key, value in self._options.items(): sparkConf.set(key, value) # This SparkContext may be an existing one. sc = SparkContext.getOrCreate(sparkConf) # Do not update `SparkConf` for existing `SparkContext`, as it's shared # by all sessions. session = SparkSession(sc) for key, value in self._options.items(): session._jsparkSession.sessionState().conf().setConfString( key, value) return session
def getOrCreate(self): """Gets an existing :class:`SparkSession` or, if there is no existing one, creates a new one based on the options set in this builder. This method first checks whether there is a valid global default SparkSession, and if yes, return that one. If no valid global default SparkSession exists, the method creates a new SparkSession and assigns the newly created SparkSession as the global default. >>> s1 = SparkSession.builder.config("k1", "v1").getOrCreate() >>> s1.conf.get("k1") == s1.sparkContext.getConf().get("k1") == "v1" True In case an existing SparkSession is returned, the config options specified in this builder will be applied to the existing SparkSession. >>> s2 = SparkSession.builder.config("k2", "v2").getOrCreate() >>> s1.conf.get("k1") == s2.conf.get("k1") True >>> s1.conf.get("k2") == s2.conf.get("k2") True """ with self._lock: from pyspark.context import SparkContext from pyspark.conf import SparkConf session = SparkSession._instantiatedContext if session is None: sparkConf = SparkConf() for key, value in self._options.items(): sparkConf.set(key, value) sc = SparkContext.getOrCreate(sparkConf) # This SparkContext may be an existing one. for key, value in self._options.items(): # we need to propagate the confs # before we create the SparkSession. Otherwise, confs like # warehouse path and metastore url will not be set correctly ( # these confs cannot be changed once the SparkSession is created). sc._conf.set(key, value) session = SparkSession(sc) for key, value in self._options.items(): session.conf.set(key, value) for key, value in self._options.items(): session.sparkContext._conf.set(key, value) return session
def getOrCreate(self): """Gets an existing :class:`SparkSession` or, if there is no existing one, creates a new one based on the options set in this builder. This method first checks whether there is a valid thread-local SparkSession, and if yes, return that one. It then checks whether there is a valid global default SparkSession, and if yes, return that one. If no valid global default SparkSession exists, the method creates a new SparkSession and assigns the newly created SparkSession as the global default. In case an existing SparkSession is returned, the config options specified in this builder will be applied to the existing SparkSession. """ with self._lock: from pyspark.conf import SparkConf from pyspark.context import SparkContext from pyspark.sql.context import SQLContext sparkConf = SparkConf() for key, value in self._options.items(): sparkConf.set(key, value) sparkContext = SparkContext.getOrCreate(sparkConf) return SQLContext.getOrCreate(sparkContext).sparkSession
def getOrCreate(self): """Gets an existing :class:`SparkSession` or, if there is no existing one, creates a new one based on the options set in this builder. This method first checks whether there is a valid thread-local SparkSession, and if yes, return that one. It then checks whether there is a valid global default SparkSession, and if yes, return that one. If no valid global default SparkSession exists, the method creates a new SparkSession and assigns the newly created SparkSession as the global default. In case an existing SparkSession is returned, the config options specified in this builder will be applied to the existing SparkSession. """ with self._lock: from pyspark.conf import SparkConf from pyspark.context import SparkContext from pyspark.sql.context import SQLContext sparkConf = SparkConf() for key, value in self._options.items(): sparkConf.set(key, value) sparkContext = SparkContext.getOrCreate(sparkConf) return SQLContext.getOrCreate(sparkContext).sparkSession
def spark(): conf = SparkConf() conf.set("spark.master", "local[1]") conf.set("spark.app.name", "tests_helper") conf.set("spark.sql.shuffle.partitions", "3") if not os.path.exists(TEST_DIR): os.mkdir(TEST_DIR) yield SparkSession.builder.config(conf=conf).getOrCreate() shutil.rmtree(TEST_DIR, ignore_errors=True)
def getOrCreate(self): """Gets an existing :class:`SparkSession` or, if there is no existing one, creates a new one based on the options set in this builder. This method first checks whether there is a valid global default SparkSession, and if yes, return that one. If no valid global default SparkSession exists, the method creates a new SparkSession and assigns the newly created SparkSession as the global default. >>> s1 = SparkSession.builder.config("k1", "v1").getOrCreate() >>> s1.conf.get("k1") == s1.sparkContext.getConf().get("k1") == "v1" True In case an existing SparkSession is returned, the config options specified in this builder will be applied to the existing SparkSession. >>> s2 = SparkSession.builder.config("k2", "v2").getOrCreate() >>> s1.conf.get("k1") == s2.conf.get("k1") True >>> s1.conf.get("k2") == s2.conf.get("k2") True """ with self._lock: from pyspark.context import SparkContext from pyspark.conf import SparkConf session = SparkSession._instantiatedContext if session is None: sparkConf = SparkConf() for key, value in self._options.items(): sparkConf.set(key, value) sc = SparkContext.getOrCreate(sparkConf) session = SparkSession(sc) for key, value in self._options.items(): session.conf.set(key, value) for key, value in self._options.items(): session.sparkContext._conf.set(key, value) return session
def main(): number_threads = "16" conf = SparkConf() conf.setMaster("local[" + number_threads + "]") conf.setAppName("image_ingest_application") conf.set("spark.executor.memory", "1g") conf.set("spark.connection.timeout", "10000s") conf.set("spark.network.timeout", "600s") conf.set("spark.files.fetchTimeout", "500s") #conf.set("spark.ui.port", "4045") ### #conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer") ### #conf.set("spark.cores.max", 200) sc = SparkContext(conf=conf) print "[Starting Spark Job]" start = (sys.argv[1]).strip() end = (sys.argv[2]).strip() print 'start:end::', start, "::", end start_time = time.time() print 'start_time: ', start_time start_end = start + ":" + end image_dir = start + 'images' + end hdfs_path = '/hive/catint.db/temp2_ds_images_spark3/' hdfs_dir = hdfs_path + image_dir command = 'hadoop fs -mkdir ' + hdfs_dir execShellCmd(command) rawfile = 'asset_urls_5m.txt' assetUrlsList = getAssetUrlsListFile(rawfile) start_index = int(start) end_index = int(end) assetUrlsList_chunk = assetUrlsList[start_index:end_index] print 'len(assetUrlsList_chunk): ', len(assetUrlsList_chunk) rdd1 = sc.parallelize(assetUrlsList_chunk) rdd2 = (rdd1.map(lambda x: setup_proxy(x)).map( lambda x: fetchImagelist(x, start_end, hdfs_path, number_threads))) rdd2.collect() end_time = time.time() print 'end_time: ', end_time total_execution_time = (end_time - start_time) / 60 print 'total execution time/mins: ', total_execution_time print "[Spark Job Completed]"
def _get_or_create_context(): global _sc if not _sc: from pyspark.conf import SparkConf from pyspark.context import SparkContext settings = GlobalPreferences['spark_configuration'] SparkContext.setSystemProperty("spark.executor.memory",settings['executor-memory']) SparkContext.setSystemProperty("spark.driver.memory",settings['driver-memory']) conf = SparkConf() conf = conf.setMaster(settings['master']) conf = conf.set('spark.driver.maxResultSize',settings['driver-memory']) _sc = SparkContext.getOrCreate(conf=conf) _sc.setLogLevel("WARN") return _sc
def main(): conf = SparkConf() conf.setMaster("local") conf.setAppName("My_sp_application") conf.set("spark.executor.memory", "4g") conf.set("spark.connection.timeout", "10000s") conf.set("spark.network.timeout", "600s") conf.set("spark.files.fetchTimeout", "500s") # conf.set("spark.cores.max", 200) sc = SparkContext(conf=conf) # sc = SparkContext("local", "Product Attributes Changed") print "[Starting Spark Job]" # sc.setCheckpointDir("hive/catint.db/temp2_checkpoint/") start = (sys.argv[1]).strip() end = (sys.argv[2]).strip() print 'start:end::', start, "::", end start_end = start + ":" + end image_dir = start + 'images' + end hdfs_dir = '/hive/catint.db/temp2_ds_images_spark3/' + image_dir command = 'hadoop fs -mkdir ' + hdfs_dir execShellCmd(command) rawfile = 'asset_urls_5m.txt' #rawfile = 'asset_urls_1k.txt' assetUrlsList = getAssetUrlsListFile(rawfile) start_index = int(start) end_index = int(end) assetUrlsList_chunk = assetUrlsList[start_index:end_index] print 'len(assetUrlsList_chunk): ', len(assetUrlsList_chunk) rdd1 = sc.parallelize(assetUrlsList_chunk) #rdd1 = getFullDataSet(sc) # rdd2 = (rdd1.repartition(10).map(lambda x: setup_proxy(x)).repartition( 10).map(lambda x: fetchImagelist(x, start_end)).reduce( lambda x: saveImagelist(x))) rdd2.collect() print "[Spark Job Completed]"
# import Libraries from pyspark.conf import SparkConf from pyspark.sql import SparkSession from pyspark.sql.types import * # init sparkConf conf = SparkConf() conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer") \ .set("spark.executor.cores", "1") \ .set("spark.executor.memory", "1G") # Initialize Spark Session spark = SparkSession.builder.appName("021-mds_libelle_traduction").config( conf=SparkConf()).enableHiveSupport().getOrCreate() # Data path path = "s3://decathlon-ods/ods/mds_libelle_traduction/*.gz" # From gz files on S3 to Spark Dataframe df = spark.read.option("header", "false").option("delimiter", "|").csv(path) # write to hdfs df.select("_c1", "_c4", "_c0", "_c2") \ .withColumn("lib_num_libelle_lib", df["_c1"].cast(IntegerType())) \ .withColumn("lit_libelle_long", df["_c4"].cast(StringType())) \ .withColumn("tlb_typ_libelle_lib", df["_c0"].cast(StringType())) \ .withColumn("lan_code_langue_lan", df["_c2"].cast(StringType())) \ .select("lib_num_libelle_lib", "lit_libelle_long", "tlb_typ_libelle_lib", "lan_code_langue_lan") \ .repartition(80).write.option("compression", "snappy").mode("overwrite").format("parquet").saveAsTable( "kylin_usb_mqb.mds_libelle_traduction")
config = json.load(f) def getData(spark, user, password, database, qryStr): jdbcDF = spark.read.format('jdbc')\ .option('url', f"jdbc:sqlserver://localhost:1433;databaseName={database}")\ .option('driver', 'com.microsoft.sqlserver.jdbc.SQLServerDriver')\ .option('dbtable', qryStr)\ .option("user", user) \ .option("password", password) \ .load() return jdbcDF conf = SparkConf().setMaster("local").setAppName(config['db']['database']) conf.set('spark.driver.extraClassPath', config['db']['jdbcDrivePath']) conf.set('spark.executor.extraClassPath', config['db']['jdbcDrivePath']) sc = SparkContext(conf=conf) sc = SparkContext.getOrCreate() sqlContext = SQLContext(sc) sqlContext.sql(f"CREATE DATABASE IF NOT EXISTS {config['db']['database']}") sqlContext.sql(f"USE {config['db']['database']}") spark = sqlContext.sparkSession qryStrAll = """ ( SELECT * FROM todasBolsas ) t"""
'/home/dan/Desktop/IMN432-CW01/Word_Freq/', '/home/dan/Desktop/IMN432-CW01/IDF/', '/home/dan/Desktop/IMN432-CW01/IDF/IDF-Pairs', '/home/dan/Desktop/IMN432-CW01/IDF', '/home/dan/Desktop/IMN432-CW01/TF_IDF/TF_IDF_File', '/home/dan/Desktop/IMN432-CW01/processXML/', '/home/dan/Desktop/IMN432-CW01/meta/', '/home/dan/Desktop/IMN432-CW01/TF_IDF', '/home/dan/Desktop/IMN432-CW01/processXML/Subject', '/home/dan/Spark_Files/Books/stopwords_en.txt'] allFiles = getFileList(directory[0]) # Find the Number of Files in the Directory numFiles = len(allFiles) # Create Spark Job Name and Configuration Settings config = SparkConf().setMaster("local[*]") config.set("spark.executor.memory", "5g") sc = SparkContext(conf=config, appName="ACKF415-Coursework-1") # Create a File Details List N = numFiles fileEbook = [] print('################################################') print('###### Process Files > Word Freq to Pickle #####\n') # Start Timer WordFreq_Time = time() # Pickled Word Frequencies pickleWordF = getFileList(directory[2]) # Ascertain if Section has already been completed if len(pickleWordF) < 1: print 'Creating Work Freq Pickles and RDDs \n' # Import the Stop and Save as a List stop_words = sc.textFile(
class SparkContext(object): """ Main entry point for Spark functionality. A SparkContext represents the connection to a Spark cluster, and can be used to create L{RDD} and broadcast variables on that cluster. """ _gateway = None _jvm = None _next_accum_id = 0 _active_spark_context = None _lock = RLock() _python_includes = None # zip and egg files that need to be added to PYTHONPATH PACKAGE_EXTENSIONS = ('.zip', '.egg', '.jar') def __init__(self, master=None, appName=None, sparkHome=None, pyFiles=None, environment=None, batchSize=0, serializer=PickleSerializer(), conf=None, gateway=None, jsc=None, profiler_cls=BasicProfiler): """ Create a new SparkContext. At least the master and app name should be set, either through the named parameters here or through C{conf}. :param master: Cluster URL to connect to (e.g. mesos://host:port, spark://host:port, local[4]). :param appName: A name for your job, to display on the cluster web UI. :param sparkHome: Location where Spark is installed on cluster nodes. :param pyFiles: Collection of .zip or .py files to send to the cluster and add to PYTHONPATH. These can be paths on the local file system or HDFS, HTTP, HTTPS, or FTP URLs. :param environment: A dictionary of environment variables to set on worker nodes. :param batchSize: The number of Python objects represented as a single Java object. Set 1 to disable batching, 0 to automatically choose the batch size based on object sizes, or -1 to use an unlimited batch size :param serializer: The serializer for RDDs. :param conf: A L{SparkConf} object setting Spark properties. :param gateway: Use an existing gateway and JVM, otherwise a new JVM will be instantiated. :param jsc: The JavaSparkContext instance (optional). :param profiler_cls: A class of custom Profiler used to do profiling (default is pyspark.profiler.BasicProfiler). >>> from pyspark.context import SparkContext >>> sc = SparkContext('local', 'test') >>> sc2 = SparkContext('local', 'test2') # doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): ... ValueError:... """ self._callsite = first_spark_call() or CallSite(None, None, None) if gateway is not None and gateway.gateway_parameters.auth_token is None: allow_insecure_env = os.environ.get("PYSPARK_ALLOW_INSECURE_GATEWAY", "0") if allow_insecure_env == "1" or allow_insecure_env.lower() == "true": warnings.warn( "You are passing in an insecure Py4j gateway. This " "presents a security risk, and will be completely forbidden in Spark 3.0") else: raise ValueError( "You are trying to pass an insecure Py4j gateway to Spark. This" " presents a security risk. If you are sure you understand and accept this" " risk, you can set the environment variable" " 'PYSPARK_ALLOW_INSECURE_GATEWAY=1', but" " note this option will be removed in Spark 3.0") SparkContext._ensure_initialized(self, gateway=gateway, conf=conf) try: self._do_init(master, appName, sparkHome, pyFiles, environment, batchSize, serializer, conf, jsc, profiler_cls) except: # If an error occurs, clean up in order to allow future SparkContext creation: self.stop() raise def _do_init(self, master, appName, sparkHome, pyFiles, environment, batchSize, serializer, conf, jsc, profiler_cls): self.environment = environment or {} # java gateway must have been launched at this point. if conf is not None and conf._jconf is not None: # conf has been initialized in JVM properly, so use conf directly. This represent the # scenario that JVM has been launched before SparkConf is created (e.g. SparkContext is # created and then stopped, and we create a new SparkConf and new SparkContext again) self._conf = conf else: self._conf = SparkConf(_jvm=SparkContext._jvm) if conf is not None: for k, v in conf.getAll(): self._conf.set(k, v) self._batchSize = batchSize # -1 represents an unlimited batch size self._unbatched_serializer = serializer if batchSize == 0: self.serializer = AutoBatchedSerializer(self._unbatched_serializer) else: self.serializer = BatchedSerializer(self._unbatched_serializer, batchSize) # Set any parameters passed directly to us on the conf if master: self._conf.setMaster(master) if appName: self._conf.setAppName(appName) if sparkHome: self._conf.setSparkHome(sparkHome) if environment: for key, value in environment.items(): self._conf.setExecutorEnv(key, value) for key, value in DEFAULT_CONFIGS.items(): self._conf.setIfMissing(key, value) # Check that we have at least the required parameters if not self._conf.contains("spark.master"): raise Exception("A master URL must be set in your configuration") if not self._conf.contains("spark.app.name"): raise Exception("An application name must be set in your configuration") # Read back our properties from the conf in case we loaded some of them from # the classpath or an external config file self.master = self._conf.get("spark.master") self.appName = self._conf.get("spark.app.name") self.sparkHome = self._conf.get("spark.home", None) for (k, v) in self._conf.getAll(): if k.startswith("spark.executorEnv."): varName = k[len("spark.executorEnv."):] self.environment[varName] = v self.environment["PYTHONHASHSEED"] = os.environ.get("PYTHONHASHSEED", "0") # Create the Java SparkContext through Py4J self._jsc = jsc or self._initialize_context(self._conf._jconf) # Reset the SparkConf to the one actually used by the SparkContext in JVM. self._conf = SparkConf(_jconf=self._jsc.sc().conf()) # Create a single Accumulator in Java that we'll send all our updates through; # they will be passed back to us through a TCP server auth_token = self._gateway.gateway_parameters.auth_token self._accumulatorServer = accumulators._start_update_server(auth_token) (host, port) = self._accumulatorServer.server_address self._javaAccumulator = self._jvm.PythonAccumulatorV2(host, port, auth_token) self._jsc.sc().register(self._javaAccumulator) # If encryption is enabled, we need to setup a server in the jvm to read broadcast # data via a socket. # scala's mangled names w/ $ in them require special treatment. self._encryption_enabled = self._jvm.PythonUtils.getEncryptionEnabled(self._jsc) self.pythonExec = os.environ.get("PYSPARK_PYTHON", 'python') self.pythonVer = "%d.%d" % sys.version_info[:2] # Broadcast's __reduce__ method stores Broadcast instances here. # This allows other code to determine which Broadcast instances have # been pickled, so it can determine which Java broadcast objects to # send. self._pickled_broadcast_vars = BroadcastPickleRegistry() SparkFiles._sc = self root_dir = SparkFiles.getRootDirectory() sys.path.insert(1, root_dir) # Deploy any code dependencies specified in the constructor self._python_includes = list() for path in (pyFiles or []): self.addPyFile(path) # Deploy code dependencies set by spark-submit; these will already have been added # with SparkContext.addFile, so we just need to add them to the PYTHONPATH for path in self._conf.get("spark.submit.pyFiles", "").split(","): if path != "": (dirname, filename) = os.path.split(path) try: filepath = os.path.join(SparkFiles.getRootDirectory(), filename) if not os.path.exists(filepath): # In case of YARN with shell mode, 'spark.submit.pyFiles' files are # not added via SparkContext.addFile. Here we check if the file exists, # try to copy and then add it to the path. See SPARK-21945. shutil.copyfile(path, filepath) if filename[-4:].lower() in self.PACKAGE_EXTENSIONS: self._python_includes.append(filename) sys.path.insert(1, filepath) except Exception: warnings.warn( "Failed to add file [%s] speficied in 'spark.submit.pyFiles' to " "Python path:\n %s" % (path, "\n ".join(sys.path)), RuntimeWarning) # Create a temporary directory inside spark.local.dir: local_dir = self._jvm.org.apache.spark.util.Utils.getLocalDir(self._jsc.sc().conf()) self._temp_dir = \ self._jvm.org.apache.spark.util.Utils.createTempDir(local_dir, "pyspark") \ .getAbsolutePath() # profiling stats collected for each PythonRDD if self._conf.get("spark.python.profile", "false") == "true": dump_path = self._conf.get("spark.python.profile.dump", None) self.profiler_collector = ProfilerCollector(profiler_cls, dump_path) else: self.profiler_collector = None # create a signal handler which would be invoked on receiving SIGINT def signal_handler(signal, frame): self.cancelAllJobs() raise KeyboardInterrupt() # see http://stackoverflow.com/questions/23206787/ if isinstance(threading.current_thread(), threading._MainThread): signal.signal(signal.SIGINT, signal_handler) def __repr__(self): return "<SparkContext master={master} appName={appName}>".format( master=self.master, appName=self.appName, ) def _repr_html_(self): return """ <div> <p><b>SparkContext</b></p> <p><a href="{sc.uiWebUrl}">Spark UI</a></p> <dl> <dt>Version</dt> <dd><code>v{sc.version}</code></dd> <dt>Master</dt> <dd><code>{sc.master}</code></dd> <dt>AppName</dt> <dd><code>{sc.appName}</code></dd> </dl> </div> """.format( sc=self ) def _initialize_context(self, jconf): """ Initialize SparkContext in function to allow subclass specific initialization """ return self._jvm.JavaSparkContext(jconf) @classmethod def _ensure_initialized(cls, instance=None, gateway=None, conf=None): """ Checks whether a SparkContext is initialized or not. Throws error if a SparkContext is already running. """ with SparkContext._lock: if not SparkContext._gateway: SparkContext._gateway = gateway or launch_gateway(conf) SparkContext._jvm = SparkContext._gateway.jvm if instance: if (SparkContext._active_spark_context and SparkContext._active_spark_context != instance): currentMaster = SparkContext._active_spark_context.master currentAppName = SparkContext._active_spark_context.appName callsite = SparkContext._active_spark_context._callsite # Raise error if there is already a running Spark context raise ValueError( "Cannot run multiple SparkContexts at once; " "existing SparkContext(app=%s, master=%s)" " created by %s at %s:%s " % (currentAppName, currentMaster, callsite.function, callsite.file, callsite.linenum)) else: SparkContext._active_spark_context = instance def __getnewargs__(self): # This method is called when attempting to pickle SparkContext, which is always an error: raise Exception( "It appears that you are attempting to reference SparkContext from a broadcast " "variable, action, or transformation. SparkContext can only be used on the driver, " "not in code that it run on workers. For more information, see SPARK-5063." ) def __enter__(self): """ Enable 'with SparkContext(...) as sc: app(sc)' syntax. """ return self def __exit__(self, type, value, trace): """ Enable 'with SparkContext(...) as sc: app' syntax. Specifically stop the context on exit of the with block. """ self.stop() @classmethod def getOrCreate(cls, conf=None): """ Get or instantiate a SparkContext and register it as a singleton object. :param conf: SparkConf (optional) """ with SparkContext._lock: if SparkContext._active_spark_context is None: SparkContext(conf=conf or SparkConf()) return SparkContext._active_spark_context def setLogLevel(self, logLevel): """ Control our logLevel. This overrides any user-defined log settings. Valid log levels include: ALL, DEBUG, ERROR, FATAL, INFO, OFF, TRACE, WARN """ self._jsc.setLogLevel(logLevel) @classmethod def setSystemProperty(cls, key, value): """ Set a Java system property, such as spark.executor.memory. This must must be invoked before instantiating SparkContext. """ SparkContext._ensure_initialized() SparkContext._jvm.java.lang.System.setProperty(key, value) @property def version(self): """ The version of Spark on which this application is running. """ return self._jsc.version() @property @ignore_unicode_prefix def applicationId(self): """ A unique identifier for the Spark application. Its format depends on the scheduler implementation. * in case of local spark app something like 'local-1433865536131' * in case of YARN something like 'application_1433865536131_34483' >>> sc.applicationId # doctest: +ELLIPSIS u'local-...' """ return self._jsc.sc().applicationId() @property def uiWebUrl(self): """Return the URL of the SparkUI instance started by this SparkContext""" return self._jsc.sc().uiWebUrl().get() @property def startTime(self): """Return the epoch time when the Spark Context was started.""" return self._jsc.startTime() @property def defaultParallelism(self): """ Default level of parallelism to use when not given by user (e.g. for reduce tasks) """ return self._jsc.sc().defaultParallelism() @property def defaultMinPartitions(self): """ Default min number of partitions for Hadoop RDDs when not given by user """ return self._jsc.sc().defaultMinPartitions() def stop(self): """ Shut down the SparkContext. """ if getattr(self, "_jsc", None): try: self._jsc.stop() except Py4JError: # Case: SPARK-18523 warnings.warn( 'Unable to cleanly shutdown Spark JVM process.' ' It is possible that the process has crashed,' ' been killed or may also be in a zombie state.', RuntimeWarning ) pass finally: self._jsc = None if getattr(self, "_accumulatorServer", None): self._accumulatorServer.shutdown() self._accumulatorServer = None with SparkContext._lock: SparkContext._active_spark_context = None def emptyRDD(self): """ Create an RDD that has no partitions or elements. """ return RDD(self._jsc.emptyRDD(), self, NoOpSerializer()) def range(self, start, end=None, step=1, numSlices=None): """ Create a new RDD of int containing elements from `start` to `end` (exclusive), increased by `step` every element. Can be called the same way as python's built-in range() function. If called with a single argument, the argument is interpreted as `end`, and `start` is set to 0. :param start: the start value :param end: the end value (exclusive) :param step: the incremental step (default: 1) :param numSlices: the number of partitions of the new RDD :return: An RDD of int >>> sc.range(5).collect() [0, 1, 2, 3, 4] >>> sc.range(2, 4).collect() [2, 3] >>> sc.range(1, 7, 2).collect() [1, 3, 5] """ if end is None: end = start start = 0 return self.parallelize(xrange(start, end, step), numSlices) def parallelize(self, c, numSlices=None): """ Distribute a local Python collection to form an RDD. Using xrange is recommended if the input represents a range for performance. >>> sc.parallelize([0, 2, 3, 4, 6], 5).glom().collect() [[0], [2], [3], [4], [6]] >>> sc.parallelize(xrange(0, 6, 2), 5).glom().collect() [[], [0], [], [2], [4]] """ numSlices = int(numSlices) if numSlices is not None else self.defaultParallelism if isinstance(c, xrange): size = len(c) if size == 0: return self.parallelize([], numSlices) step = c[1] - c[0] if size > 1 else 1 start0 = c[0] def getStart(split): return start0 + int((split * size / numSlices)) * step def f(split, iterator): return xrange(getStart(split), getStart(split + 1), step) return self.parallelize([], numSlices).mapPartitionsWithIndex(f) # Make sure we distribute data evenly if it's smaller than self.batchSize if "__len__" not in dir(c): c = list(c) # Make it a list so we can compute its length batchSize = max(1, min(len(c) // numSlices, self._batchSize or 1024)) serializer = BatchedSerializer(self._unbatched_serializer, batchSize) jrdd = self._serialize_to_jvm(c, numSlices, serializer) return RDD(jrdd, self, serializer) def _serialize_to_jvm(self, data, parallelism, serializer): """ Using py4j to send a large dataset to the jvm is really slow, so we use either a file or a socket if we have encryption enabled. """ if self._encryption_enabled: # with encryption, we open a server in java and send the data directly server = self._jvm.PythonParallelizeServer(self._jsc.sc(), parallelism) (sock_file, _) = local_connect_and_auth(server.port(), server.secret()) chunked_out = ChunkedStream(sock_file, 8192) serializer.dump_stream(data, chunked_out) chunked_out.close() # this call will block until the server has read all the data and processed it (or # throws an exception) return server.getResult() else: # without encryption, we serialize to a file, and we read the file in java and # parallelize from there. tempFile = NamedTemporaryFile(delete=False, dir=self._temp_dir) try: serializer.dump_stream(data, tempFile) tempFile.close() readRDDFromFile = self._jvm.PythonRDD.readRDDFromFile return readRDDFromFile(self._jsc, tempFile.name, parallelism) finally: # we eagerly read the file so we can delete right after. os.unlink(tempFile.name) def pickleFile(self, name, minPartitions=None): """ Load an RDD previously saved using L{RDD.saveAsPickleFile} method. >>> tmpFile = NamedTemporaryFile(delete=True) >>> tmpFile.close() >>> sc.parallelize(range(10)).saveAsPickleFile(tmpFile.name, 5) >>> sorted(sc.pickleFile(tmpFile.name, 3).collect()) [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] """ minPartitions = minPartitions or self.defaultMinPartitions return RDD(self._jsc.objectFile(name, minPartitions), self) @ignore_unicode_prefix def textFile(self, name, minPartitions=None, use_unicode=True): """ Read a text file from HDFS, a local file system (available on all nodes), or any Hadoop-supported file system URI, and return it as an RDD of Strings. If use_unicode is False, the strings will be kept as `str` (encoding as `utf-8`), which is faster and smaller than unicode. (Added in Spark 1.2) >>> path = os.path.join(tempdir, "sample-text.txt") >>> with open(path, "w") as testFile: ... _ = testFile.write("Hello world!") >>> textFile = sc.textFile(path) >>> textFile.collect() [u'Hello world!'] """ minPartitions = minPartitions or min(self.defaultParallelism, 2) return RDD(self._jsc.textFile(name, minPartitions), self, UTF8Deserializer(use_unicode)) @ignore_unicode_prefix def wholeTextFiles(self, path, minPartitions=None, use_unicode=True): """ Read a directory of text files from HDFS, a local file system (available on all nodes), or any Hadoop-supported file system URI. Each file is read as a single record and returned in a key-value pair, where the key is the path of each file, the value is the content of each file. If use_unicode is False, the strings will be kept as `str` (encoding as `utf-8`), which is faster and smaller than unicode. (Added in Spark 1.2) For example, if you have the following files:: hdfs://a-hdfs-path/part-00000 hdfs://a-hdfs-path/part-00001 ... hdfs://a-hdfs-path/part-nnnnn Do C{rdd = sparkContext.wholeTextFiles("hdfs://a-hdfs-path")}, then C{rdd} contains:: (a-hdfs-path/part-00000, its content) (a-hdfs-path/part-00001, its content) ... (a-hdfs-path/part-nnnnn, its content) .. note:: Small files are preferred, as each file will be loaded fully in memory. >>> dirPath = os.path.join(tempdir, "files") >>> os.mkdir(dirPath) >>> with open(os.path.join(dirPath, "1.txt"), "w") as file1: ... _ = file1.write("1") >>> with open(os.path.join(dirPath, "2.txt"), "w") as file2: ... _ = file2.write("2") >>> textFiles = sc.wholeTextFiles(dirPath) >>> sorted(textFiles.collect()) [(u'.../1.txt', u'1'), (u'.../2.txt', u'2')] """ minPartitions = minPartitions or self.defaultMinPartitions return RDD(self._jsc.wholeTextFiles(path, minPartitions), self, PairDeserializer(UTF8Deserializer(use_unicode), UTF8Deserializer(use_unicode))) def binaryFiles(self, path, minPartitions=None): """ .. note:: Experimental Read a directory of binary files from HDFS, a local file system (available on all nodes), or any Hadoop-supported file system URI as a byte array. Each file is read as a single record and returned in a key-value pair, where the key is the path of each file, the value is the content of each file. .. note:: Small files are preferred, large file is also allowable, but may cause bad performance. """ minPartitions = minPartitions or self.defaultMinPartitions return RDD(self._jsc.binaryFiles(path, minPartitions), self, PairDeserializer(UTF8Deserializer(), NoOpSerializer())) def binaryRecords(self, path, recordLength): """ .. note:: Experimental Load data from a flat binary file, assuming each record is a set of numbers with the specified numerical format (see ByteBuffer), and the number of bytes per record is constant. :param path: Directory to the input data files :param recordLength: The length at which to split the records """ return RDD(self._jsc.binaryRecords(path, recordLength), self, NoOpSerializer()) def _dictToJavaMap(self, d): jm = self._jvm.java.util.HashMap() if not d: d = {} for k, v in d.items(): jm[k] = v return jm def sequenceFile(self, path, keyClass=None, valueClass=None, keyConverter=None, valueConverter=None, minSplits=None, batchSize=0): """ Read a Hadoop SequenceFile with arbitrary key and value Writable class from HDFS, a local file system (available on all nodes), or any Hadoop-supported file system URI. The mechanism is as follows: 1. A Java RDD is created from the SequenceFile or other InputFormat, and the key and value Writable classes 2. Serialization is attempted via Pyrolite pickling 3. If this fails, the fallback is to call 'toString' on each key and value 4. C{PickleSerializer} is used to deserialize pickled objects on the Python side :param path: path to sequncefile :param keyClass: fully qualified classname of key Writable class (e.g. "org.apache.hadoop.io.Text") :param valueClass: fully qualified classname of value Writable class (e.g. "org.apache.hadoop.io.LongWritable") :param keyConverter: :param valueConverter: :param minSplits: minimum splits in dataset (default min(2, sc.defaultParallelism)) :param batchSize: The number of Python objects represented as a single Java object. (default 0, choose batchSize automatically) """ minSplits = minSplits or min(self.defaultParallelism, 2) jrdd = self._jvm.PythonRDD.sequenceFile(self._jsc, path, keyClass, valueClass, keyConverter, valueConverter, minSplits, batchSize) return RDD(jrdd, self) def newAPIHadoopFile(self, path, inputFormatClass, keyClass, valueClass, keyConverter=None, valueConverter=None, conf=None, batchSize=0): """ Read a 'new API' Hadoop InputFormat with arbitrary key and value class from HDFS, a local file system (available on all nodes), or any Hadoop-supported file system URI. The mechanism is the same as for sc.sequenceFile. A Hadoop configuration can be passed in as a Python dict. This will be converted into a Configuration in Java :param path: path to Hadoop file :param inputFormatClass: fully qualified classname of Hadoop InputFormat (e.g. "org.apache.hadoop.mapreduce.lib.input.TextInputFormat") :param keyClass: fully qualified classname of key Writable class (e.g. "org.apache.hadoop.io.Text") :param valueClass: fully qualified classname of value Writable class (e.g. "org.apache.hadoop.io.LongWritable") :param keyConverter: (None by default) :param valueConverter: (None by default) :param conf: Hadoop configuration, passed in as a dict (None by default) :param batchSize: The number of Python objects represented as a single Java object. (default 0, choose batchSize automatically) """ jconf = self._dictToJavaMap(conf) jrdd = self._jvm.PythonRDD.newAPIHadoopFile(self._jsc, path, inputFormatClass, keyClass, valueClass, keyConverter, valueConverter, jconf, batchSize) return RDD(jrdd, self) def newAPIHadoopRDD(self, inputFormatClass, keyClass, valueClass, keyConverter=None, valueConverter=None, conf=None, batchSize=0): """ Read a 'new API' Hadoop InputFormat with arbitrary key and value class, from an arbitrary Hadoop configuration, which is passed in as a Python dict. This will be converted into a Configuration in Java. The mechanism is the same as for sc.sequenceFile. :param inputFormatClass: fully qualified classname of Hadoop InputFormat (e.g. "org.apache.hadoop.mapreduce.lib.input.TextInputFormat") :param keyClass: fully qualified classname of key Writable class (e.g. "org.apache.hadoop.io.Text") :param valueClass: fully qualified classname of value Writable class (e.g. "org.apache.hadoop.io.LongWritable") :param keyConverter: (None by default) :param valueConverter: (None by default) :param conf: Hadoop configuration, passed in as a dict (None by default) :param batchSize: The number of Python objects represented as a single Java object. (default 0, choose batchSize automatically) """ jconf = self._dictToJavaMap(conf) jrdd = self._jvm.PythonRDD.newAPIHadoopRDD(self._jsc, inputFormatClass, keyClass, valueClass, keyConverter, valueConverter, jconf, batchSize) return RDD(jrdd, self) def hadoopFile(self, path, inputFormatClass, keyClass, valueClass, keyConverter=None, valueConverter=None, conf=None, batchSize=0): """ Read an 'old' Hadoop InputFormat with arbitrary key and value class from HDFS, a local file system (available on all nodes), or any Hadoop-supported file system URI. The mechanism is the same as for sc.sequenceFile. A Hadoop configuration can be passed in as a Python dict. This will be converted into a Configuration in Java. :param path: path to Hadoop file :param inputFormatClass: fully qualified classname of Hadoop InputFormat (e.g. "org.apache.hadoop.mapred.TextInputFormat") :param keyClass: fully qualified classname of key Writable class (e.g. "org.apache.hadoop.io.Text") :param valueClass: fully qualified classname of value Writable class (e.g. "org.apache.hadoop.io.LongWritable") :param keyConverter: (None by default) :param valueConverter: (None by default) :param conf: Hadoop configuration, passed in as a dict (None by default) :param batchSize: The number of Python objects represented as a single Java object. (default 0, choose batchSize automatically) """ jconf = self._dictToJavaMap(conf) jrdd = self._jvm.PythonRDD.hadoopFile(self._jsc, path, inputFormatClass, keyClass, valueClass, keyConverter, valueConverter, jconf, batchSize) return RDD(jrdd, self) def hadoopRDD(self, inputFormatClass, keyClass, valueClass, keyConverter=None, valueConverter=None, conf=None, batchSize=0): """ Read an 'old' Hadoop InputFormat with arbitrary key and value class, from an arbitrary Hadoop configuration, which is passed in as a Python dict. This will be converted into a Configuration in Java. The mechanism is the same as for sc.sequenceFile. :param inputFormatClass: fully qualified classname of Hadoop InputFormat (e.g. "org.apache.hadoop.mapred.TextInputFormat") :param keyClass: fully qualified classname of key Writable class (e.g. "org.apache.hadoop.io.Text") :param valueClass: fully qualified classname of value Writable class (e.g. "org.apache.hadoop.io.LongWritable") :param keyConverter: (None by default) :param valueConverter: (None by default) :param conf: Hadoop configuration, passed in as a dict (None by default) :param batchSize: The number of Python objects represented as a single Java object. (default 0, choose batchSize automatically) """ jconf = self._dictToJavaMap(conf) jrdd = self._jvm.PythonRDD.hadoopRDD(self._jsc, inputFormatClass, keyClass, valueClass, keyConverter, valueConverter, jconf, batchSize) return RDD(jrdd, self) def _checkpointFile(self, name, input_deserializer): jrdd = self._jsc.checkpointFile(name) return RDD(jrdd, self, input_deserializer) @ignore_unicode_prefix def union(self, rdds): """ Build the union of a list of RDDs. This supports unions() of RDDs with different serialized formats, although this forces them to be reserialized using the default serializer: >>> path = os.path.join(tempdir, "union-text.txt") >>> with open(path, "w") as testFile: ... _ = testFile.write("Hello") >>> textFile = sc.textFile(path) >>> textFile.collect() [u'Hello'] >>> parallelized = sc.parallelize(["World!"]) >>> sorted(sc.union([textFile, parallelized]).collect()) [u'Hello', 'World!'] """ first_jrdd_deserializer = rdds[0]._jrdd_deserializer if any(x._jrdd_deserializer != first_jrdd_deserializer for x in rdds): rdds = [x._reserialize() for x in rdds] first = rdds[0]._jrdd rest = [x._jrdd for x in rdds[1:]] return RDD(self._jsc.union(first, rest), self, rdds[0]._jrdd_deserializer) def broadcast(self, value): """ Broadcast a read-only variable to the cluster, returning a L{Broadcast<pyspark.broadcast.Broadcast>} object for reading it in distributed functions. The variable will be sent to each cluster only once. """ return Broadcast(self, value, self._pickled_broadcast_vars) def accumulator(self, value, accum_param=None): """ Create an L{Accumulator} with the given initial value, using a given L{AccumulatorParam} helper object to define how to add values of the data type if provided. Default AccumulatorParams are used for integers and floating-point numbers if you do not provide one. For other types, a custom AccumulatorParam can be used. """ if accum_param is None: if isinstance(value, int): accum_param = accumulators.INT_ACCUMULATOR_PARAM elif isinstance(value, float): accum_param = accumulators.FLOAT_ACCUMULATOR_PARAM elif isinstance(value, complex): accum_param = accumulators.COMPLEX_ACCUMULATOR_PARAM else: raise TypeError("No default accumulator param for type %s" % type(value)) SparkContext._next_accum_id += 1 return Accumulator(SparkContext._next_accum_id - 1, value, accum_param) def addFile(self, path, recursive=False): """ Add a file to be downloaded with this Spark job on every node. The C{path} passed can be either a local file, a file in HDFS (or other Hadoop-supported filesystems), or an HTTP, HTTPS or FTP URI. To access the file in Spark jobs, use L{SparkFiles.get(fileName)<pyspark.files.SparkFiles.get>} with the filename to find its download location. A directory can be given if the recursive option is set to True. Currently directories are only supported for Hadoop-supported filesystems. >>> from pyspark import SparkFiles >>> path = os.path.join(tempdir, "test.txt") >>> with open(path, "w") as testFile: ... _ = testFile.write("100") >>> sc.addFile(path) >>> def func(iterator): ... with open(SparkFiles.get("test.txt")) as testFile: ... fileVal = int(testFile.readline()) ... return [x * fileVal for x in iterator] >>> sc.parallelize([1, 2, 3, 4]).mapPartitions(func).collect() [100, 200, 300, 400] """ self._jsc.sc().addFile(path, recursive) def addPyFile(self, path): """ Add a .py or .zip dependency for all tasks to be executed on this SparkContext in the future. The C{path} passed can be either a local file, a file in HDFS (or other Hadoop-supported filesystems), or an HTTP, HTTPS or FTP URI. """ self.addFile(path) (dirname, filename) = os.path.split(path) # dirname may be directory or HDFS/S3 prefix if filename[-4:].lower() in self.PACKAGE_EXTENSIONS: self._python_includes.append(filename) # for tests in local mode sys.path.insert(1, os.path.join(SparkFiles.getRootDirectory(), filename)) if sys.version > '3': import importlib importlib.invalidate_caches() def setCheckpointDir(self, dirName): """ Set the directory under which RDDs are going to be checkpointed. The directory must be a HDFS path if running on a cluster. """ self._jsc.sc().setCheckpointDir(dirName) def _getJavaStorageLevel(self, storageLevel): """ Returns a Java StorageLevel based on a pyspark.StorageLevel. """ if not isinstance(storageLevel, StorageLevel): raise Exception("storageLevel must be of type pyspark.StorageLevel") newStorageLevel = self._jvm.org.apache.spark.storage.StorageLevel return newStorageLevel(storageLevel.useDisk, storageLevel.useMemory, storageLevel.useOffHeap, storageLevel.deserialized, storageLevel.replication) def setJobGroup(self, groupId, description, interruptOnCancel=False): """ Assigns a group ID to all the jobs started by this thread until the group ID is set to a different value or cleared. Often, a unit of execution in an application consists of multiple Spark actions or jobs. Application programmers can use this method to group all those jobs together and give a group description. Once set, the Spark web UI will associate such jobs with this group. The application can use L{SparkContext.cancelJobGroup} to cancel all running jobs in this group. >>> import threading >>> from time import sleep >>> result = "Not Set" >>> lock = threading.Lock() >>> def map_func(x): ... sleep(100) ... raise Exception("Task should have been cancelled") >>> def start_job(x): ... global result ... try: ... sc.setJobGroup("job_to_cancel", "some description") ... result = sc.parallelize(range(x)).map(map_func).collect() ... except Exception as e: ... result = "Cancelled" ... lock.release() >>> def stop_job(): ... sleep(5) ... sc.cancelJobGroup("job_to_cancel") >>> supress = lock.acquire() >>> supress = threading.Thread(target=start_job, args=(10,)).start() >>> supress = threading.Thread(target=stop_job).start() >>> supress = lock.acquire() >>> print(result) Cancelled If interruptOnCancel is set to true for the job group, then job cancellation will result in Thread.interrupt() being called on the job's executor threads. This is useful to help ensure that the tasks are actually stopped in a timely manner, but is off by default due to HDFS-1208, where HDFS may respond to Thread.interrupt() by marking nodes as dead. """ self._jsc.setJobGroup(groupId, description, interruptOnCancel) def setLocalProperty(self, key, value): """ Set a local property that affects jobs submitted from this thread, such as the Spark fair scheduler pool. """ self._jsc.setLocalProperty(key, value) def getLocalProperty(self, key): """ Get a local property set in this thread, or null if it is missing. See L{setLocalProperty} """ return self._jsc.getLocalProperty(key) def setJobDescription(self, value): """ Set a human readable description of the current job. """ self._jsc.setJobDescription(value) def sparkUser(self): """ Get SPARK_USER for user who is running SparkContext. """ return self._jsc.sc().sparkUser() def cancelJobGroup(self, groupId): """ Cancel active jobs for the specified group. See L{SparkContext.setJobGroup} for more information. """ self._jsc.sc().cancelJobGroup(groupId) def cancelAllJobs(self): """ Cancel all jobs that have been scheduled or are running. """ self._jsc.sc().cancelAllJobs() def statusTracker(self): """ Return :class:`StatusTracker` object """ return StatusTracker(self._jsc.statusTracker()) def runJob(self, rdd, partitionFunc, partitions=None, allowLocal=False): """ Executes the given partitionFunc on the specified set of partitions, returning the result as an array of elements. If 'partitions' is not specified, this will run over all partitions. >>> myRDD = sc.parallelize(range(6), 3) >>> sc.runJob(myRDD, lambda part: [x * x for x in part]) [0, 1, 4, 9, 16, 25] >>> myRDD = sc.parallelize(range(6), 3) >>> sc.runJob(myRDD, lambda part: [x * x for x in part], [0, 2], True) [0, 1, 16, 25] """ if partitions is None: partitions = range(rdd._jrdd.partitions().size()) # Implementation note: This is implemented as a mapPartitions followed # by runJob() in order to avoid having to pass a Python lambda into # SparkContext#runJob. mappedRDD = rdd.mapPartitions(partitionFunc) sock_info = self._jvm.PythonRDD.runJob(self._jsc.sc(), mappedRDD._jrdd, partitions) return list(_load_from_socket(sock_info, mappedRDD._jrdd_deserializer)) def show_profiles(self): """ Print the profile stats to stdout """ if self.profiler_collector is not None: self.profiler_collector.show_profiles() else: raise RuntimeError("'spark.python.profile' configuration must be set " "to 'true' to enable Python profile.") def dump_profiles(self, path): """ Dump the profile stats into directory `path` """ if self.profiler_collector is not None: self.profiler_collector.dump_profiles(path) else: raise RuntimeError("'spark.python.profile' configuration must be set " "to 'true' to enable Python profile.") def getConf(self): conf = SparkConf() conf.setAll(self._conf.getAll()) return conf
def get_spark_session(): conf = SparkConf() conf.setMaster('yarn-client') conf.set("spark.yarn.am.cores", 7) conf.set("spark.executor.memory", "70g") conf.set("spark.executor.instances", 20) conf.set("spark.executor.cores", 15) conf.set("spark.python.worker.memory", "2g") conf.set("spark.default.parallelism", 3000) conf.set("spark.sql.shuffle.partitions", 3000) conf.set("spark.broadcast.blockSize", 1024) conf.set("spark.shuffle.file.buffer", '512k') conf.set("spark.speculation", True) conf.set("spark.speculation.quantile", 0.98) conf.set("spark.submit.pyFiles", WTYH_DEDUCTION_COMPANY_LIST) spark = SparkSession \ .builder \ .appName("hgongjing2_one_prd_common_static_v2") \ .config(conf=conf) \ .enableHiveSupport() \ .getOrCreate() return spark
def test_existing_spark_context_with_settings(self): conf = SparkConf() conf.set("spark.cleaner.ttl", "10") self.sc = SparkContext(master=self.master, appName=self.appName, conf=conf) self.ssc = StreamingContext(sparkContext=self.sc, duration=self.batachDuration) self.assertEqual(int(self.ssc.sparkContext._conf.get("spark.cleaner.ttl")), 10)
''' Created on Oct 30, 2015 @author: dyerke ''' from pyspark.context import SparkContext from pyspark.conf import SparkConf if __name__ == '__main__': m_hostname= "dyerke-Inspiron-7537" # conf= SparkConf() conf.setAppName("MyTestApp") conf.setMaster("spark://" + m_hostname + ":7077") conf.setSparkHome("/usr/local/spark") conf.set("spark.driver.host", m_hostname) logFile = "/usr/local/spark/README.md" # Should be some file on your system # sc= SparkContext(conf=conf) logData= sc.textFile(logFile).cache() # countAs= logData.filter(lambda x: 'a' in x).count() countBs= logData.filter(lambda x: 'b' in x).count() # print("Lines with a: %i, lines with b: %i" % (countAs, countBs)) sc.stop()
# this is the deprecated equivalent of ADD_JARS add_files = None if os.environ.get("ADD_FILES") is not None: add_files = os.environ.get("ADD_FILES").split(",") if os.environ.get("SPARK_EXECUTOR_URI"): SparkContext.setSystemProperty("spark.executor.uri", os.environ["SPARK_EXECUTOR_URI"]) # setup mesos-based connection conf = SparkConf().setMaster(os.environ["SPARK_MASTER"]) # optionally set memory limits if os.environ.get("SPARK_RAM_DRIVER"): conf.set("spark.driver.memory", os.environ["SPARK_RAM_DRIVER"]) if os.environ.get("SPARK_RAM_WORKER"): conf.set("spark.executor_memory", os.environ["SPARK_RAM_WORKER"]) # set the UI port conf.set("spark.ui.port", ui_get_available_port()) # optionally set the Spark binary if os.environ.get("SPARK_BINARY"): conf.set("spark.executor.uri", os.environ["SPARK_BINARY"]) # establish config-based context sc = SparkContext(appName="DockerIPythonShell", pyFiles=add_files, conf=conf) atexit.register(lambda: sc.stop()) try:
def main(): # 解析配置 app_id = int(sys.argv[1]) master = sys.argv[2] app_name = sys.argv[3] # 应用配置 assert APP_CONFIG.get(app_id) is not None, \ '[myapp streaming_app_main.main()] configuration error invalid APP_CONFIG with app.id = ' + str(app_id) app_conf = map_conf_properties(APP_CONFIG.get(app_id), 'app.id')[app_id] spark_home = app_conf['sparkHome'] pyFiles = app_conf['pyFiles.list'] di_id = app_conf.get('app.interfaceId') # 数据接口配置 di_in_conf_with_ds_conf = get_di_conf_with_ds_conf( di_id, DATAINTERFACE_CONFIG, DATASOURCE_CONFIG, di_key='interface.id', di_ds_key='interface.sourceId', ds_key='source.id', merge_key_name='interface.id' )[di_id] print('= = ' * 20, type(di_in_conf_with_ds_conf), 'di_in_conf_with_ds_conf = ') pprint(di_in_conf_with_ds_conf) schema_conf_string = di_in_conf_with_ds_conf['schema'] struct_type = generate_df_schmea(schema_conf_string) # schema_field_list = [x.name for x in struct_type.fields] di_in_conf_with_ds_conf['struct.type'] = struct_type # di_in_conf_with_ds_conf['struct.field.list'] = schema_field_list di_out_confs = [kv for kv in DATAINTERFACE_CONFIG.iteritems() if kv[1].get('interface.type', '') == 'output'] print('= = ' * 20, type(di_out_confs), 'di_out_confs = ') pprint(di_out_confs) di_out_confs_with_ds_conf = list_dict_merge( [get_di_conf_with_ds_conf( kv[0], DATAINTERFACE_CONFIG, DATASOURCE_CONFIG, di_key='interface.id', di_ds_key='interface.sourceId', ds_key='source.id', merge_key_name='interface.id') for kv in DATAINTERFACE_CONFIG.iteritems() if kv[1].get('interface.type', '') == 'output'] ) print('= = ' * 20, type(di_out_confs_with_ds_conf), 'di_out_confs_with_ds_conf = ') pprint(di_out_confs_with_ds_conf) # 外部缓存配置 cache_confs_with_ds_conf = list_dict_merge( [get_di_conf_with_ds_conf( kv[0], CACHE_CONFIG, DATASOURCE_CONFIG, di_key='cache.id', di_ds_key='cache.sourceId', ds_key='source.id', merge_key_name='cache.id') for kv in CACHE_CONFIG.iteritems()] ) print('= = ' * 20, type(cache_confs_with_ds_conf), 'cache_confs_with_ds_conf = ') pprint(cache_confs_with_ds_conf) # 指定输入接口准备阶段的配置 # 准备阶段配置中有效步骤的配置 # Note: 对 dict 进行 filter,传给function的参数是 dict 的 key prepares_config_active = PREPARES_CONFIG[di_id] \ if PREPARES_CONFIG.get(di_id, {}).get('prepares.enabled', False) else {} # print('= = ' * 20, type(prepares_config_active), 'prepares_config_active = ') # pprint(prepares_config_active) # TODO: 2中方法的结果==测试False, 删除注释 # prepares_config_active_steps = filter( # lambda step_conf: step_conf[1].get('step.enabled', False), # map(lambda step_conf: (step_conf[0], map_conf_properties(step_conf[1])), # prepares_config_active.get('steps', {}).iteritems() # ) # ) prepares_config_active_steps = \ [(k, map_conf_properties(v)) for k, v in prepares_config_active.get('steps', {}).iteritems() if v.get('step.enabled', False)] print('= = ' * 20, type(prepares_config_active_steps), 'prepares_config_active_steps = ') pprint(prepares_config_active_steps) # 指定输入接口计算阶段的配置 # filter 之后变成 list,list 的每个元素是 tuple(computeStatistics.id, computeStatistics.conf_dict) computes_config_active = COMPUTES_CONFIG[di_id] \ if COMPUTES_CONFIG.get(di_id, {}).get('computeStatistics.enabled', False) else {} # list[{computeStatistic.id: {conf}}, ...] # # TODO: 2中方法的结果==测试False, 删除注释 # compute_computeStatistics_config_active = filter( # lambda computeStatistic_conf: computeStatistic_conf[1].get('computeStatistic.enabled', False), # computes_config_active.get('computeStatistics', {}).iteritems()) compute_computeStatistics_config_active = [ kv for kv in computes_config_active.get('computeStatistics', {}).iteritems() if kv[1].get('computeStatistic.enabled', False)] print('= = ' * 20, type(compute_computeStatistics_config_active), 'compute_computeStatistics_config_active = ') pprint(compute_computeStatistics_config_active) # {computeStatistic.id -> list[step_conf_tuple]}, 其中 step_conf_tuple = (step_id, step_conf_dict) compute_prepares_config_active = dict(map( lambda computeStatistic_conf: (computeStatistic_conf[0], sorted(list_dict_merge( map(lambda step_conf: map_conf_properties(step_conf[1], 'step.id'), filter( lambda step_conf: step_conf[1].get('step.enabled', False), computeStatistic_conf[1].get('prepares.steps', {}).iteritems()) )).iteritems()) ), compute_computeStatistics_config_active)) # print('= = ' * 30, compute_prepares_config_active2 == compute_prepares_config_active) print('= = ' * 20, type(compute_prepares_config_active), 'compute_prepares_config_active = ') pprint(compute_prepares_config_active) compute_computes_config_active = dict(map( lambda computeStatistic_conf: (computeStatistic_conf[0], sorted(list_dict_merge( map(lambda step_conf: map_conf_properties(step_conf[1], 'step.id'), filter(lambda step_conf: step_conf[1].get('step.enabled', False), computeStatistic_conf[1].get('computes.steps', {}).iteritems()) )).iteritems()) ), compute_computeStatistics_config_active)) print('= = ' * 20, type(compute_computes_config_active), 'compute_computes_config_active = ') pprint(compute_computes_config_active) test_flag = False if not test_flag: # 初始化 # 测试 serializer # serializer 默认取值 PickleSerializer() #UnpicklingError: invalid load key, '{'. # serializer=MarshalSerializer() # ValueError: bad marshal data # serializer=AutoSerializer() # ValueError: invalid sevialization type: { # serializer=CompressedSerializer(PickleSerializer()) # error: Error -3 while decompressing data: incorrect header check # sc = SparkContext(master, app_name, sparkHome = spark_home, pyFiles=pyFiles) # sc = SparkContext(master, app_name, sparkHome = sparkHome, pyFiles=pyFiles, serializer=MarshalSerializer()) # sc = SparkContext(master, app_name, sparkHome = sparkHome, pyFiles=pyFiles, serializer=AutoSerializer()) # sc = SparkContext(master, app_name, sparkHome = sparkHome, pyFiles=pyFiles, serializer=CompressedSerializer(PickleSerializer())) spark_conf = SparkConf() spark_conf.setMaster(master).setAppName(app_name).setSparkHome(spark_home) # spark streaming 调优配置 spark_streaming_blockInterval = str(app_conf.get('spark.streaming.blockInterval', '')).strip() if spark_streaming_blockInterval: spark_conf.set('spark.streaming.blockInterval', spark_streaming_blockInterval) spark_streaming_kafka_maxRatePerPartition = str( app_conf.get('spark.streaming.kafka.maxRatePerPartition', '')).strip() if spark_streaming_kafka_maxRatePerPartition: spark_conf.set('spark.streaming.kafka.maxRatePerPartition', spark_streaming_kafka_maxRatePerPartition) spark_streaming_receiver_maxRate = str(app_conf.get('spark.streaming.receiver.maxRate', '')).strip() if spark_streaming_receiver_maxRate: spark_conf.set('spark.streaming.receiver.maxRate', spark_streaming_receiver_maxRate) spark_streaming_concurrentJobs = str(app_conf.get('spark.streaming.concurrentJobs', '')).strip() if spark_streaming_concurrentJobs: spark_conf.set('spark.streaming.concurrentJobs', spark_streaming_concurrentJobs) # spark sql 调优配置 spark_sql_shuffle_partitions = str(app_conf.get('spark.sql.shuffle.partitions', '')).strip() if spark_sql_shuffle_partitions: spark_conf.set('spark.sql.shuffle.partitions', spark_sql_shuffle_partitions) sc = SparkContext(conf=spark_conf) for path in (pyFiles or []): sc.addPyFile(path) # 外部缓存优化,broadcast 分发 cache_manager = CacheManager() cache_broadcast_list = \ [(cache_id, cache_manager.cache_dataset(sc, cache_conf)) for cache_id, cache_conf in cache_confs_with_ds_conf.iteritems() if cache_conf.get('broadcast.enabled', False)] for cache_id, cache_broadcast in cache_broadcast_list: cache_confs_with_ds_conf[cache_id]['broadcast'] = cache_broadcast batchDruationSeconds = app_conf['batchDuration.seconds'] ssc = StreamingContext(sc, batchDruationSeconds) sqlc = SQLContext(sc) # 读取数据源 stream = StreamingReader.readSource(ssc, di_in_conf_with_ds_conf, app_conf) # 流处理: 1 根据配置初始化处理指定数据接口的类的实例, 2 调用指定处理类实例的流数据处理方法 # 测试 kafka_wordcount # counts = stream.flatMap(lambda line: line.split(" ")) \ # .map(lambda word: (word, 1)) \ # .reduceByKey(lambda a, b: a+b) # counts.pprint() StreamingApp.process( stream, sc, sqlc, di_in_conf_with_ds_conf, di_out_confs_with_ds_conf, cache_confs_with_ds_conf, prepares_config_active_steps, compute_prepares_config_active, compute_computes_config_active) ssc.start() ssc.awaitTermination()
def get_spark_session(): conf = SparkConf() conf.setMaster('yarn-client') conf.set("spark.yarn.am.cores", 7) conf.set("spark.executor.memory", "50g") conf.set("spark.executor.instances", 10) conf.set("spark.executor.cores", 10) conf.set("spark.python.worker.memory", "2g") conf.set("spark.default.parallelism", 1000) conf.set("spark.sql.shuffle.partitions", 1000) conf.set("spark.broadcast.blockSize", 1024) conf.set("spark.shuffle.file.buffer", '512k') conf.set("spark.speculation", True) conf.set("spark.speculation.quantile", 0.98) spark = SparkSession \ .builder \ .appName("hgongjing2_two_prd_pe_risk_score") \ .config(conf = conf) \ .enableHiveSupport() \ .getOrCreate() return spark
# this is the deprecated equivalent of ADD_JARS add_files = None if os.environ.get("ADD_FILES") is not None: add_files = os.environ.get("ADD_FILES").split(",") if os.environ.get("SPARK_EXECUTOR_URI"): SparkContext.setSystemProperty("spark.executor.uri", os.environ["SPARK_EXECUTOR_URI"]) # setup mesos-based connection conf = SparkConf().setMaster(os.environ["SPARK_MASTER"]) # set the UI port conf.set("spark.ui.port", ui_get_available_port()) # optionally set the Spark binary if os.environ.get("SPARK_BINARY"): conf.set("spark.executor.uri", os.environ["SPARK_BINARY"]) # establish config-based context sc = SparkContext(appName="DockerIPythonShell", pyFiles=add_files, conf=conf) atexit.register(lambda: sc.stop()) try: # Try to access HiveConf, it will raise exception if Hive is not added sc._jvm.org.apache.hadoop.hive.conf.HiveConf()
class SparkContext(object): """ Main entry point for Spark functionality. A SparkContext represents the connection to a Spark cluster, and can be used to create L{RDD} and broadcast variables on that cluster. """ _gateway = None _jvm = None _next_accum_id = 0 _active_spark_context = None _lock = RLock() _python_includes = None # zip and egg files that need to be added to PYTHONPATH PACKAGE_EXTENSIONS = ('.zip', '.egg', '.jar') def __init__(self, master=None, appName=None, sparkHome=None, pyFiles=None, environment=None, batchSize=0, serializer=PickleSerializer(), conf=None, gateway=None, jsc=None, profiler_cls=BasicProfiler): """ Create a new SparkContext. At least the master and app name should be set, either through the named parameters here or through C{conf}. :param master: Cluster URL to connect to (e.g. mesos://host:port, spark://host:port, local[4]). :param appName: A name for your job, to display on the cluster web UI. :param sparkHome: Location where Spark is installed on cluster nodes. :param pyFiles: Collection of .zip or .py files to send to the cluster and add to PYTHONPATH. These can be paths on the local file system or HDFS, HTTP, HTTPS, or FTP URLs. :param environment: A dictionary of environment variables to set on worker nodes. :param batchSize: The number of Python objects represented as a single Java object. Set 1 to disable batching, 0 to automatically choose the batch size based on object sizes, or -1 to use an unlimited batch size :param serializer: The serializer for RDDs. :param conf: A L{SparkConf} object setting Spark properties. :param gateway: Use an existing gateway and JVM, otherwise a new JVM will be instantiated. :param jsc: The JavaSparkContext instance (optional). :param profiler_cls: A class of custom Profiler used to do profiling (default is pyspark.profiler.BasicProfiler). >>> from pyspark.context import SparkContext >>> sc = SparkContext('local', 'test') >>> sc2 = SparkContext('local', 'test2') # doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): ... ValueError:... """ self._callsite = first_spark_call() or CallSite(None, None, None) SparkContext._ensure_initialized(self, gateway=gateway, conf=conf) try: self._do_init(master, appName, sparkHome, pyFiles, environment, batchSize, serializer, conf, jsc, profiler_cls) except: # If an error occurs, clean up in order to allow future SparkContext creation: self.stop() raise def _do_init(self, master, appName, sparkHome, pyFiles, environment, batchSize, serializer, conf, jsc, profiler_cls): self.environment = environment or {} # java gateway must have been launched at this point. if conf is not None and conf._jconf is not None: # conf has been initialized in JVM properly, so use conf directly. This represent the # scenario that JVM has been launched before SparkConf is created (e.g. SparkContext is # created and then stopped, and we create a new SparkConf and new SparkContext again) self._conf = conf else: self._conf = SparkConf(_jvm=SparkContext._jvm) if conf is not None: for k, v in conf.getAll(): self._conf.set(k, v) self._batchSize = batchSize # -1 represents an unlimited batch size self._unbatched_serializer = serializer if batchSize == 0: self.serializer = AutoBatchedSerializer(self._unbatched_serializer) else: self.serializer = BatchedSerializer(self._unbatched_serializer, batchSize) # Set any parameters passed directly to us on the conf if master: self._conf.setMaster(master) if appName: self._conf.setAppName(appName) if sparkHome: self._conf.setSparkHome(sparkHome) if environment: for key, value in environment.items(): self._conf.setExecutorEnv(key, value) for key, value in DEFAULT_CONFIGS.items(): self._conf.setIfMissing(key, value) # Check that we have at least the required parameters if not self._conf.contains("spark.master"): raise Exception("A master URL must be set in your configuration") if not self._conf.contains("spark.app.name"): raise Exception("An application name must be set in your configuration") # Read back our properties from the conf in case we loaded some of them from # the classpath or an external config file self.master = self._conf.get("spark.master") self.appName = self._conf.get("spark.app.name") self.sparkHome = self._conf.get("spark.home", None) for (k, v) in self._conf.getAll(): if k.startswith("spark.executorEnv."): varName = k[len("spark.executorEnv."):] self.environment[varName] = v self.environment["PYTHONHASHSEED"] = os.environ.get("PYTHONHASHSEED", "0") # Create the Java SparkContext through Py4J self._jsc = jsc or self._initialize_context(self._conf._jconf) # Reset the SparkConf to the one actually used by the SparkContext in JVM. self._conf = SparkConf(_jconf=self._jsc.sc().conf()) # Create a single Accumulator in Java that we'll send all our updates through; # they will be passed back to us through a TCP server self._accumulatorServer = accumulators._start_update_server() (host, port) = self._accumulatorServer.server_address self._javaAccumulator = self._jvm.PythonAccumulatorV2(host, port) self._jsc.sc().register(self._javaAccumulator) self.pythonExec = os.environ.get("PYSPARK_PYTHON", 'python') self.pythonVer = "%d.%d" % sys.version_info[:2] # Broadcast's __reduce__ method stores Broadcast instances here. # This allows other code to determine which Broadcast instances have # been pickled, so it can determine which Java broadcast objects to # send. self._pickled_broadcast_vars = BroadcastPickleRegistry() SparkFiles._sc = self root_dir = SparkFiles.getRootDirectory() sys.path.insert(1, root_dir) # Deploy any code dependencies specified in the constructor self._python_includes = list() for path in (pyFiles or []): self.addPyFile(path) # Deploy code dependencies set by spark-submit; these will already have been added # with SparkContext.addFile, so we just need to add them to the PYTHONPATH for path in self._conf.get("spark.submit.pyFiles", "").split(","): if path != "": (dirname, filename) = os.path.split(path) if filename[-4:].lower() in self.PACKAGE_EXTENSIONS: self._python_includes.append(filename) sys.path.insert(1, os.path.join(SparkFiles.getRootDirectory(), filename)) # Create a temporary directory inside spark.local.dir: local_dir = self._jvm.org.apache.spark.util.Utils.getLocalDir(self._jsc.sc().conf()) self._temp_dir = \ self._jvm.org.apache.spark.util.Utils.createTempDir(local_dir, "pyspark") \ .getAbsolutePath() # profiling stats collected for each PythonRDD if self._conf.get("spark.python.profile", "false") == "true": dump_path = self._conf.get("spark.python.profile.dump", None) self.profiler_collector = ProfilerCollector(profiler_cls, dump_path) else: self.profiler_collector = None # create a signal handler which would be invoked on receiving SIGINT def signal_handler(signal, frame): self.cancelAllJobs() raise KeyboardInterrupt() # see http://stackoverflow.com/questions/23206787/ if isinstance(threading.current_thread(), threading._MainThread): signal.signal(signal.SIGINT, signal_handler) def __repr__(self): return "<SparkContext master={master} appName={appName}>".format( master=self.master, appName=self.appName, ) def _repr_html_(self): return """ <div> <p><b>SparkContext</b></p> <p><a href="{sc.uiWebUrl}">Spark UI</a></p> <dl> <dt>Version</dt> <dd><code>v{sc.version}</code></dd> <dt>Master</dt> <dd><code>{sc.master}</code></dd> <dt>AppName</dt> <dd><code>{sc.appName}</code></dd> </dl> </div> """.format( sc=self ) def _initialize_context(self, jconf): """ Initialize SparkContext in function to allow subclass specific initialization """ return self._jvm.JavaSparkContext(jconf) @classmethod def _ensure_initialized(cls, instance=None, gateway=None, conf=None): """ Checks whether a SparkContext is initialized or not. Throws error if a SparkContext is already running. """ with SparkContext._lock: if not SparkContext._gateway: SparkContext._gateway = gateway or launch_gateway(conf) SparkContext._jvm = SparkContext._gateway.jvm if instance: if (SparkContext._active_spark_context and SparkContext._active_spark_context != instance): currentMaster = SparkContext._active_spark_context.master currentAppName = SparkContext._active_spark_context.appName callsite = SparkContext._active_spark_context._callsite # Raise error if there is already a running Spark context raise ValueError( "Cannot run multiple SparkContexts at once; " "existing SparkContext(app=%s, master=%s)" " created by %s at %s:%s " % (currentAppName, currentMaster, callsite.function, callsite.file, callsite.linenum)) else: SparkContext._active_spark_context = instance def __getnewargs__(self): # This method is called when attempting to pickle SparkContext, which is always an error: raise Exception( "It appears that you are attempting to reference SparkContext from a broadcast " "variable, action, or transformation. SparkContext can only be used on the driver, " "not in code that it run on workers. For more information, see SPARK-5063." ) def __enter__(self): """ Enable 'with SparkContext(...) as sc: app(sc)' syntax. """ return self def __exit__(self, type, value, trace): """ Enable 'with SparkContext(...) as sc: app' syntax. Specifically stop the context on exit of the with block. """ self.stop() @classmethod def getOrCreate(cls, conf=None): """ Get or instantiate a SparkContext and register it as a singleton object. :param conf: SparkConf (optional) """ with SparkContext._lock: if SparkContext._active_spark_context is None: SparkContext(conf=conf or SparkConf()) return SparkContext._active_spark_context def setLogLevel(self, logLevel): """ Control our logLevel. This overrides any user-defined log settings. Valid log levels include: ALL, DEBUG, ERROR, FATAL, INFO, OFF, TRACE, WARN """ self._jsc.setLogLevel(logLevel) @classmethod def setSystemProperty(cls, key, value): """ Set a Java system property, such as spark.executor.memory. This must must be invoked before instantiating SparkContext. """ SparkContext._ensure_initialized() SparkContext._jvm.java.lang.System.setProperty(key, value) @property def version(self): """ The version of Spark on which this application is running. """ return self._jsc.version() @property @ignore_unicode_prefix def applicationId(self): """ A unique identifier for the Spark application. Its format depends on the scheduler implementation. * in case of local spark app something like 'local-1433865536131' * in case of YARN something like 'application_1433865536131_34483' >>> sc.applicationId # doctest: +ELLIPSIS u'local-...' """ return self._jsc.sc().applicationId() @property def uiWebUrl(self): """Return the URL of the SparkUI instance started by this SparkContext""" return self._jsc.sc().uiWebUrl().get() @property def startTime(self): """Return the epoch time when the Spark Context was started.""" return self._jsc.startTime() @property def defaultParallelism(self): """ Default level of parallelism to use when not given by user (e.g. for reduce tasks) """ return self._jsc.sc().defaultParallelism() @property def defaultMinPartitions(self): """ Default min number of partitions for Hadoop RDDs when not given by user """ return self._jsc.sc().defaultMinPartitions() def stop(self): """ Shut down the SparkContext. """ if getattr(self, "_jsc", None): try: self._jsc.stop() except Py4JError: # Case: SPARK-18523 warnings.warn( 'Unable to cleanly shutdown Spark JVM process.' ' It is possible that the process has crashed,' ' been killed or may also be in a zombie state.', RuntimeWarning ) pass finally: self._jsc = None if getattr(self, "_accumulatorServer", None): self._accumulatorServer.shutdown() self._accumulatorServer = None with SparkContext._lock: SparkContext._active_spark_context = None def emptyRDD(self): """ Create an RDD that has no partitions or elements. """ return RDD(self._jsc.emptyRDD(), self, NoOpSerializer()) def range(self, start, end=None, step=1, numSlices=None): """ Create a new RDD of int containing elements from `start` to `end` (exclusive), increased by `step` every element. Can be called the same way as python's built-in range() function. If called with a single argument, the argument is interpreted as `end`, and `start` is set to 0. :param start: the start value :param end: the end value (exclusive) :param step: the incremental step (default: 1) :param numSlices: the number of partitions of the new RDD :return: An RDD of int >>> sc.range(5).collect() [0, 1, 2, 3, 4] >>> sc.range(2, 4).collect() [2, 3] >>> sc.range(1, 7, 2).collect() [1, 3, 5] """ if end is None: end = start start = 0 return self.parallelize(xrange(start, end, step), numSlices) def parallelize(self, c, numSlices=None): """ Distribute a local Python collection to form an RDD. Using xrange is recommended if the input represents a range for performance. >>> sc.parallelize([0, 2, 3, 4, 6], 5).glom().collect() [[0], [2], [3], [4], [6]] >>> sc.parallelize(xrange(0, 6, 2), 5).glom().collect() [[], [0], [], [2], [4]] """ numSlices = int(numSlices) if numSlices is not None else self.defaultParallelism if isinstance(c, xrange): size = len(c) if size == 0: return self.parallelize([], numSlices) step = c[1] - c[0] if size > 1 else 1 start0 = c[0] def getStart(split): return start0 + int((split * size / numSlices)) * step def f(split, iterator): return xrange(getStart(split), getStart(split + 1), step) return self.parallelize([], numSlices).mapPartitionsWithIndex(f) # Calling the Java parallelize() method with an ArrayList is too slow, # because it sends O(n) Py4J commands. As an alternative, serialized # objects are written to a file and loaded through textFile(). tempFile = NamedTemporaryFile(delete=False, dir=self._temp_dir) try: # Make sure we distribute data evenly if it's smaller than self.batchSize if "__len__" not in dir(c): c = list(c) # Make it a list so we can compute its length batchSize = max(1, min(len(c) // numSlices, self._batchSize or 1024)) serializer = BatchedSerializer(self._unbatched_serializer, batchSize) serializer.dump_stream(c, tempFile) tempFile.close() readRDDFromFile = self._jvm.PythonRDD.readRDDFromFile jrdd = readRDDFromFile(self._jsc, tempFile.name, numSlices) finally: # readRDDFromFile eagerily reads the file so we can delete right after. os.unlink(tempFile.name) return RDD(jrdd, self, serializer) def pickleFile(self, name, minPartitions=None): """ Load an RDD previously saved using L{RDD.saveAsPickleFile} method. >>> tmpFile = NamedTemporaryFile(delete=True) >>> tmpFile.close() >>> sc.parallelize(range(10)).saveAsPickleFile(tmpFile.name, 5) >>> sorted(sc.pickleFile(tmpFile.name, 3).collect()) [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] """ minPartitions = minPartitions or self.defaultMinPartitions return RDD(self._jsc.objectFile(name, minPartitions), self) @ignore_unicode_prefix def textFile(self, name, minPartitions=None, use_unicode=True): """ Read a text file from HDFS, a local file system (available on all nodes), or any Hadoop-supported file system URI, and return it as an RDD of Strings. If use_unicode is False, the strings will be kept as `str` (encoding as `utf-8`), which is faster and smaller than unicode. (Added in Spark 1.2) >>> path = os.path.join(tempdir, "sample-text.txt") >>> with open(path, "w") as testFile: ... _ = testFile.write("Hello world!") >>> textFile = sc.textFile(path) >>> textFile.collect() [u'Hello world!'] """ minPartitions = minPartitions or min(self.defaultParallelism, 2) return RDD(self._jsc.textFile(name, minPartitions), self, UTF8Deserializer(use_unicode)) @ignore_unicode_prefix def wholeTextFiles(self, path, minPartitions=None, use_unicode=True): """ Read a directory of text files from HDFS, a local file system (available on all nodes), or any Hadoop-supported file system URI. Each file is read as a single record and returned in a key-value pair, where the key is the path of each file, the value is the content of each file. If use_unicode is False, the strings will be kept as `str` (encoding as `utf-8`), which is faster and smaller than unicode. (Added in Spark 1.2) For example, if you have the following files:: hdfs://a-hdfs-path/part-00000 hdfs://a-hdfs-path/part-00001 ... hdfs://a-hdfs-path/part-nnnnn Do C{rdd = sparkContext.wholeTextFiles("hdfs://a-hdfs-path")}, then C{rdd} contains:: (a-hdfs-path/part-00000, its content) (a-hdfs-path/part-00001, its content) ... (a-hdfs-path/part-nnnnn, its content) .. note:: Small files are preferred, as each file will be loaded fully in memory. >>> dirPath = os.path.join(tempdir, "files") >>> os.mkdir(dirPath) >>> with open(os.path.join(dirPath, "1.txt"), "w") as file1: ... _ = file1.write("1") >>> with open(os.path.join(dirPath, "2.txt"), "w") as file2: ... _ = file2.write("2") >>> textFiles = sc.wholeTextFiles(dirPath) >>> sorted(textFiles.collect()) [(u'.../1.txt', u'1'), (u'.../2.txt', u'2')] """ minPartitions = minPartitions or self.defaultMinPartitions return RDD(self._jsc.wholeTextFiles(path, minPartitions), self, PairDeserializer(UTF8Deserializer(use_unicode), UTF8Deserializer(use_unicode))) def binaryFiles(self, path, minPartitions=None): """ .. note:: Experimental Read a directory of binary files from HDFS, a local file system (available on all nodes), or any Hadoop-supported file system URI as a byte array. Each file is read as a single record and returned in a key-value pair, where the key is the path of each file, the value is the content of each file. .. note:: Small files are preferred, large file is also allowable, but may cause bad performance. """ minPartitions = minPartitions or self.defaultMinPartitions return RDD(self._jsc.binaryFiles(path, minPartitions), self, PairDeserializer(UTF8Deserializer(), NoOpSerializer())) def binaryRecords(self, path, recordLength): """ .. note:: Experimental Load data from a flat binary file, assuming each record is a set of numbers with the specified numerical format (see ByteBuffer), and the number of bytes per record is constant. :param path: Directory to the input data files :param recordLength: The length at which to split the records """ return RDD(self._jsc.binaryRecords(path, recordLength), self, NoOpSerializer()) def _dictToJavaMap(self, d): jm = self._jvm.java.util.HashMap() if not d: d = {} for k, v in d.items(): jm[k] = v return jm def sequenceFile(self, path, keyClass=None, valueClass=None, keyConverter=None, valueConverter=None, minSplits=None, batchSize=0): """ Read a Hadoop SequenceFile with arbitrary key and value Writable class from HDFS, a local file system (available on all nodes), or any Hadoop-supported file system URI. The mechanism is as follows: 1. A Java RDD is created from the SequenceFile or other InputFormat, and the key and value Writable classes 2. Serialization is attempted via Pyrolite pickling 3. If this fails, the fallback is to call 'toString' on each key and value 4. C{PickleSerializer} is used to deserialize pickled objects on the Python side :param path: path to sequncefile :param keyClass: fully qualified classname of key Writable class (e.g. "org.apache.hadoop.io.Text") :param valueClass: fully qualified classname of value Writable class (e.g. "org.apache.hadoop.io.LongWritable") :param keyConverter: :param valueConverter: :param minSplits: minimum splits in dataset (default min(2, sc.defaultParallelism)) :param batchSize: The number of Python objects represented as a single Java object. (default 0, choose batchSize automatically) """ minSplits = minSplits or min(self.defaultParallelism, 2) jrdd = self._jvm.PythonRDD.sequenceFile(self._jsc, path, keyClass, valueClass, keyConverter, valueConverter, minSplits, batchSize) return RDD(jrdd, self) def newAPIHadoopFile(self, path, inputFormatClass, keyClass, valueClass, keyConverter=None, valueConverter=None, conf=None, batchSize=0): """ Read a 'new API' Hadoop InputFormat with arbitrary key and value class from HDFS, a local file system (available on all nodes), or any Hadoop-supported file system URI. The mechanism is the same as for sc.sequenceFile. A Hadoop configuration can be passed in as a Python dict. This will be converted into a Configuration in Java :param path: path to Hadoop file :param inputFormatClass: fully qualified classname of Hadoop InputFormat (e.g. "org.apache.hadoop.mapreduce.lib.input.TextInputFormat") :param keyClass: fully qualified classname of key Writable class (e.g. "org.apache.hadoop.io.Text") :param valueClass: fully qualified classname of value Writable class (e.g. "org.apache.hadoop.io.LongWritable") :param keyConverter: (None by default) :param valueConverter: (None by default) :param conf: Hadoop configuration, passed in as a dict (None by default) :param batchSize: The number of Python objects represented as a single Java object. (default 0, choose batchSize automatically) """ jconf = self._dictToJavaMap(conf) jrdd = self._jvm.PythonRDD.newAPIHadoopFile(self._jsc, path, inputFormatClass, keyClass, valueClass, keyConverter, valueConverter, jconf, batchSize) return RDD(jrdd, self) def newAPIHadoopRDD(self, inputFormatClass, keyClass, valueClass, keyConverter=None, valueConverter=None, conf=None, batchSize=0): """ Read a 'new API' Hadoop InputFormat with arbitrary key and value class, from an arbitrary Hadoop configuration, which is passed in as a Python dict. This will be converted into a Configuration in Java. The mechanism is the same as for sc.sequenceFile. :param inputFormatClass: fully qualified classname of Hadoop InputFormat (e.g. "org.apache.hadoop.mapreduce.lib.input.TextInputFormat") :param keyClass: fully qualified classname of key Writable class (e.g. "org.apache.hadoop.io.Text") :param valueClass: fully qualified classname of value Writable class (e.g. "org.apache.hadoop.io.LongWritable") :param keyConverter: (None by default) :param valueConverter: (None by default) :param conf: Hadoop configuration, passed in as a dict (None by default) :param batchSize: The number of Python objects represented as a single Java object. (default 0, choose batchSize automatically) """ jconf = self._dictToJavaMap(conf) jrdd = self._jvm.PythonRDD.newAPIHadoopRDD(self._jsc, inputFormatClass, keyClass, valueClass, keyConverter, valueConverter, jconf, batchSize) return RDD(jrdd, self) def hadoopFile(self, path, inputFormatClass, keyClass, valueClass, keyConverter=None, valueConverter=None, conf=None, batchSize=0): """ Read an 'old' Hadoop InputFormat with arbitrary key and value class from HDFS, a local file system (available on all nodes), or any Hadoop-supported file system URI. The mechanism is the same as for sc.sequenceFile. A Hadoop configuration can be passed in as a Python dict. This will be converted into a Configuration in Java. :param path: path to Hadoop file :param inputFormatClass: fully qualified classname of Hadoop InputFormat (e.g. "org.apache.hadoop.mapred.TextInputFormat") :param keyClass: fully qualified classname of key Writable class (e.g. "org.apache.hadoop.io.Text") :param valueClass: fully qualified classname of value Writable class (e.g. "org.apache.hadoop.io.LongWritable") :param keyConverter: (None by default) :param valueConverter: (None by default) :param conf: Hadoop configuration, passed in as a dict (None by default) :param batchSize: The number of Python objects represented as a single Java object. (default 0, choose batchSize automatically) """ jconf = self._dictToJavaMap(conf) jrdd = self._jvm.PythonRDD.hadoopFile(self._jsc, path, inputFormatClass, keyClass, valueClass, keyConverter, valueConverter, jconf, batchSize) return RDD(jrdd, self) def hadoopRDD(self, inputFormatClass, keyClass, valueClass, keyConverter=None, valueConverter=None, conf=None, batchSize=0): """ Read an 'old' Hadoop InputFormat with arbitrary key and value class, from an arbitrary Hadoop configuration, which is passed in as a Python dict. This will be converted into a Configuration in Java. The mechanism is the same as for sc.sequenceFile. :param inputFormatClass: fully qualified classname of Hadoop InputFormat (e.g. "org.apache.hadoop.mapred.TextInputFormat") :param keyClass: fully qualified classname of key Writable class (e.g. "org.apache.hadoop.io.Text") :param valueClass: fully qualified classname of value Writable class (e.g. "org.apache.hadoop.io.LongWritable") :param keyConverter: (None by default) :param valueConverter: (None by default) :param conf: Hadoop configuration, passed in as a dict (None by default) :param batchSize: The number of Python objects represented as a single Java object. (default 0, choose batchSize automatically) """ jconf = self._dictToJavaMap(conf) jrdd = self._jvm.PythonRDD.hadoopRDD(self._jsc, inputFormatClass, keyClass, valueClass, keyConverter, valueConverter, jconf, batchSize) return RDD(jrdd, self) def _checkpointFile(self, name, input_deserializer): jrdd = self._jsc.checkpointFile(name) return RDD(jrdd, self, input_deserializer) @ignore_unicode_prefix def union(self, rdds): """ Build the union of a list of RDDs. This supports unions() of RDDs with different serialized formats, although this forces them to be reserialized using the default serializer: >>> path = os.path.join(tempdir, "union-text.txt") >>> with open(path, "w") as testFile: ... _ = testFile.write("Hello") >>> textFile = sc.textFile(path) >>> textFile.collect() [u'Hello'] >>> parallelized = sc.parallelize(["World!"]) >>> sorted(sc.union([textFile, parallelized]).collect()) [u'Hello', 'World!'] """ first_jrdd_deserializer = rdds[0]._jrdd_deserializer if any(x._jrdd_deserializer != first_jrdd_deserializer for x in rdds): rdds = [x._reserialize() for x in rdds] first = rdds[0]._jrdd rest = [x._jrdd for x in rdds[1:]] return RDD(self._jsc.union(first, rest), self, rdds[0]._jrdd_deserializer) def broadcast(self, value): """ Broadcast a read-only variable to the cluster, returning a L{Broadcast<pyspark.broadcast.Broadcast>} object for reading it in distributed functions. The variable will be sent to each cluster only once. """ return Broadcast(self, value, self._pickled_broadcast_vars) def accumulator(self, value, accum_param=None): """ Create an L{Accumulator} with the given initial value, using a given L{AccumulatorParam} helper object to define how to add values of the data type if provided. Default AccumulatorParams are used for integers and floating-point numbers if you do not provide one. For other types, a custom AccumulatorParam can be used. """ if accum_param is None: if isinstance(value, int): accum_param = accumulators.INT_ACCUMULATOR_PARAM elif isinstance(value, float): accum_param = accumulators.FLOAT_ACCUMULATOR_PARAM elif isinstance(value, complex): accum_param = accumulators.COMPLEX_ACCUMULATOR_PARAM else: raise TypeError("No default accumulator param for type %s" % type(value)) SparkContext._next_accum_id += 1 return Accumulator(SparkContext._next_accum_id - 1, value, accum_param) def addFile(self, path, recursive=False): """ Add a file to be downloaded with this Spark job on every node. The C{path} passed can be either a local file, a file in HDFS (or other Hadoop-supported filesystems), or an HTTP, HTTPS or FTP URI. To access the file in Spark jobs, use L{SparkFiles.get(fileName)<pyspark.files.SparkFiles.get>} with the filename to find its download location. A directory can be given if the recursive option is set to True. Currently directories are only supported for Hadoop-supported filesystems. >>> from pyspark import SparkFiles >>> path = os.path.join(tempdir, "test.txt") >>> with open(path, "w") as testFile: ... _ = testFile.write("100") >>> sc.addFile(path) >>> def func(iterator): ... with open(SparkFiles.get("test.txt")) as testFile: ... fileVal = int(testFile.readline()) ... return [x * fileVal for x in iterator] >>> sc.parallelize([1, 2, 3, 4]).mapPartitions(func).collect() [100, 200, 300, 400] """ self._jsc.sc().addFile(path, recursive) def addPyFile(self, path): """ Add a .py or .zip dependency for all tasks to be executed on this SparkContext in the future. The C{path} passed can be either a local file, a file in HDFS (or other Hadoop-supported filesystems), or an HTTP, HTTPS or FTP URI. """ self.addFile(path) (dirname, filename) = os.path.split(path) # dirname may be directory or HDFS/S3 prefix if filename[-4:].lower() in self.PACKAGE_EXTENSIONS: self._python_includes.append(filename) # for tests in local mode sys.path.insert(1, os.path.join(SparkFiles.getRootDirectory(), filename)) if sys.version > '3': import importlib importlib.invalidate_caches() def setCheckpointDir(self, dirName): """ Set the directory under which RDDs are going to be checkpointed. The directory must be a HDFS path if running on a cluster. """ self._jsc.sc().setCheckpointDir(dirName) def _getJavaStorageLevel(self, storageLevel): """ Returns a Java StorageLevel based on a pyspark.StorageLevel. """ if not isinstance(storageLevel, StorageLevel): raise Exception("storageLevel must be of type pyspark.StorageLevel") newStorageLevel = self._jvm.org.apache.spark.storage.StorageLevel return newStorageLevel(storageLevel.useDisk, storageLevel.useMemory, storageLevel.useOffHeap, storageLevel.deserialized, storageLevel.replication) def setJobGroup(self, groupId, description, interruptOnCancel=False): """ Assigns a group ID to all the jobs started by this thread until the group ID is set to a different value or cleared. Often, a unit of execution in an application consists of multiple Spark actions or jobs. Application programmers can use this method to group all those jobs together and give a group description. Once set, the Spark web UI will associate such jobs with this group. The application can use L{SparkContext.cancelJobGroup} to cancel all running jobs in this group. >>> import threading >>> from time import sleep >>> result = "Not Set" >>> lock = threading.Lock() >>> def map_func(x): ... sleep(100) ... raise Exception("Task should have been cancelled") >>> def start_job(x): ... global result ... try: ... sc.setJobGroup("job_to_cancel", "some description") ... result = sc.parallelize(range(x)).map(map_func).collect() ... except Exception as e: ... result = "Cancelled" ... lock.release() >>> def stop_job(): ... sleep(5) ... sc.cancelJobGroup("job_to_cancel") >>> supress = lock.acquire() >>> supress = threading.Thread(target=start_job, args=(10,)).start() >>> supress = threading.Thread(target=stop_job).start() >>> supress = lock.acquire() >>> print(result) Cancelled If interruptOnCancel is set to true for the job group, then job cancellation will result in Thread.interrupt() being called on the job's executor threads. This is useful to help ensure that the tasks are actually stopped in a timely manner, but is off by default due to HDFS-1208, where HDFS may respond to Thread.interrupt() by marking nodes as dead. """ self._jsc.setJobGroup(groupId, description, interruptOnCancel) def setLocalProperty(self, key, value): """ Set a local property that affects jobs submitted from this thread, such as the Spark fair scheduler pool. """ self._jsc.setLocalProperty(key, value) def getLocalProperty(self, key): """ Get a local property set in this thread, or null if it is missing. See L{setLocalProperty} """ return self._jsc.getLocalProperty(key) def setJobDescription(self, value): """ Set a human readable description of the current job. """ self._jsc.setJobDescription(value) def sparkUser(self): """ Get SPARK_USER for user who is running SparkContext. """ return self._jsc.sc().sparkUser() def cancelJobGroup(self, groupId): """ Cancel active jobs for the specified group. See L{SparkContext.setJobGroup} for more information. """ self._jsc.sc().cancelJobGroup(groupId) def cancelAllJobs(self): """ Cancel all jobs that have been scheduled or are running. """ self._jsc.sc().cancelAllJobs() def statusTracker(self): """ Return :class:`StatusTracker` object """ return StatusTracker(self._jsc.statusTracker()) def runJob(self, rdd, partitionFunc, partitions=None, allowLocal=False): """ Executes the given partitionFunc on the specified set of partitions, returning the result as an array of elements. If 'partitions' is not specified, this will run over all partitions. >>> myRDD = sc.parallelize(range(6), 3) >>> sc.runJob(myRDD, lambda part: [x * x for x in part]) [0, 1, 4, 9, 16, 25] >>> myRDD = sc.parallelize(range(6), 3) >>> sc.runJob(myRDD, lambda part: [x * x for x in part], [0, 2], True) [0, 1, 16, 25] """ if partitions is None: partitions = range(rdd._jrdd.partitions().size()) # Implementation note: This is implemented as a mapPartitions followed # by runJob() in order to avoid having to pass a Python lambda into # SparkContext#runJob. mappedRDD = rdd.mapPartitions(partitionFunc) port = self._jvm.PythonRDD.runJob(self._jsc.sc(), mappedRDD._jrdd, partitions) return list(_load_from_socket(port, mappedRDD._jrdd_deserializer)) def show_profiles(self): """ Print the profile stats to stdout """ self.profiler_collector.show_profiles() def dump_profiles(self, path): """ Dump the profile stats into directory `path` """ self.profiler_collector.dump_profiles(path) def getConf(self): conf = SparkConf() conf.setAll(self._conf.getAll()) return conf
def setUpClass(cls): conf = SparkConf() conf.set('spark.ui.showConsoleProgress', 'false') cls.sc = EsSparkContext(conf=conf.setAppName("PySpark Elastic Test"))