Python SparkBox.role Examples

Example #1

File: test_spark.py Project: arkal/cgcloud

import os
from textwrap import dedent
import time
import logging
import unittest
from tempfile import mkstemp

from cgcloud.core.test import CgcloudTestCase
from cgcloud.lib.util import heredoc
from cgcloud.spark.spark_box import install_dir, SparkBox, SparkMaster, SparkSlave

log = logging.getLogger( __name__ )

master = SparkMaster.role( )
slave = SparkSlave.role( )
node = SparkBox.role( )

num_slaves = 2


class SparkClusterTests( CgcloudTestCase ):
    """
    Covers the creation of a Spark cluster from scratch and running a simple Spark job on it.
    Also covers persistant HDFS between two cluster incarnations.
    """
    cleanup = True
    create_image = True

    @classmethod
    def setUpClass( cls ):
        os.environ[ 'CGCLOUD_PLUGINS' ] = 'cgcloud.spark'

Example #2

File: test_spark.py Project: fnothaft/cgcloud

import os
from textwrap import dedent
import time
import logging
import unittest
from tempfile import mkstemp

from cgcloud.core.test import CgcloudTestCase
from cgcloud.lib.util import heredoc
from cgcloud.spark.spark_box import install_dir, SparkBox, SparkMaster, SparkSlave

log = logging.getLogger(__name__)

master = SparkMaster.role()
slave = SparkSlave.role()
node = SparkBox.role()

num_slaves = 2


class SparkClusterTests(CgcloudTestCase):
    """
    Covers the creation of a Spark cluster from scratch and running a simple Spark job on it.
    Also covers persistant HDFS between two cluster incarnations.
    """
    cleanup = True
    create_image = True

    @classmethod
    def setUpClass(cls):
        os.environ['CGCLOUD_PLUGINS'] = 'cgcloud.spark'

Example #3

File: test_spark.py Project: joelarmstrong/cgcloud

import time
import logging
import unittest
from tempfile import mkstemp
import itertools

from cgcloud.core.test import CgcloudTestCase
from cgcloud.core.ui import main
from cgcloud.lib.util import heredoc
from cgcloud.spark.spark_box import install_dir, SparkBox, SparkMaster, SparkSlave

log = logging.getLogger( __name__ )

master = SparkMaster.role( )
slave = SparkSlave.role( )
role = SparkBox.role( )

num_slaves = 2

cleanup = True
create_image = True


class ClusterTests( CgcloudTestCase ):
    """
    Tests the typical life-cycle of instances and images
    """

    @classmethod
    def setUpClass( cls ):
        os.environ[ 'CGCLOUD_PLUGINS' ] = 'cgcloud.spark'