def __init__( self, application ):
super( CreateSparkCluster, self ).__init__( application )
self.option( '--num-slaves', '-s', metavar='NUM',
type=int, default=1,
help='The number of slaves to start.' )
# We want --instance-type for the slaves and --master-instance-type for the master and we
# want --master-instance-type to default to the value of --instance-type.
super( CreateSparkCluster, self ).option(
'--instance-type', '-t', metavar='TYPE', dest='slave_instance_type',
default=SparkBox.recommended_instance_type( ),
help='The type of EC2 instance to launch for the slaves, e.g. t2.micro, '
'm3.small, m3.medium, or m3.large etc. ' )
self.option( '--master-instance-type', metavar='TYPE', dest='instance_type',
help='The type of EC2 instance to launch for the master, e.g. t2.micro, '
'm3.small, m3.medium, or m3.large etc. The default is the instance type '
'used for the slaves.' )
self.option( '--ebs-volume-size', metavar='GB', default=0,
help='The size in GB of an EBS volume to be attached to each node for '
'persistent data such as that backing HDFS. By default HDFS will be '
'backed instance store ( ephemeral) only, or the root volume for '
'instance types that do not offer instance store.' )
import os
from textwrap import dedent
import time
import logging
import unittest
from tempfile import mkstemp
from cgcloud.core.test import CgcloudTestCase
from cgcloud.lib.util import heredoc
from cgcloud.spark.spark_box import install_dir, SparkBox, SparkMaster, SparkSlave
log = logging.getLogger(__name__)
master = SparkMaster.role()
slave = SparkSlave.role()
node = SparkBox.role()
num_slaves = 2
class SparkClusterTests(CgcloudTestCase):
"""
Covers the creation of a Spark cluster from scratch and running a simple Spark job on it.
Also covers persistant HDFS between two cluster incarnations.
"""
cleanup = True
create_image = True
@classmethod
def setUpClass(cls):
os.environ['CGCLOUD_PLUGINS'] = 'cgcloud.spark'
import os
from textwrap import dedent
import time
import logging
import unittest
from tempfile import mkstemp
from cgcloud.core.test import CgcloudTestCase
from cgcloud.lib.util import heredoc
from cgcloud.spark.spark_box import install_dir, SparkBox, SparkMaster, SparkSlave
log = logging.getLogger( __name__ )
master = SparkMaster.role( )
slave = SparkSlave.role( )
node = SparkBox.role( )
num_slaves = 2
class SparkClusterTests( CgcloudTestCase ):
"""
Covers the creation of a Spark cluster from scratch and running a simple Spark job on it.
Also covers persistant HDFS between two cluster incarnations.
"""
cleanup = True
create_image = True
@classmethod
def setUpClass( cls ):
os.environ[ 'CGCLOUD_PLUGINS' ] = 'cgcloud.spark'
import time
import logging
import unittest
from tempfile import mkstemp
import itertools
from cgcloud.core.test import CgcloudTestCase
from cgcloud.core.ui import main
from cgcloud.lib.util import heredoc
from cgcloud.spark.spark_box import install_dir, SparkBox, SparkMaster, SparkSlave
log = logging.getLogger( __name__ )
master = SparkMaster.role( )
slave = SparkSlave.role( )
role = SparkBox.role( )
num_slaves = 2
cleanup = True
create_image = True
class ClusterTests( CgcloudTestCase ):
"""
Tests the typical life-cycle of instances and images
"""
@classmethod
def setUpClass( cls ):
os.environ[ 'CGCLOUD_PLUGINS' ] = 'cgcloud.spark'