def get_cluster_create_clone_update(arguments, action):
customer_ssh_key = util._read_file(arguments.customer_ssh_key_file)
# This will set cluster info and monitoring settings
cluster_info = ClusterInfoV2(arguments.label)
cluster_info.set_cluster_info(
disallow_cluster_termination=arguments.
disallow_cluster_termination,
enable_ganglia_monitoring=arguments.enable_ganglia_monitoring,
datadog_api_token=arguments.datadog_api_token,
datadog_app_token=arguments.datadog_app_token,
node_bootstrap=arguments.node_bootstrap_file,
master_instance_type=arguments.master_instance_type,
slave_instance_type=arguments.slave_instance_type,
min_nodes=arguments.initial_nodes,
max_nodes=arguments.max_nodes,
slave_request_type=arguments.slave_request_type,
fallback_to_ondemand=arguments.fallback_to_ondemand,
node_base_cooldown_period=arguments.node_base_cooldown_period,
node_spot_cooldown_period=arguments.node_spot_cooldown_period,
custom_tags=arguments.custom_tags,
heterogeneous_config=arguments.heterogeneous_config,
maximum_bid_price_percentage=arguments.
maximum_bid_price_percentage,
timeout_for_request=arguments.timeout_for_request,
maximum_spot_instance_percentage=arguments.
maximum_spot_instance_percentage,
stable_maximum_bid_price_percentage=arguments.
stable_maximum_bid_price_percentage,
stable_timeout_for_request=arguments.stable_timeout_for_request,
stable_spot_fallback=arguments.stable_spot_fallback,
spot_block_duration=arguments.spot_block_duration,
idle_cluster_timeout=arguments.idle_cluster_timeout,
disk_count=arguments.count,
disk_type=arguments.disk_type,
disk_size=arguments.size,
root_disk_size=arguments.root_disk_size,
upscaling_config=arguments.upscaling_config,
enable_encryption=arguments.encrypted_ephemerals,
customer_ssh_key=customer_ssh_key,
image_uri_overrides=arguments.image_uri_overrides,
env_name=arguments.env_name,
python_version=arguments.python_version,
r_version=arguments.r_version)
# This will set cloud config settings
cloud_config = Qubole.get_cloud()
cloud_config.set_cloud_config_from_arguments(arguments)
# This will set engine settings
engine_config = Engine(flavour=arguments.flavour)
engine_config.set_engine_config_settings(arguments)
cluster_request = ClusterCmdLine.get_cluster_request_parameters(
cluster_info, cloud_config, engine_config)
action = action
if action == "create":
return arguments.func(cluster_request)
else:
return arguments.func(arguments.cluster_id_label, cluster_request)
def create_update_clone_parser(subparser, action=None):
# cloud config parser
cloud = Qubole.get_cloud()
cloud.create_parser(subparser)
# cluster info parser
ClusterInfoV2.cluster_info_parser(subparser, action)
# engine config parser
Engine.engine_parser(subparser)
def create_update_clone_parser(subparser, action=None):
# cloud config parser
cloud = Qubole.get_cloud()
cloud.create_parser(subparser)
# cluster info parser
ClusterInfoV2.cluster_info_parser(subparser, action)
# engine config parser
Engine.engine_parser(subparser)
def get_cluster_create_clone_update(arguments, action):
customer_ssh_key = util._read_file(arguments.customer_ssh_key_file)
# This will set cluster info and monitoring settings
cluster_info = ClusterInfoV2(arguments.label)
cluster_info.set_cluster_info(disallow_cluster_termination=arguments.disallow_cluster_termination,
enable_ganglia_monitoring=arguments.enable_ganglia_monitoring,
datadog_api_token=arguments.datadog_api_token,
datadog_app_token=arguments.datadog_app_token,
node_bootstrap=arguments.node_bootstrap_file,
master_instance_type=arguments.master_instance_type,
slave_instance_type=arguments.slave_instance_type,
min_nodes=arguments.initial_nodes,
max_nodes=arguments.max_nodes,
slave_request_type=arguments.slave_request_type,
fallback_to_ondemand=arguments.fallback_to_ondemand,
custom_tags=arguments.custom_tags,
heterogeneous_config=arguments.heterogeneous_config,
maximum_bid_price_percentage=arguments.maximum_bid_price_percentage,
timeout_for_request=arguments.timeout_for_request,
maximum_spot_instance_percentage=arguments.maximum_spot_instance_percentage,
stable_maximum_bid_price_percentage=arguments.stable_maximum_bid_price_percentage,
stable_timeout_for_request=arguments.stable_timeout_for_request,
stable_spot_fallback=arguments.stable_spot_fallback,
idle_cluster_timeout=arguments.idle_cluster_timeout,
disk_count=arguments.count,
disk_type=arguments.disk_type,
disk_size=arguments.size,
upscaling_config=arguments.upscaling_config,
enable_encryption=arguments.encrypted_ephemerals,
customer_ssh_key=customer_ssh_key,
image_uri_overrides=arguments.image_uri_overrides)
# This will set cloud config settings
cloud_config = Qubole.get_cloud()
cloud_config.set_cloud_config_from_arguments(arguments)
# This will set engine settings
engine_config = Engine(flavour=arguments.flavour)
engine_config.set_engine_config_settings(arguments)
cluster_request = ClusterCmdLine.get_cluster_request_parameters(cluster_info, cloud_config, engine_config)
action = action
if action == "create":
return arguments.func(cluster_request)
else:
return arguments.func(arguments.cluster_id_label, cluster_request)
def _create_spark_cluster_info(config):
cluster_info = ClusterInfoV2(config['spark_cluster_name'])
cluster_info.set_cluster_info(
master_instance_type=config['hadoop_master_instance_type'],
slave_instance_type=config['hadoop_slave_instance_type'],
min_nodes=1,
max_nodes=config['hadoop_max_nodes_count'],
slave_request_type='spot')
cloud_config = Qubole.get_cloud(cloud_name='aws')
cloud_config.set_cloud_config(aws_region=config['region_name'],
aws_availability_zone='Any',
vpc_id=config['cluster_vpc_id'],
subnet_id=config['cluster_subnet_id'])
engine_config = Engine(flavour='spark')
engine_config.set_engine_config(spark_version='2.1.0')
cluster_request = ClusterCmdLine.get_cluster_request_parameters(
cluster_info, cloud_config, engine_config)
return cluster_request
def get_cluster_create_clone_update(arguments, action):
# This will set cluster info and monitoring settings
cluster_info_cls = ClusterInfoFactory.get_cluster_info_cls()
cluster_info = cluster_info_cls(arguments.label)
cluster_info.set_cluster_info_from_arguments(arguments)
# This will set cloud config settings
cloud_config = Qubole.get_cloud()
cloud_config.set_cloud_config_from_arguments(arguments)
# This will set engine settings
engine_config = Engine(flavour=arguments.flavour)
engine_config.set_engine_config_settings(arguments)
cluster_request = ClusterCmdLine.get_cluster_request_parameters(cluster_info, cloud_config, engine_config)
action = action
if action == "create":
return arguments.func(cluster_request)
else:
return arguments.func(arguments.cluster_id_label, cluster_request)