def _register_managers(self, client):
self.clusters = clusters.ClusterManagerV2(client)
self.cluster_templates = (
cluster_templates.ClusterTemplateManagerV2(client))
self.node_group_templates = (
node_group_templates.NodeGroupTemplateManagerV2(client))
self.plugins = plugins.PluginManagerV2(client)
self.images = images.ImageManagerV2(client)
self.data_sources = data_sources.DataSourceManagerV2(client)
self.job_templates = job_templates.JobTemplatesManagerV2(client)
self.jobs = jobs_v2.JobsManagerV2(client)
self.job_binaries = job_binaries.JobBinariesManagerV2(client)
self.job_types = job_types.JobTypesManager(client)
def data(TEST):
TEST.plugins = utils.TestDataContainer()
TEST.plugins_configs = utils.TestDataContainer()
TEST.nodegroup_templates = utils.TestDataContainer()
TEST.cluster_templates = utils.TestDataContainer()
TEST.clusters = utils.TestDataContainer()
TEST.data_sources = utils.TestDataContainer()
TEST.job_binaries = utils.TestDataContainer()
TEST.jobs = utils.TestDataContainer()
TEST.job_executions = utils.TestDataContainer()
TEST.registered_images = copy.copy(TEST.images)
TEST.job_types = utils.TestDataContainer()
plugin1_dict = {
"description": "vanilla plugin",
"name": "vanilla",
"title": "Vanilla Apache Hadoop",
"versions": ["2.3.0", "1.2.1"],
'version_labels': {
'2.3.0': {
'enabled': {
'status': True
}
},
'1.2.1': {
'enabled': {
'status': True
}
}
},
'plugin_labels': {
'enabled': {
'status': True
}
}
}
plugin1 = plugins.Plugin(plugins.PluginManager(None), plugin1_dict)
TEST.plugins.add(plugin1)
plugin_config1_dict = {
"node_processes": {
"HDFS": ["namenode", "datanode", "secondarynamenode"],
"MapReduce": ["tasktracker", "jobtracker"]
},
"description":
"This plugin provides an ability to launch vanilla "
"Apache Hadoop cluster without any management "
"consoles.",
"versions": ["1.2.1"],
"required_image_tags": ["vanilla", "1.2.1"],
"configs": [
{
"default_value": "/tmp/hadoop-${user.name}",
"name": "hadoop.tmp.dir",
"priority": 2,
"config_type": "string",
"applicable_target": "HDFS",
"is_optional": True,
"scope": "node",
"description": "A base for other temporary directories."
},
{
"default_value":
True,
"name":
"hadoop.native.lib",
"priority":
2,
"config_type":
"bool",
"applicable_target":
"HDFS",
"is_optional":
True,
"scope":
"node",
"description":
"Should native hadoop libraries, if present, "
"be used."
},
],
"title":
"Vanilla Apache Hadoop",
"name":
"vanilla",
'version_labels': {
'1.2.1': {
'enabled': {
'status': True
}
}
},
'plugin_labels': {
'enabled': {
'status': True
}
}
}
TEST.plugins_configs.add(
plugins.Plugin(plugins.PluginManager(None), plugin_config1_dict))
# Nodegroup_Templates.
ngt1_dict = {
"created_at":
"2014-06-04T14:01:03",
"description":
None,
"flavor_id":
"aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa",
"availability_zone":
None,
"floating_ip_pool":
None,
"auto_security_group":
True,
"hadoop_version":
"1.2.1",
"id":
"c166dfcc-9cc7-4b48-adc9-f0946169bb36",
"image_id":
None,
"name":
"sample-template",
"node_configs": {},
"node_processes":
["namenode", "jobtracker", "secondarynamenode", "hiveserver", "oozie"],
"plugin_name":
"vanilla",
"tenant_id":
"429ad8447c2d47bc8e0382d244e1d1df",
"updated_at":
None,
"volume_mount_prefix":
"/volumes/disk",
"volumes_per_node":
0,
"volumes_size":
0,
"volume_type":
None,
"volume_local_to_instance":
False,
"security_groups": [],
"volumes_availability_zone":
None,
"is_proxy_gateway":
False,
"use_autoconfig":
True,
"shares": [],
'is_public':
False,
'is_protected':
False,
}
ngt1 = node_group_templates.NodeGroupTemplate(
node_group_templates.NodeGroupTemplateManager(None), ngt1_dict)
TEST.nodegroup_templates.add(ngt1)
# Cluster_templates.
ct1_dict = {
"anti_affinity": [],
"cluster_configs": {},
"created_at":
"2014-06-04T14:01:06",
"default_image_id":
None,
"description":
"Sample description",
"hadoop_version":
"1.2.1",
"id":
"a2c3743f-31a2-4919-8d02-792138a87a98",
"name":
"sample-cluster-template",
"neutron_management_network":
None,
"use_autoconfig":
True,
'is_public':
False,
'is_protected':
False,
"node_groups": [{
"count":
1,
"created_at":
"2014-06-04T14:01:06",
"flavor_id":
"aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa",
"floating_ip_pool":
None,
"image_id":
None,
"name":
"master",
"node_configs": {},
"node_group_template_id":
"c166dfcc-9cc7-4b48-adc9",
"node_processes": [
"namenode", "jobtracker", "secondarynamenode", "hiveserver",
"oozie"
],
"updated_at":
None,
"volume_mount_prefix":
"/volumes/disk",
"volumes_per_node":
0,
"volumes_size":
0,
"volume_type":
None,
"volume_local_to_instance":
False,
"volumes_availability_zone":
None,
"use_autoconfig":
True,
"is_proxy_gateway":
False,
'is_public':
False,
'is_protected':
False,
}, {
"count": 2,
"created_at": "2014-06-04T14:01:06",
"flavor_id": "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa",
"floating_ip_pool": None,
"image_id": None,
"name": "workers",
"node_configs": {},
"node_group_template_id": "4eb5504c-94c9-4049-a440",
"node_processes": ["datanode", "tasktracker"],
"updated_at": None,
"volume_mount_prefix": "/volumes/disk",
"volumes_per_node": 0,
"volumes_size": 0,
"volume_type": None,
"volume_local_to_instance": False,
"volumes_availability_zone": None,
"use_autoconfig": True,
"is_proxy_gateway": False,
'is_public': False,
'is_protected': False,
}],
"shares": [],
"plugin_name":
"vanilla",
"tenant_id":
"429ad8447c2d47bc8e0382d244e1d1df",
"updated_at":
None,
"domain_name":
None
}
ct1 = cluster_templates.ClusterTemplate(
cluster_templates.ClusterTemplateManager(None), ct1_dict)
TEST.cluster_templates.add(ct1)
# Clusters.
cluster1_dict = {
"anti_affinity": [],
"cluster_configs": {},
"cluster_template_id":
"a2c3743f-31a2-4919-8d02-792138a87a98",
"created_at":
"2014-06-04T20:02:14",
"default_image_id":
"9eb4643c-dca8-4ea7-92d2-b773f88a8dc6",
"description":
"",
"hadoop_version":
"1.2.1",
"id":
"ec9a0d28-5cfb-4028-a0b5-40afe23f1533",
"info": {},
"is_transient":
False,
"management_public_key":
"fakekey",
"name":
"cercluster",
"neutron_management_network":
None,
"use_autoconfig":
True,
"is_public":
False,
"is_protected":
False,
"node_groups": [{
"count":
1,
"created_at":
"2014-06-04T20:02:14",
"flavor_id":
"0",
"floating_ip_pool":
None,
"image_id":
None,
"instances": [{
"created_at": "2014-06-04T20:02:14",
"id": "c3b8004b-7063-4b99-a082-820cdc6e961c",
"instance_id": "a45f5495-4a10-4f17-8fae",
"instance_name": "cercluster-master-001",
"internal_ip": None,
"management_ip": None,
"updated_at": None,
"volumes": []
}],
"name":
"master",
"node_configs": {},
"node_group_template_id":
"c166dfcc-9cc7-4b48-adc9",
"node_processes": [
"namenode", "jobtracker", "secondarynamenode", "hiveserver",
"oozie"
],
"updated_at":
"2014-06-04T20:02:14",
"volume_mount_prefix":
"/volumes/disk",
"volumes_per_node":
0,
"volumes_size":
0,
"volume_type":
None,
"volume_local_to_instance":
False,
"security_groups": [],
"volumes_availability_zone":
None,
"id":
"ng1",
"use_autoconfig":
True,
"is_proxy_gateway":
False
}, {
"count":
2,
"created_at":
"2014-06-04T20:02:14",
"flavor_id":
"0",
"floating_ip_pool":
None,
"image_id":
None,
"instances": [{
"created_at": "2014-06-04T20:02:15",
"id": "6a8ae0b1-bb28-4de2-bfbb-bdd3fd2d72b2",
"instance_id": "38bf8168-fb30-483f-8d52",
"instance_name": "cercluster-workers-001",
"internal_ip": None,
"management_ip": None,
"updated_at": None,
"volumes": []
}, {
"created_at": "2014-06-04T20:02:15",
"id": "17b98ed3-a776-467a-90cf-9f46a841790b",
"instance_id": "85606938-8e53-46a5-a50b",
"instance_name": "cercluster-workers-002",
"internal_ip": None,
"management_ip": None,
"updated_at": None,
"volumes": []
}],
"name":
"workers",
"node_configs": {},
"node_group_template_id":
"4eb5504c-94c9-4049-a440",
"node_processes": ["datanode", "tasktracker"],
"updated_at":
"2014-06-04T20:02:15",
"volume_mount_prefix":
"/volumes/disk",
"volumes_per_node":
0,
"volumes_size":
0,
"volume_type":
None,
"volume_local_to_instance":
False,
"security_groups": ["b7857890-09bf-4ee0-a0d5-322d7a6978bf"],
"volumes_availability_zone":
None,
"id":
"ng2",
"use_autoconfig":
True,
"is_proxy_gateway":
False
}],
"plugin_name":
"vanilla",
"status":
"Active",
"status_description":
"",
"tenant_id":
"429ad8447c2d47bc8e0382d244e1d1df",
"trust_id":
None,
"updated_at":
"2014-06-04T20:02:15",
"user_keypair_id":
"stackboxkp"
}
cluster1_dict.update({
'verification': {
'status':
'CHECKING',
'checks': [
{
'status': 'CHECKING',
'name': "Stupid check",
'description': "Stupid description",
"created_at": "2015-03-27T15:51:54",
"updated_at": "2015-03-27T15:59:34",
},
{
'status': 'RED',
'name': "Stupid check",
'description': "Houston, we have a problem",
"created_at": "2015-03-27T15:51:54",
"updated_at": "2015-03-27T15:59:34",
},
]
}
})
cluster1 = clusters.Cluster(clusters.ClusterManager(None), cluster1_dict)
TEST.clusters.add(cluster1)
cluster2_dict = copy.deepcopy(cluster1_dict)
cluster2_dict.update({
"id":
"cl2",
"name":
"cl2_name",
'verification': {
'status':
'RED',
'checks': [
{
'status': 'GREEN',
'name': "Stupid check",
'description': "Stupid description",
"created_at": "2015-03-27T15:51:54",
"updated_at": "2015-03-27T15:59:34",
},
{
'status': 'RED',
'name': "Stupid check",
'description': "Houston, we have a problem",
"created_at": "2015-03-27T15:51:54",
"updated_at": "2015-03-27T15:59:34",
},
]
},
"provision_progress": [{
"created_at": "2015-03-27T15:51:54",
"updated_at": "2015-03-27T15:59:34",
"step_name": "first_step",
"step_type": "some_type",
"successful": True,
"events": [],
"total": 3
}, {
"created_at":
"2015-03-27T16:01:54",
"updated_at":
"2015-03-27T16:10:22",
"step_name":
"second_step",
"step_type":
"some_other_type",
"successful":
None,
"events": [{
"id": "evt1",
"created_at": "2015-03-27T16:01:22",
"node_group_id": "ng1",
"instance_name": "cercluster-master-001",
"successful": True,
"event_info": None
}, {
"id": "evt2",
"created_at": "2015-03-27T16:04:51",
"node_group_id": "ng2",
"instance_name": "cercluster-workers-001",
"successful": True,
"event_info": None
}],
"total":
3
}]
})
cluster2 = clusters.Cluster(clusters.ClusterManager(None), cluster2_dict)
TEST.clusters.add(cluster2)
# Data Sources.
data_source1_dict = {
"created_at": "2014-06-04T14:01:10",
"description": "sample output",
"id": "426fb01c-5c7e-472d-bba2-b1f0fe7e0ede",
"name": "sampleOutput",
"tenant_id": "429ad8447c2d47bc8e0382d244e1d1df",
"type": "swift",
"updated_at": None,
"url": "swift://example.sahara/output",
'is_public': False,
'is_protected': False
}
data_source2_dict = {
"created_at": "2014-06-05T15:01:12",
"description": "second sample output",
"id": "ab3413-adfb-bba2-123456785675",
"name": "sampleOutput2",
"tenant_id": "429ad8447c2d47bc8e0382d244e1d1df",
"type": "hdfs",
"updated_at": None,
"url": "hdfs://example.sahara/output",
'is_public': False,
'is_protected': False
}
data_source1 = data_sources.DataSources(
data_sources.DataSourceManager(None), data_source1_dict)
data_source2 = data_sources.DataSources(
data_sources.DataSourceManager(None), data_source2_dict)
TEST.data_sources.add(data_source1)
TEST.data_sources.add(data_source2)
# Job Binaries.
job_binary1_dict = {
"created_at": "2014-06-05T18:15:15",
"description": "",
"id": "3f3a07ac-7d6f-49e8-8669-40b25ee891b7",
"name": "example.pig",
"tenant_id": "429ad8447c2d47bc8e0382d244e1d1df",
"updated_at": None,
"url": "internal-db://80121dea-f8bd-4ad3-bcc7-096f4bfc722d",
'is_public': False,
'is_protected': False
}
job_binary2_dict = {
"created_at": "2014-10-10T13:12:15",
"description": "Test for spaces in name",
"id": "abcdef56-1234-abcd-abcd-defabcdaedcb",
"name": "example with spaces.pig",
"tenant_id": "429ad8447c2d47bc8e0382d244e1d1df",
"updated_at": None,
"url": "internal-db://abcdef56-1234-abcd-abcd-defabcdaedcb",
'is_public': False,
'is_protected': False
}
job_binary3_dict = {
"created_at": "2015-10-10T13:12:15",
"description": "Test Manila",
"id": "abcdef56-1234-abcd-1345-defabcdaedcb",
"name": "manilashared",
"tenant_id": "429ad8447c2d47bc8e0382d244e1d1df",
"updated_at": None,
"url": "manila://tuvwxy56-1234-abcd-abcd-defabcdaedcb/testfile.bin"
}
job_binary1 = job_binaries.JobBinaries(
job_binaries.JobBinariesManager(None), job_binary1_dict)
job_binary2 = job_binaries.JobBinaries(
job_binaries.JobBinariesManager(None), job_binary2_dict)
job_binary3 = job_binaries.JobBinaries(
job_binaries.JobBinariesManager(None), job_binary3_dict)
TEST.job_binaries.add(job_binary1)
TEST.job_binaries.add(job_binary2)
TEST.job_binaries.add(job_binary3)
# Jobs.
job1_dict = {
"created_at":
"2014-06-05T19:23:59",
"description":
"",
"id":
"a077b851-46be-4ad7-93c3-2d83894546ef",
"libs": [{
"created_at": "2014-06-05T19:23:42",
"description": "",
"id": "ab140807-59f8-4235-b4f2-e03daf946256",
"name": "udf.jar",
"tenant_id": "429ad8447c2d47bc8e0382d244e1d1df",
"updated_at": None,
"url": "internal-db://d186e2bb-df93-47eb-8c0e-ce21e7ecb78b"
}],
"mains": [{
"created_at": "2014-06-05T18:15:15",
"description": "",
"id": "3f3a07ac-7d6f-49e8-8669-40b25ee891b7",
"name": "example.pig",
"tenant_id": "429ad8447c2d47bc8e0382d244e1d1df",
"updated_at": None,
"url": "internal-db://80121dea-f8bd-4ad3-bcc7-096f4bfc722d"
}],
"interface": [],
"name":
"pigjob",
"tenant_id":
"429ad8447c2d47bc8e0382d244e1d1df",
"type":
"Pig",
"updated_at":
None,
"job_config": {
"configs": {}
}
}
job1 = jobs.Job(jobs.JobsManager(None), job1_dict)
TEST.jobs.add(job1)
# Job Executions.
jobex1_dict = {
"cluster_id": "ec9a0d28-5cfb-4028-a0b5-40afe23f1533",
"created_at": "2014-06-05T20:03:06",
"end_time": None,
"id": "4b6c1cbf-c713-49d3-8025-808a87c514a6",
"info": {
"acl":
None,
"actions": [{
"consoleUrl": "-",
"cred": "None",
"data": None,
"endTime": "Thu,05 Jun 2014 20:03:32 GMT",
"errorCode": None,
"errorMessage": None,
"externalChildIDs": None,
"externalId": "-",
"externalStatus": "OK",
"id": "0000000-140604200538581-oozie-hado-W@:start:",
"name": ":start:",
"retries": 0,
"startTime": "Thu,05 Jun 2014 20:03:32 GMT",
"stats": None,
"status": "OK",
"toString": "Action name[:start:] status[OK]",
"trackerUri": "-",
"transition": "job-node",
"type": ":START:"
}, {
"consoleUrl": "fake://console.url",
"cred": "None",
"data": None,
"endTime": None,
"errorCode": None,
"errorMessage": None,
"externalChildIDs": None,
"externalId": "job_201406042004_0001",
"externalStatus": "RUNNING",
"id": "0000000-140604200538581-oozie-hado-W@job-node",
"name": "job-node",
"retries": 0,
"startTime": "Thu,05 Jun 2014 20:03:33 GMT",
"stats": None,
"status": "RUNNING",
"toString": "Action name[job-node] status[RUNNING]",
"trackerUri": "cercluster-master-001:8021",
"transition": None,
"type": "pig"
}],
"appName":
"job-wf",
"appPath":
"hdfs://fakepath/workflow.xml",
"conf":
"<configuration>fakeconfig</configuration>",
"consoleUrl":
"fake://consoleURL",
"createdTime":
"Thu,05 Jun 2014 20:03:32 GMT",
"endTime":
None,
"externalId":
None,
"group":
None,
"id":
"0000000-140604200538581-oozie-hado-W",
"lastModTime":
"Thu,05 Jun 2014 20:03:35 GMT",
"parentId":
None,
"run":
0,
"startTime":
"Thu,05 Jun 2014 20:03:32 GMT",
"status":
"RUNNING",
"toString":
"Workflow ...status[RUNNING]",
"user":
"******"
},
"input_id": "85884883-3083-49eb-b442-71dd3734d02c",
"job_configs": {
"args": [],
"configs": {},
"params": {}
},
"interface": {},
"job_id": "a077b851-46be-4ad7-93c3-2d83894546ef",
"oozie_job_id": "0000000-140604200538581-oozie-hado-W",
"output_id": "426fb01c-5c7e-472d-bba2-b1f0fe7e0ede",
"progress": None,
"return_code": None,
"start_time": "2014-06-05T16:03:32",
"tenant_id": "429ad8447c2d47bc8e0382d244e1d1df",
"updated_at": "2014-06-05T20:03:46",
"cluster_name_set": True,
"job_name_set": True,
"cluster_name": "cluster-1",
"job_name": "job-1",
"data_source_urls": {
"85884883-3083-49eb-b442-71dd3734d02c": "swift://a.sahara/input",
"426fb01c-5c7e-472d-bba2-b1f0fe7e0ede": "hdfs://a.sahara/output"
}
}
jobex1 = job_executions.JobExecution(
job_executions.JobExecutionsManager(None), jobex1_dict)
TEST.job_executions.add(jobex1)
augmented_image = TEST.registered_images.first()
augmented_image.tags = {}
augmented_image.username = '******'
augmented_image.description = 'mydescription'
job_type1_dict = {
"name":
"Pig",
"plugins": [
{
"description": "Fake description",
"versions": {
"2.6.0": {},
"1.2.1": {}
},
"name": "vanilla",
"title": "Vanilla Apache Hadoop"
},
]
}
job_types1 = job_types.JobType(job_types.JobTypesManager(None),
job_type1_dict)
TEST.job_types.add(job_types1)
def __init__(self,
username=None,
api_key=None,
project_id=None,
project_name=None,
auth_url=None,
sahara_url=None,
endpoint_type='publicURL',
service_type='data-processing',
input_auth_token=None,
session=None,
auth=None,
insecure=False,
cacert=None,
region_name=None,
**kwargs):
if not session:
warnings.simplefilter('once', category=DeprecationWarning)
warnings.warn(
'Passing authentication parameters to saharaclient '
'is deprecated. Please construct and pass an '
'authenticated session object directly.', DeprecationWarning)
warnings.resetwarnings()
if input_auth_token:
auth = token_endpoint.Token(sahara_url, input_auth_token)
else:
auth = self._get_keystone_auth(auth_url=auth_url,
username=username,
api_key=api_key,
project_id=project_id,
project_name=project_name)
verify = True
if insecure:
verify = False
elif cacert:
verify = cacert
session = keystone_session.Session(verify=verify)
if not auth:
auth = session.auth
# NOTE(Toan): bug #1512801. If sahara_url is provided, it does not
# matter if service_type is orthographically correct or not.
# Only find Sahara service_type and endpoint in Keystone catalog
# if sahara_url is not provided.
if not sahara_url:
service_type = self._determine_service_type(
session, auth, service_type, endpoint_type)
kwargs['user_agent'] = USER_AGENT
kwargs.setdefault('interface', endpoint_type)
kwargs.setdefault('endpoint_override', sahara_url)
client = HTTPClient(session=session,
auth=auth,
service_type=service_type,
region_name=region_name,
**kwargs)
self.clusters = clusters.ClusterManager(client)
self.cluster_templates = (
cluster_templates.ClusterTemplateManager(client))
self.node_group_templates = (
node_group_templates.NodeGroupTemplateManager(client))
self.plugins = plugins.PluginManager(client)
self.images = images.ImageManager(client)
self.data_sources = data_sources.DataSourceManager(client)
self.jobs = jobs.JobsManager(client)
self.job_executions = job_executions.JobExecutionsManager(client)
self.job_binaries = job_binaries.JobBinariesManager(client)
self.job_binary_internals = (
job_binary_internals.JobBinaryInternalsManager(client))
self.job_types = job_types.JobTypesManager(client)