def __init__(self, *args, **kwargs):
self.api_version = '1.1'
if 'api_version' in kwargs:
self.api_version = kwargs['api_version']
del kwargs['api_version']
self.sahara_client = sahara_client.Client(self.api_version, *args,
**kwargs)
def _setup_clients(self):
keystone = keystone_client.Client(
username=self.common_config.OS_USERNAME,
password=self.common_config.OS_PASSWORD,
tenant_name=self.common_config.OS_TENANT_NAME,
auth_url=self.common_config.OS_AUTH_URL)
keystone.management_url = self.common_config.OS_AUTH_URL
tenant_id = [
tenant.id for tenant in keystone.tenants.list()
if tenant.name == self.common_config.OS_TENANT_NAME
][0]
self.sahara = sahara_client.Client(
version=self.common_config.SAHARA_API_VERSION,
username=self.common_config.OS_USERNAME,
api_key=self.common_config.OS_PASSWORD,
project_name=self.common_config.OS_TENANT_NAME,
auth_url=self.common_config.OS_AUTH_URL,
sahara_url='http://%s:%s/v%s/%s' %
(self.common_config.SAHARA_HOST, self.common_config.SAHARA_PORT,
self.common_config.SAHARA_API_VERSION, tenant_id))
self.nova = nova_client.Client(
username=self.common_config.OS_USERNAME,
api_key=self.common_config.OS_PASSWORD,
project_id=self.common_config.OS_TENANT_NAME,
auth_url=self.common_config.OS_AUTH_URL)
self.neutron = neutron_client.Client(
username=self.common_config.OS_USERNAME,
password=self.common_config.OS_PASSWORD,
tenant_name=self.common_config.OS_TENANT_NAME,
auth_url=self.common_config.OS_AUTH_URL)
def _connect_to_sahara(*args, **kwargs):
"""
Recommend authenticating sahara with session auth
"""
try:
if "session" in kwargs:
session = kwargs.get("session")
sahara = sahara_client.Client("1.1",
session=session,
endpoint_type="internalURL")
else:
sahara = sahara_client.Client(*args, **kwargs)
except RuntimeError as client_failure:
if "Could not find Sahara endpoint" in client_failure.message:
return None
raise
return sahara
def create_client(self, version=None, service_type=None):
"""Return Sahara client."""
from saharaclient import client as sahara
client = sahara.Client(self.choose_version(version),
session=self.keystone.get_session()[0],
sahara_url=self._get_endpoint(service_type))
return client
def sahara(self, version="1.1"):
"""Return Sahara client."""
client = sahara.Client(version,
username=self.endpoint.username,
api_key=self.endpoint.password,
project_name=self.endpoint.tenant_name,
auth_url=self.endpoint.auth_url)
return client
def create_client(self, version=None):
"""Return Sahara client."""
from saharaclient import client as sahara
client = sahara.Client(self.choose_version(version),
**self._get_auth_info(
password_key="api_key",
project_name_key="project_name"))
return client
def sahara(self, version="1.1"):
"""Return Sahara client."""
from saharaclient import client as sahara
client = sahara.Client(
version,
**self._get_auth_info(password_key="api_key",
project_name_key="project_name"))
return client
def _create(self):
con = self.context
endpoint_type = self._get_client_option(CLIENT_NAME, 'endpoint_type')
args = {
'endpoint_type': endpoint_type,
'service_type': self.DATA_PROCESSING,
'session': con.keystone_session
}
client = sahara_client.Client('1.1', **args)
return client
def client(request):
endpoint_type = get_horizon_parameter('OPENSTACK_ENDPOINT_TYPE',
'internalURL')
auth_url = keystone._get_endpoint_url(request, endpoint_type)
return api_client.Client('1.1',
sahara_url=get_sahara_url(request),
service_type=SAHARA_SERVICE,
project_id=request.user.tenant_id,
input_auth_token=request.user.token.id,
auth_url=auth_url)
def create_client(self, version=None, service_type=None):
"""Return Sahara client."""
from saharaclient import client as sahara
client = sahara.Client(
self.choose_version(version),
service_type=self.choose_service_type(service_type),
insecure=self.credential.insecure,
**self._get_auth_info(password_key="api_key",
project_name_key="project_name"))
return client
def _create(self):
con = self.context
endpoint_type = self._get_client_option(CLIENT_NAME, 'endpoint_type')
args = {
'endpoint_type': endpoint_type,
'service_type': self.DATA_PROCESSING,
'session': con.keystone_session,
'connect_retries': cfg.CONF.client_retry_limit,
'region_name': self._get_region_name()
}
client = sahara_client.Client('1.1', **args)
return client
def setUp(self):
super(ITestCase, self).setUp()
self.common_config = cfg.ITConfig().common_config
self.vanilla_config = cfg.ITConfig().vanilla_config
self.vanilla_two_config = cfg.ITConfig().vanilla_two_config
self.hdp_config = cfg.ITConfig().hdp_config
self.idh2_config = cfg.ITConfig().idh2_config
self.idh3_config = cfg.ITConfig().idh3_config
telnetlib.Telnet(self.common_config.SAHARA_HOST,
self.common_config.SAHARA_PORT)
self.sahara = sahara_client.Client(
version=self.common_config.SAHARA_API_VERSION,
username=self.common_config.OS_USERNAME,
api_key=self.common_config.OS_PASSWORD,
project_name=self.common_config.OS_TENANT_NAME,
auth_url=self.common_config.OS_AUTH_URL,
sahara_url='http://%s:%s/v%s/%s' %
(self.common_config.SAHARA_HOST, self.common_config.SAHARA_PORT,
self.common_config.SAHARA_API_VERSION,
self.common_config.OS_TENANT_ID))
self.nova = nova_client.Client(
username=self.common_config.OS_USERNAME,
api_key=self.common_config.OS_PASSWORD,
project_id=self.common_config.OS_TENANT_NAME,
auth_url=self.common_config.OS_AUTH_URL)
self.neutron = neutron_client.Client(
username=self.common_config.OS_USERNAME,
password=self.common_config.OS_PASSWORD,
tenant_name=self.common_config.OS_TENANT_NAME,
auth_url=self.common_config.OS_AUTH_URL)
if not self.common_config.FLAVOR_ID:
self.flavor_id = self.nova.flavors.create(name='i-test-flavor-%s' %
str(uuid.uuid4())[:8],
ram=1024,
vcpus=1,
disk=10,
ephemeral=10).id
else:
self.flavor_id = self.common_config.FLAVOR_ID
if not self.common_config.PATH_TO_SSH_KEY:
self.common_config.USER_KEYPAIR_ID += str(uuid.uuid4())[:8]
self.private_key = self.nova.keypairs.create(
self.common_config.USER_KEYPAIR_ID).private_key
else:
self.private_key = open(self.common_config.PATH_TO_SSH_KEY).read()
def client(request):
try:
service_type = SAHARA_SERVICE
sahara_url = base.url_for(request, service_type)
except exceptions.ServiceCatalogException:
# if no endpoint found, fallback to the old service_type
service_type = SAHARA_SERVICE_FALLBACK
sahara_url = base.url_for(request, service_type)
return api_client.Client(VERSIONS.get_active_version()["version"],
sahara_url=sahara_url,
service_type=service_type,
project_id=request.user.project_id,
input_auth_token=request.user.token.id)
def _create(self):
con = self.context
endpoint_type = self._get_client_option('sahara', 'endpoint_type')
endpoint = self.url_for(service_type='data_processing',
endpoint_type=endpoint_type)
args = {
'service_type': 'data_processing',
'input_auth_token': self.auth_token,
'auth_url': con.auth_url,
'project_name': con.tenant,
'sahara_url': endpoint
}
client = sahara_client.Client('1.1', **args)
return client
def client(request):
insecure = getattr(settings, 'OPENSTACK_SSL_NO_VERIFY', False)
cacert = getattr(settings, 'OPENSTACK_SSL_CACERT', None)
auth = identity.Token(auth_url=request.user.endpoint,
token=request.user.token.id,
project_id=request.user.project_id)
verify = False
if cacert:
verify = cacert
elif not insecure:
verify = True
sess = session.Session(auth=auth, verify=verify)
return api_client.Client(VERSIONS.get_active_version()["version"],
service_type=SAHARA_SERVICE,
session=sess)
def _create(self):
con = self.context
endpoint_type = self._get_client_option(CLIENT_NAME, 'endpoint_type')
endpoint = self.url_for(service_type=self.DATA_PROCESSING,
endpoint_type=endpoint_type)
args = {
'service_type': self.DATA_PROCESSING,
'input_auth_token': self.auth_token,
'auth_url': con.auth_url,
'project_name': con.tenant,
'sahara_url': endpoint,
'insecure': self._get_client_option(CLIENT_NAME, 'insecure'),
'cacert': self._get_client_option(CLIENT_NAME, 'ca_file')
}
client = sahara_client.Client('1.1', **args)
return client
def client(request):
try:
service_type = SAHARA_SERVICE
sahara_url = base.url_for(request, service_type)
except exceptions.ServiceCatalogException:
# if no endpoint found, fallback to the old service_type
service_type = SAHARA_SERVICE_FALLBACK
sahara_url = base.url_for(request, service_type)
insecure = getattr(settings, 'OPENSTACK_SSL_NO_VERIFY', False)
cacert = getattr(settings, 'OPENSTACK_SSL_CACERT', None)
return api_client.Client(VERSIONS.get_active_version()["version"],
sahara_url=sahara_url,
service_type=service_type,
project_id=request.user.project_id,
input_auth_token=request.user.token.id,
insecure=insecure,
cacert=cacert)
def __init__(self, *args, **kwargs):
self.sahara_client = sahara_client.Client('1.1', *args, **kwargs)
from keystoneauth1.identity import v3
from keystoneauth1 import session
from keystoneclient.v3 import client as keystone_client
from saharaclient import client as sahara_client
auth = v3.Password(auth_url='http://192.168.51.5000/v3', username='******', password='******', project_name='admin', user_domain_name='default', project_domain_name='default')
sess = session.Session(auth=auth)
sahara_client.Client('1.1', session=sess)
auth = v3.Password(auth_url='http://192.168.51.71:5000/v3', username='******', password='******', project_name='admin', user_domain_name='default', project_domain_name='default')
sess = session.Session(auth=auth)
sahara_client.Client('1.1', session=sess)
sc = sahara_client.Client('1.1', session=sess)
def client(request):
return api_client.Client(VERSIONS.get_active_version()["version"],
sahara_url=base.url_for(request, SAHARA_SERVICE),
service_type=SAHARA_SERVICE,
project_id=request.user.project_id,
input_auth_token=request.user.token.id)
def resource_setup(cls):
super(BaseDataProcessingTest, cls).resource_setup()
endpoint_type = TEMPEST_CONF.data_processing.endpoint_type
catalog_type = TEMPEST_CONF.data_processing.catalog_type
auth_url = TEMPEST_CONF.identity.uri
credentials = cls.os_admin.credentials
auth = v3.Password(auth_url=auth_url.replace('/v2.0', '/v3'),
username=credentials.username,
password=credentials.password,
project_name=credentials.tenant_name,
user_domain_name='default',
project_domain_name='default')
ses = session.Session(auth=auth)
cls.client = sahara_client.Client(
TEMPEST_CONF.data_processing.saharaclient_version,
session=ses,
service_type=catalog_type,
endpoint_type=endpoint_type)
if TEMPEST_CONF.service_available.glance:
# Check if glance v1 is available to determine which client to use.
if TEMPEST_CONF.image_feature_enabled.api_v1:
cls.image_client = cls.os_admin.image_client
elif TEMPEST_CONF.image_feature_enabled.api_v2:
cls.image_client = cls.os_admin.image_client_v2
else:
raise lib_exc.InvalidConfiguration(
'Either api_v1 or api_v2 must be True in '
'[image-feature-enabled].')
cls.object_client = cls.os_primary.object_client
cls.container_client = cls.os_primary.container_client
cls.networks_client = cls.os_primary.compute_networks_client
if TEMPEST_CONF.network.floating_network_name:
cls.floating_ip_pool = TEMPEST_CONF.network.floating_network_name
if TEMPEST_CONF.service_available.neutron:
cls.floating_ip_pool = \
cls.get_floating_ip_pool_id_for_neutron()
else:
cls.floating_ip_pool = TEMPEST_CONF.network.public_network_id
test_image_name = TEMPEST_CONF.data_processing.test_image_name
cls.test_image_id = cls.get_image_id(test_image_name)
default_plugin = cls.get_plugin()
plugin_dict = default_plugin.to_dict()
default_version = plugin_utils.get_default_version(plugin_dict)
cls.worker_template = (plugin_utils.get_node_group_template(
'worker1', default_version, cls.floating_ip_pool))
cls.master_template = (plugin_utils.get_node_group_template(
'master1', default_version, cls.floating_ip_pool))
cls.cluster_template = (plugin_utils.get_cluster_template(
default_version=default_version))
cls.swift_data_source_with_creds = {
'url': 'swift://sahara-container/input-source',
'description': 'Test data source',
'type': 'swift',
'credentials': {
'user': '******',
'password': '******'
}
}
cls.local_hdfs_data_source = {
'url': 'input-source',
'description': 'Test data source',
'type': 'hdfs',
}
cls.external_hdfs_data_source = {
'url': 'hdfs://test-master-node/usr/hadoop/input-source',
'description': 'Test data source',
'type': 'hdfs'
}
from oslo_utils import importutils
AUTH_URL = 'http://192.168.104.100:5000/v2.0'
USERNAME = '******'
PASSWORD = '******'
PROJECT_ID = 'sln1'
auth = v2.Password(auth_url=AUTH_URL,
username=USERNAME,
password=PASSWORD,
tenant_name=PROJECT_ID)
ses = session.Session(auth=auth)
sahara = client.Client('1.1', session=ses)
#print(sahara.plugins.list()[0])
#print(sahara.node_group_templates.list()[0])
#print(sahara.cluster_templates.list()[0])
#print(sahara.clusters.list()[0])
#print(sahara.clusters.list(search_opts={'name':'test1'}))
#print(sahara.images.list({'tags': ['vanilla', '2.7.1']})[0])
#a = sahara.node_group_templates.create("tient", "vanilla", "2.7.1", "fb388319-518d-402c-a102-9d7ac6150836", node_processes=["namenode","resourcemanager"])
cluster_template = {
"plugin_name": "vanilla",
"hadoop_version": "2.7.1",
"node_groups": [
def resource_setup(cls):
cls.set_network_resources()
super(BaseDataProcessingTest, cls).resource_setup()
endpoint_type = TEMPEST_CONF.data_processing.endpoint_type
catalog_type = TEMPEST_CONF.data_processing.catalog_type
auth_url = TEMPEST_CONF.identity.uri
credentials = cls.credentials()
cls.client = sahara_client.Client(
CONF.data_processing.saharaclient_version,
credentials.username,
credentials.password,
project_name=credentials.tenant_name,
endpoint_type=endpoint_type,
service_type=catalog_type,
auth_url=auth_url,
sahara_url=CONF.data_processing.sahara_url)
cls.object_client = cls.manager.object_client
cls.container_client = cls.manager.container_client
cls.floating_ip_pool = CONF.data_processing.floating_ip_pool
if TEMPEST_CONF.service_available.neutron:
cls.floating_ip_pool = cls.get_floating_ip_pool_id_for_neutron()
cls.worker_template = {
'description': 'Test node group template',
'plugin_name': 'fake',
'hadoop_version': '0.1',
'node_processes': ['datanode', 'tasktracker'],
'flavor_id': CONF.data_processing.flavor_id,
'floating_ip_pool': cls.floating_ip_pool
}
cls.master_template = {
'description': 'Test node group template',
'plugin_name': 'fake',
'hadoop_version': '0.1',
'node_processes': ['namenode', 'jobtracker'],
'flavor_id': CONF.data_processing.flavor_id,
'floating_ip_pool': cls.floating_ip_pool,
'auto_security_group': True
}
cls.cluster_template = {
'description': 'Test cluster template',
'plugin_name': 'fake',
'hadoop_version': '0.1'
}
cls.swift_data_source_with_creds = {
'url': 'swift://sahara-container/input-source',
'description': 'Test data source',
'type': 'swift',
'credentials': {
'user': '******',
'password': '******'
}
}
cls.local_hdfs_data_source = {
'url': 'input-source',
'description': 'Test data source',
'type': 'hdfs',
}
cls.external_hdfs_data_source = {
'url': 'hdfs://test-master-node/usr/hadoop/input-source',
'description': 'Test data source',
'type': 'hdfs'
}
def get_sahara_client(self):
"""
Return Sahara client
"""
return sahara_client.Client('1.1', session=self.sess)
def sahara_create(num, env_config):
master_template_name = "masterTT"
cluster_template_name = "clusterTT"
worker_template_name = "workerTT"
worker_flavor_id = "c9a6f4cd-1a7c-451a-afe7-ee1358d2fd67"
availability_zone = "default_availability_zone"
worker_volume_size = 5
volumes_availability_zone = "default_availability_zone"
worker_volume_type = "ceph"
worker_count = 1
default_image_id = "c2b6210e-a421-4a4f-96c1-70f4be9a23cb"
networks = env_config['network_id']
bfc_volume_size = 60
bfc_volume_type = "ceph"
# 如果是bfc模式
dev_mapping = {
'device_name': 'vda',
'delete_on_termination': True,
'image_id': default_image_id,
'boot_index': 0,
'volume_size': bfc_volume_size,
'volume_type': bfc_volume_type
}
auth = v2.Password(auth_url=env_config['OS_AUTH_URL'],
username=env_config['USERNAME'],
password=env_config['PASSWORD'],
tenant_name=env_config['OPENSTACK_USER_TENANT'])
ses = session.Session(auth=auth)
sahara = sahara_client.Client('1.1',
session=ses,
region_name=env_config['OPENSTACK_REGION'])
SAHARA_NODE_GROUP_TEMPLATE_DICT = {
"vanilla": {
"master": ["namenode", "resourcemanager"],
"slave": ["nodemanager", "datanode"]
},
"spark": {
"master": ["master", "namenode"],
"slave": ["slave", "datanode"]
}
}
#
plugin_name = "vanilla" # 或者 "spark"
hadoop_version = "2.7.1" # 或者 spark时为"1.3"
master_node_group_template = sahara.node_group_templates.create(
master_template_name,
plugin_name,
hadoop_version,
worker_flavor_id,
node_processes=SAHARA_NODE_GROUP_TEMPLATE_DICT[plugin_name]['master'],
availability_zone=availability_zone)
print "master_node_group_template is: %s" % master_node_group_template
worker_node_group_template = sahara.node_group_templates.create(
worker_template_name,
plugin_name,
hadoop_version,
worker_flavor_id,
node_processes=SAHARA_NODE_GROUP_TEMPLATE_DICT[plugin_name]['slave'],
availability_zone=availability_zone,
volumes_size=worker_volume_size,
volumes_availability_zone=volumes_availability_zone,
volume_type=worker_volume_type,
volumes_per_node=1)
print "worker_node_group_template is: %s" % worker_node_group_template
cluster_template = sahara.cluster_templates.create(
cluster_template_name,
plugin_name,
hadoop_version,
node_groups=[{
"name": "worker",
"count": int(worker_count),
"node_group_template_id": worker_node_group_template.id
}, {
"name": "master",
"count": 1,
"node_group_template_id": master_node_group_template.id
}])
print "cluster_template is: %s" % cluster_template
# cluster_template = {'id': 'e6d4eddb-e688-48c8-beae-11ffac6f9c8b'}
for arg in xrange(1, 21):
name = 'saharaTT%s' % arg
new_cluster = sahara.clusters.create(
name,
plugin_name,
hadoop_version,
# cluster_template['id'],
cluster_template.id,
default_image_id,
block_device_mapping_v2=None, # 该参数不为none时为BFC创建
neutron_internal_network=networks)