def __init__(self, auth_config, env, frmt=u'json'):
ComponentManager.__init__(self, auth_config, env, frmt)
config = auth_config[u'environments'][env]
if config[u'endpoint'] is None:
raise Exception(u'Auth endpoint is not configured')
client_config = config.get(u'oauth2-client', None)
self.client = BeehiveApiClient(config[u'endpoint'],
config[u'authtype'],
config[u'user'],
config[u'pwd'],
config[u'catalog'],
client_config=client_config)
self.subsytem = None
self.baseuri = None
# get token
self.client.uid, self.client.seckey = self.get_token()
if self.client.uid is None:
# create token
self.client.create_token()
# set token
self.save_token(self.client.uid, self.client.seckey)
def setUp(self):
logging.getLogger(u'beehive.test')\
.info(u'========== %s ==========' % self.id()[9:])
self.start = time.time()
# ssl
path = os.path.dirname(__file__).replace(u'beehive/common',
u'beehive/tests')
pos = path.find(u'tests')
path = path[:pos + 6]
#keyfile = u'%s/ssl/nginx.key' % path
#certfile = u'%s/ssl/nginx.key' % path
keyfile = None
certfile = None
# load config
config = self.load_config(u'%s/params.json' % path)
#for k,v in self.load_config():
# setattr(self, k, v)
env = config.get(u'env')
current_user = config.get(u'user')
current_schema = config.get(u'schema')
cfg = config.get(env)
# endpoints
self.endpoints = cfg.get(u'endpoints')
# redis connection
self.redis_uri = cfg.get(u'redis-uri')
rhost, rport, db = self.redis_uri.split(u';')
self.redis = redis.StrictRedis(host=rhost, port=int(rport), db=int(db))
# celery broker
self.broker = cfg.get(u'broker')
# mysql connection
self.db_uri = cfg.get(u'db-uris').get(current_schema)
# get users
self.users = cfg.get(u'users')
self.user = self.users.get(current_user).get(u'user')
self.pwd = self.users.get(current_user).get(u'pwd')
self.ip = self.users.get(current_user).get(u'ip')
# create auth client
self.auth_client = BeehiveApiClient([], u'keyauth', None, None)
# create api endpoint
self.api = {}
for subsystem, endpoint in self.endpoints.items():
self.api[subsystem] = RemoteClient(endpoint,
keyfile=keyfile,
certfile=certfile)
def invoke_api(method, uri, token, seckey):
headers = {'Content-type': 'application/json'}
auth_client = BeehiveApiClient([], 'keyauth', None, '', None)
sign = auth_client.sign_request(seckey, uri)
headers.update({'uid': token, 'sign': sign})
res = requests.request(method,
'http://localhost:8080' + uri,
headers=headers,
timeout=5,
verify=False)
return res.json()
def setUp(self):
BeehiveTestCase.setUp(self)
global uid, seckey
endpoints = [self.endpoints.get(self.test_config['default-endpoint'])]
user = self.users.get('test3')
self.user_name = user.get('user')
self.pwd = user.get('pwd')
self.ip = user.get('ip')
self.catalog_id = user.get('catalog')
authtype = user.get('auth')
self.client = BeehiveApiClient(endpoints, authtype, self.user_name,
self.pwd, None, self.catalog_id)
if uid is not None:
self.client.uid = uid
self.client.seckey = seckey
class AuthJobTask(JobTask):
"""AuthJobTask class.
:param list args: Free job params passed as list
:param dict kwargs: Free job params passed as dict
"""
abstract = True
ops = [
User,
Role,
Group,
]
def __init__(self, *args, **kwargs):
JobTask.__init__(self, *args, **kwargs)
self.apiclient = BeehiveApiClient([], None, None, None, None)
def get_endpoints(self, oid=None):
"""Get all endpoints
"""
endpoints = task_local.controller.get_endpoints(oid=oid)
logger.debug('Get endpoints: %s' % endpoints)
return endpoints
def ping_endpoint(self, endpoint):
"""Ping endpoint
:param endpoint: CatalogEndpoint instance
"""
uri = endpoint.model.uri
res = self.apiclient.ping(endpoint=uri)
logger.warn('Ping endpoint %s: %s' % (uri, res))
return res
def setUp(self):
BeehiveTestCase.setUp(self)
global uid, seckey
endpoints = [
u'http://10.102.160.240:6060', u'http://10.102.160.240:60601',
u'http://10.102.160.240:60602'
]
endpoints = [
u'http://10.102.184.52:6060', u'http://10.102.184.53:6060'
]
user = u'admin@local'
pwd = u'testlab'
ip = u'127.0.0.1'
catalog_id = 1
self.client = BeehiveApiClient(endpoints, user, pwd, catalog_id)
if uid is not None:
self.client.uid = uid
self.client.seckey = seckey
class CatalogJobTask(JobTask):
"""CatalogJobTask class.
:param list args: Free job params passed as list
:param dict kwargs: Free job params passed as dict
"""
abstract = True
ops = [
Catalog,
CatalogEndpoint,
]
def __init__(self, *args, **kwargs):
JobTask.__init__(self, *args, **kwargs)
self.apiclient = BeehiveApiClient([], None, None, None, None)
def get_endpoints(self, oid=None):
"""Get all endpoints
"""
'''try:
endpoints = task_local.controller.manager.get_endpoints()
except:
endpoints = []
logger.debug('Get endpoints: %s' % endpoints)'''
endpoints = task_local.controller.get_endpoints(oid=oid)
logger.debug('Get endpoints: %s' % endpoints)
return endpoints
def ping_endpoint(self, endpoint):
"""Ping endpoint
:param endpoint: CatalogEndpoint instance
"""
uri = endpoint.model.uri
res = self.apiclient.ping(endpoint=uri)
logger.warn('Ping endpoint %s: %s' % (uri, res))
return res
def remove_endpoint(self, endpoint):
"""Remove endpoint
:param endpoint: CatalogEndpoint instance
"""
res = endpoint.delete()
logger.debug('Delete endpoint: %s' % endpoint.oid)
return res
class BeehiveApiClientTestCase(BeehiveTestCase):
"""
"""
def setUp(self):
BeehiveTestCase.setUp(self)
global uid, seckey
endpoints = [
u'http://10.102.160.240:6060', u'http://10.102.160.240:60601',
u'http://10.102.160.240:60602'
]
endpoints = [
u'http://10.102.184.52:6060', u'http://10.102.184.53:6060'
]
user = u'admin@local'
pwd = u'testlab'
ip = u'127.0.0.1'
catalog_id = 1
self.client = BeehiveApiClient(endpoints, user, pwd, catalog_id)
if uid is not None:
self.client.uid = uid
self.client.seckey = seckey
def tearDown(self):
BeehiveTestCase.tearDown(self)
def test_ping_subsystem(self):
self.client.load_catalog()
res = self.client.ping(subsystem=u'auth')
self.logger.info(self.pp.pformat(res))
def test_ping_endpoint(self):
res = self.client.ping(endpoint={
u'proto': u'http',
u'host': u'10.102.160.240',
u'port': 6060
})
self.logger.info(self.pp.pformat(res))
def test_login(self):
global uid, seckey
res = self.client.login()
uid = res[u'uid']
seckey = res[u'seckey']
self.logger.info(self.client.endpoints)
def test_exist(self):
global uid, seckey
res = self.client.exist(uid)
self.logger.info(res)
def test_logout(self):
global uid, seckey
res = self.client.logout()
self.logger.info(res)
def test_load_catalog(self):
global uid, seckey
res = self.client.load_catalog()
self.logger.info(res)
#
# catalogs
#
def test_get_catalogs(self):
global uid, seckey
for i in range(1, 20):
time.sleep(2)
res = self.client.get_catalogs()
print ''
print self.client.endpoints['auth']
self.logger.info(self.pp.pformat(res))
def test_get_catalog(self):
global uid, seckey
res = self.client.get_catalog(1)
self.logger.info(self.pp.pformat(res))
def test_create_catalog(self):
res = self.client.create_catalog(u'prova', u'internal')
self.logger.info(self.pp.pformat(res))
def test_delete_catalog(self):
catalog_id = 7
res = self.client.delete_catalog(catalog_id)
#
# endpoints
#
def test_get_endpoints(self):
global uid, seckey
res = self.client.get_endpoints()
self.logger.info(self.pp.pformat(res))
def test_get_endpoint(self):
global uid, seckey
res = self.client.get_endpoint(17)
self.logger.info(self.pp.pformat(res))
def test_create_endpoint(self):
name = u'prova'
service = u'auth'
uri = u'http://localhost:5000'
res = self.client.create_endpoint(2, name, service, uri)
self.logger.info(self.pp.pformat(res))
def test_delete_endpoint(self):
endpoint_id = 17
res = self.client.delete_endpoint(endpoint_id)
#
# resources
#
def test_list_resources(self):
global uid, seckey
res = self.client.invoke(u'resource1', u'/v1.0/resources/', u'get',
u'')
class BeehiveTestCase(unittest.TestCase):
"""To execute this test you need a mysql instance, a user and a
database associated to the user.
"""
logger = logging.getLogger(u'beehive.test')
pp = pprint.PrettyPrinter(width=200)
#credentials = u'%s:%s' % (user1, pwd1)
@classmethod
def setUpClass(cls):
pass
#cls._connection = createExpensiveConnectionObject()
@classmethod
def tearDownClass(cls):
pass
#cls._connection.destroy()
def load_config(self, file_config):
f = open(file_config, u'r')
config = f.read()
config = json.loads(config)
f.close()
return config
def setUp(self):
logging.getLogger(u'beehive.test')\
.info(u'========== %s ==========' % self.id()[9:])
self.start = time.time()
# ssl
path = os.path.dirname(__file__).replace(u'beehive/common',
u'beehive/tests')
pos = path.find(u'tests')
path = path[:pos + 6]
#keyfile = u'%s/ssl/nginx.key' % path
#certfile = u'%s/ssl/nginx.key' % path
keyfile = None
certfile = None
# load config
config = self.load_config(u'%s/params.json' % path)
#for k,v in self.load_config():
# setattr(self, k, v)
env = config.get(u'env')
current_user = config.get(u'user')
current_schema = config.get(u'schema')
cfg = config.get(env)
# endpoints
self.endpoints = cfg.get(u'endpoints')
# redis connection
self.redis_uri = cfg.get(u'redis-uri')
rhost, rport, db = self.redis_uri.split(u';')
self.redis = redis.StrictRedis(host=rhost, port=int(rport), db=int(db))
# celery broker
self.broker = cfg.get(u'broker')
# mysql connection
self.db_uri = cfg.get(u'db-uris').get(current_schema)
# get users
self.users = cfg.get(u'users')
self.user = self.users.get(current_user).get(u'user')
self.pwd = self.users.get(current_user).get(u'pwd')
self.ip = self.users.get(current_user).get(u'ip')
# create auth client
self.auth_client = BeehiveApiClient([], u'keyauth', None, None)
# create api endpoint
self.api = {}
for subsystem, endpoint in self.endpoints.items():
self.api[subsystem] = RemoteClient(endpoint,
keyfile=keyfile,
certfile=certfile)
def tearDown(self):
elapsed = round(time.time() - self.start, 4)
logging.getLogger(u'beehive.test')\
.info(u'========== %s ========== : %ss\n' % (self.id()[9:], elapsed))
def open_mysql_session(self, db_uri):
engine = create_engine(db_uri)
"""
engine = create_engine(app.db_uri,
pool_size=10,
max_overflow=10,
pool_recycle=3600)
"""
db_session = sessionmaker(bind=engine,
autocommit=False,
autoflush=False)
return db_session
def invoke(self,
api,
path,
method,
data=u'',
headers={},
filter=None,
auth_method=u'keyauth',
credentials=None):
"""Invoke api
"""
global uid, seckey
base_headers = {u'Accept': u'application/json'}
if auth_method == u'keyauth':
sign = self.auth_client.sign_request(seckey, path)
base_headers.update({u'uid': uid, u'sign': sign})
elif auth_method == u'simplehttp':
base_headers.update({
u'Authorization':
u'Basic %s' % b64encode(credentials.encode(u'utf-8'))
})
base_headers.update(headers)
if filter is not None:
if isinstance(filter, dict):
filter = urllib.urlencode(filter)
path = u'%s?%s' % (path, filter)
if isinstance(data, dict):
data = json.dumps(data)
res = self.api[api].run_http_request2(path,
method,
data=data,
headers=base_headers)
if res is not None:
return res[u'response']
def invoke_no_sign(self,
api,
path,
method,
data=u'',
headers={},
filter=None):
"""Invoke api without sign"""
base_headers = {u'Accept': u'application/json'}
base_headers.update(headers)
if filter is not None:
if isinstance(filter, dict):
filter = urllib.urlencode(filter)
path = u'%s?%s' % (path, filter)
res = self.api[api].run_http_request2(path,
method,
data=data,
headers=base_headers)
return res[u'response']
#
# keyauth
#
def test_login(self):
global uid, seckey
data = {
u'user': self.user,
u'password': self.pwd,
u'login_ip': self.ip
}
path = u'/v1.0/keyauth/login/'
base_headers = {u'Accept': u'application/json'}
res = self.api[u'auth'].run_http_request2(path,
u'POST',
data=json.dumps(data),
headers=base_headers)
#self.logger.info(json.dumps(res, indent=4))
res = res[u'response']
uid = res[u'uid']
seckey = res[u'seckey']
def test_logout(self):
self.invoke(u'auth', u'/v1.0/keyauth/logout/', u'DELETE', data='')
#
# simplehttp
#
def test_simple_http_login(self):
global uid, seckey
user = u'%s:%s' % (self.user, self.pwd)
path = u'/v1.0/simplehttp/login/'
base_headers = {
u'Accept': u'application/json',
}
data = {
u'user': self.user,
u'password': self.pwd,
u'login-ip': self.ip
}
res = self.api[u'auth'].run_http_request2(path,
u'POST',
data=json.dumps(data),
headers=base_headers)
res = res[u'response']
uid = None
seckey = None
def create_client(auth_config, format, args):
"""Create and init beehive client. Beehive client can need new:
- auth object types
- auth object and permissions
- auth roles
- auth users
"""
pp = PrettyPrinter(width=200)
res = []
# set operation user
operation.user = (auth_config[u'user'], u'localhost', None)
set_operation(classes=classes)
try:
file_config = args.pop(0)
except:
print(u'ERROR : Specify client config file')
logger.error(u'Specify client config file', exc_info=1)
return 1
# read subsystem config
config = __read_subsytem_config(file_config)
update = config.get(u'update', False)
# create api client instance
client = BeehiveApiClient(auth_config[u'endpoint'], auth_config[u'user'],
auth_config[u'pwd'], auth_config[u'catalog'])
name = config[u'name']
client_type = config[u'type']
# create object types
for o in config[u'object_types']:
try:
resp = client.add_object_types(client_type, o[0], u'')
res.append(u'Add object type %s %s' % (client_type, o[0]))
except BeehiveApiClientError as ex:
res.append(u'WARN: %s' % ex)
# create objects and related permissions
for o in config[u'objects']:
try:
resp = client.add_object(client_type, o[0], o[1] % name,
u'%s %s object' % (name, o[0]))
res.append(u'Add object %s %s %s' %
(client_type, o[0], o[1] % name))
except BeehiveApiClientError as ex:
res.append(u'WARN: %s' % ex)
# create roles
for o in config[u'roles']:
# create role
try:
resp = client.add_role(o[u'name'], o[u'description'])
res.append(u'Add role %s' % (o[u'name']))
except BeehiveApiClientError as ex:
res.append(u'WARN: %s' % ex)
# assign permisssions to role
for p in o[u'perms']:
try:
resp = client.append_role_permissions(o[u'name'], p[0], p[1],
p[2], p[3])
res.append(u'Add perms %s %s %s %s to role %s' %
(client_type, p[0], p[1], p[2], p[3]))
except BeehiveApiClientError as ex:
res.append(u'WARN: %s' % ex)
# create user
for o in config[u'users']:
# create user
try:
resp = client.add_user(o[u'name'], o[u'pwd'], o[u'desc'])
res.append(u'Add user %s' % (o[u'name']))
except BeehiveApiClientError as ex:
res.append(u'WARN: %s' % ex)
# assign permisssions to role
try:
resp = client.append_user_roles(o[u'name'], o[u'roles'])
res.append(u'Add roles %s to user %s' % (o[u'roles'], o[u'name']))
except BeehiveApiClientError as ex:
res.append(u'WARN: %s' % ex)
if format == u'text':
pass
else:
pp.pprint(res)
return 0
class ApiManager(ComponentManager):
def __init__(self, auth_config, env, frmt=u'json'):
ComponentManager.__init__(self, auth_config, env, frmt)
config = auth_config[u'environments'][env]
if config[u'endpoint'] is None:
raise Exception(u'Auth endpoint is not configured')
client_config = config.get(u'oauth2-client', None)
self.client = BeehiveApiClient(config[u'endpoint'],
config[u'authtype'],
config[u'user'],
config[u'pwd'],
config[u'catalog'],
client_config=client_config)
self.subsytem = None
self.baseuri = None
# get token
self.client.uid, self.client.seckey = self.get_token()
if self.client.uid is None:
# create token
self.client.create_token()
# set token
self.save_token(self.client.uid, self.client.seckey)
def _call(self, uri, method, data=u'', headers=None):
# make request
res = self.client.invoke(self.subsystem,
uri,
method,
data=data,
other_headers=headers,
parse=True)
if self.format == u'doc':
res = self.client.get_api_doc(self.subsystem,
uri,
method,
data=data,
sync=True,
title=u'',
desc=u'',
output=res)
#self.client.logout()
# set token
self.save_token(self.client.uid, self.client.seckey)
return res
def __query_task_status(self, task_id):
uri = u'/v1.0/worker/tasks/%s/' % task_id
res = self._call(uri, u'GET').get(u'task-instance')
#print res
#self.logger.info(res)
#resp = []
#resp.append(res)
#resp.extend(res.get(u'children'))
#self.result(resp, headers=[u'task_id', u'type', u'status', u'name',
# u'start_time', u'stop_time', u'elapsed'])
return res
def query_task_status(self, task_id):
while (True):
res = self.__query_task_status(task_id)
status = res[u'status']
print status
if status in [u'SUCCESS', u'FAILURE']:
break
sleep(1)
def load_config_file(self, filename):
"""
"""
f = open(filename, 'r')
config = f.read()
config = json.loads(config)
f.close()
return config
class BeehiveTestCase(unittest.TestCase):
"""Base beehive testunit class
"""
logger = logging.getLogger('beehive.test.log')
runlogger = logging.getLogger('beehive.test.run')
pp = pprint.PrettyPrinter(width=200)
logging.addLevelName(60, 'TESTPLAN')
logging.addLevelName(70, 'TEST')
# module = 'resource'
# module_prefix = 'nrs'
main_config_file = None
main_fernet_file = None
spec_config_file = None
validation_active = False
run_test_user = '******'
@classmethod
def setUpClass(cls):
logger.log(
60,
'#################### Testplan %s - START ####################' %
cls.__name__)
logging.getLogger('beehive.test.run')\
.log(60, '#################### Testplan %s - START ####################' % cls.__name__)
self = cls
# ssl
path = os.path.dirname(__file__).replace('beehive/common',
'beehive/tests')
pos = path.find('tests')
path = path[:pos + 6]
keyfile = None
certfile = None
# load configs
try:
home = os.path.expanduser('~')
if self.main_config_file is None:
config_file = '%s/beehive.yml' % home
self.main_config_file = config_file
else:
config_file = self.main_config_file
config = self.load_file(config_file)
logger.info('Get beehive test configuration: %s' % config_file)
except Exception as ex:
raise Exception(
'Error loading config file. Search in user home. %s' % ex)
# load configs fernet key
try:
home = os.path.expanduser('~')
if self.main_config_file is None:
config_file = '%s/beehive.fernet' % home
self.main_fernet_file = config_file
else:
config_file = self.main_config_file.replace('yml', 'fernet')
fernet = self.load_file(config_file)
logger.info('Get beehive test fernet key: %s' % config_file)
except Exception as ex:
raise Exception(
'Error loading fernet key file. Search in user home. %s' % ex)
# load specific configs for a set of test
try:
if self.spec_config_file is not None:
config2 = self.load_file(self.spec_config_file)
recursive_update(config, config2)
logger.info('Get beehive test specific configuration: %s' %
self.spec_config_file)
except Exception as ex:
raise Exception(
'Error loading config file. Search in user home. %s' % ex)
logger.info('Validation active: %s' % cls.validation_active)
cfg = config
self.test_config = config.get('configs', {})
if self.test_config.get('resource', None) is not None:
for key in self.test_config.get('resource').keys():
if 'configs' in cfg.keys() and 'resource' in cfg.get(
'configs').keys():
self.test_config.get('resource').get(key).update(
cfg.get('configs').get('resource').get(key, {}))
if 'configs' in cfg.keys() and 'container' in cfg.get(
'configs').keys():
self.test_config.get('container').update(
cfg.get('configs').get('container'))
self.fernet = fernet
# endpoints
self.endpoints = cfg.get('endpoints')
self.swagger_endpoints = cfg.get('swagger')
logger.info('Endpoints: %s' % self.endpoints)
# redis connection
if cfg.get('redis') is not None:
self.redis_uri = cfg.get('redis').get('uri')
if self.redis_uri is not None and self.redis_uri != '':
rhost, rport, db = self.redis_uri.split(';')
self.redis = redis.StrictRedis(host=rhost,
port=int(rport),
db=int(db))
# celery broker
self.worker = cfg.get('worker')
# mysql connection
self.db_uris = cfg.get('db-uris')
# get users
self.users = cfg.get('users')
# create auth client
self.auth_client = BeehiveApiClient([], 'keyauth', None, '', None)
# create api endpoint
self.api = {}
self.schema = {}
for subsystem, endpoint in self.endpoints.items():
self.api[subsystem] = RemoteClient(endpoint,
keyfile=keyfile,
certfile=certfile)
self.load_result()
self.custom_headers = {}
self.endpoit_service = 'auth'
@classmethod
def tearDownClass(cls):
cls.store_result()
logger.log(
60,
'#################### Testplan %s - STOP ####################' %
cls.__name__)
logging.getLogger('beehive.test.run')\
.log(60, '#################### Testplan %s - STOP ####################' % cls.__name__)
@classmethod
def load_config(cls, file_config):
f = open(file_config, 'r')
config = f.read()
config = json.loads(config)
f.close()
return config
@classmethod
def load_file(cls, file_config):
config = read_file(file_config)
return config
@classmethod
def store_result(cls):
global result
if len(result.keys()) > 0:
f = open('/tmp/test.result', 'w')
f.write(json.dumps(result))
f.close()
@classmethod
def load_result(cls):
global result
try:
f = open('/tmp/test.result', 'r')
config = f.read()
result = json.loads(config)
f.close()
except:
result = {}
def convert(self, data, separator='.'):
if isinstance(data, dict):
for k, v in data.items():
data[k] = self.convert(v, separator)
elif isinstance(data, list):
datal = []
for v in data:
datal.append(self.convert(v, separator))
data = datal
elif isinstance(data, str) or isinstance(data, unicode):
if data.find('$REF$') == 0:
data = dict_get(self.test_config, data.lstrip('$REF$'),
separator)
data = self.convert(data, separator)
return data
def conf(self, key, separator='.'):
res = dict_get(self.test_config, key, separator)
if isinstance(res, dict):
for k, v in res.items():
res[k] = self.convert(v, separator)
elif isinstance(res, list):
newres = []
for v in res:
newres.append(self.convert(v, separator))
res = newres
return res
def set_result(self, key, value):
global result
result[key] = value
def get_result(self, key):
global result
return result.get(key, None)
def setUp(self):
logger.log(70, '========== %s ==========' % self.id()[9:])
logging.getLogger('beehive.test.run').log(
70, '========== %s ==========' % self.id()[9:])
self.start = time.time()
def tearDown(self):
elapsed = round(time.time() - self.start, 4)
logger.log(70,
'========== %s ========== : %ss' % (self.id()[9:], elapsed))
logging.getLogger('beehive.test.run').log(
70, '========== %s ========== : %ss' % (self.id()[9:], elapsed))
def open_mysql_session(self, db_uri):
engine = create_engine(db_uri)
db_session = sessionmaker(bind=engine,
autocommit=False,
autoflush=False)
return db_session
def create_keyauth_token(self, user, pwd, timeout=5):
global token, seckey
data = {'user': user, 'password': pwd}
headers = {'Content-Type': 'application/json'}
endpoint = self.endpoints['auth']
uri = '/v1.0/nas/keyauth/token'
self.logger.debug('Request token to: %s' % endpoint + uri)
response = requests.request('post',
endpoint + uri,
data=json.dumps(data),
headers=headers,
timeout=timeout,
verify=False)
res = response.json()
self.logger.debug('Respone token: %s' % res)
if res.get('code', None) is not None:
raise Exception(res.get('message', ''))
token = res['access_token']
seckey = res['seckey']
self.logger.debug('Get access token to: %s' % token)
def validate_swagger_schema(self, endpoint, timeout=5):
start = time.time()
schema_uri = endpoint
response = requests.request('GET',
schema_uri,
timeout=timeout,
verify=False)
schema = load(response.text)
logger.info('Load swagger schema from %s: %ss' %
(endpoint, time.time() - start))
return schema
def get_schema(self, subsystem, endpoint, timeout=5):
if self.validation_active is True:
schema = self.schema.get(subsystem, None)
if schema is None:
self.logger.info('Load swagger schema from %s' % endpoint)
schema = self.validate_swagger_schema(endpoint,
timeout=timeout)
self.schema[subsystem] = schema
return schema
return None
def validate_response(self, resp_content_type, schema, path, method,
response, runlog):
validate = True
if self.validation_active is True:
# validate with swagger schema
if resp_content_type.find('application/json') >= 0:
validator = ApiValidator(schema, path, method)
validate = validator.validate(response)
if runlog is True:
self.runlogger.info('validate: %s' % validate)
else:
if runlog is True:
self.runlogger.warn(
'validation supported only for application/json')
validate = True
return validate
def call(self,
subsystem,
path,
method,
params=None,
headers=None,
user=None,
pwd=None,
auth=None,
data=None,
query=None,
runlog=True,
timeout=10,
oauth2_token=None,
response_size=400,
pretty_response=False,
*args,
**kvargs):
global token, seckey
start = time.time()
validate = False
res = None
try:
cred = None
uri = path
if params is not None:
uri = path.format(**params)
if data is not None:
data = json.dumps(data)
if headers is None:
headers = {}
endpoint = self.endpoints[subsystem]
swagger_endpoint = self.swagger_endpoints[subsystem]
schema = self.get_schema(subsystem,
swagger_endpoint,
timeout=timeout)
if 'Content-Type' not in headers:
headers['Content-Type'] = 'application/json'
if auth == 'oauth2' and oauth2_token is not None:
headers.update({'Authorization': 'Bearer %s' % oauth2_token})
elif user is not None and auth == 'simplehttp':
cred = HTTPBasicAuth(user, pwd)
logger.debug('Make simple http authentication: %s' %
time.time() - start)
elif user is not None and auth == 'keyauth':
if token is None:
self.create_keyauth_token(user, pwd, timeout=timeout)
logger.debug('Create keyauth token: %s - %s' %
(token, time.time() - start))
sign = self.auth_client.sign_request(seckey, uri)
headers.update({'uid': token, 'sign': sign})
# reset start after authentication
start = time.time()
if runlog is True:
self.runlogger.info('request endpoint: %s' % endpoint)
self.runlogger.info('request path: %s' % uri)
self.runlogger.info('request method: %s' % method)
self.runlogger.info('request user: %s' % user)
self.runlogger.info('request auth: %s' % auth)
self.runlogger.info('request params: %s' % params)
self.runlogger.info('request query: %s' % query)
self.runlogger.info('request data: %s' % data)
self.runlogger.info('request headers: %s' % headers)
# execute request
response = requests.request(method,
endpoint + uri,
auth=cred,
params=query,
data=data,
headers=headers,
timeout=timeout,
verify=False)
logger.info('Call api: %s' % response.url)
if runlog is True:
self.runlogger.info('request url: %s' % response.url)
self.runlogger.info('response headers: %s' % response.headers)
self.runlogger.info('response code: %s' %
response.status_code)
resp_content_type = response.headers['content-type']
# evaluate response status
# BAD_REQUEST 400 HTTP/1.1, RFC 2616, Section 10.4.1
if response.status_code == 400:
res = response.json().get('message')
raise BadRequestException(res)
# UNAUTHORIZED 401 HTTP/1.1, RFC 2616, Section 10.4.2
elif response.status_code == 401:
res = response.json().get('message')
raise UnauthorizedException(res)
# PAYMENT_REQUIRED 402 HTTP/1.1, RFC 2616, Section 10.4.3
# FORBIDDEN 403 HTTP/1.1, RFC 2616, Section 10.4.4
elif response.status_code == 403:
res = response.json().get('message')
raise ForbiddenException(res)
# NOT_FOUND 404 HTTP/1.1, RFC 2616, Section 10.4.5
elif response.status_code == 404:
res = response.json().get('message')
raise NotFoundException(res)
# METHOD_NOT_ALLOWED 405 HTTP/1.1, RFC 2616, Section 10.4.6
elif response.status_code == 405:
res = response.json().get('message')
raise MethodNotAllowedException(res)
# NOT_ACCEPTABLE 406 HTTP/1.1, RFC 2616, Section 10.4.7
elif response.status_code == 406:
res = response.json().get('message')
raise NotAcceptableException(res)
# PROXY_AUTHENTICATION_REQUIRED 407 HTTP/1.1, RFC 2616, Section 10.4.8
# REQUEST_TIMEOUT 408
elif response.status_code == 408:
raise TimeoutException('Timeout')
# CONFLICT 409
elif response.status_code == 409:
res = response.json().get('message')
raise ConflictException(res)
# UNSUPPORTED_MEDIA_TYPE 415
elif response.status_code == 415:
res = response.json().get('message')
raise UnsupporteMediaTypeException(res)
# INTERNAL SERVER ERROR 500
elif response.status_code == 500:
raise ServerErrorException('Internal server error')
# NO_CONTENT 204 HTTP/1.1, RFC 2616, Section 10.2.5
elif response.status_code == 204:
res = None
# OK 200 HTTP/1.1, RFC 2616, Section 10.2.1
# CREATED 201 HTTP/1.1, RFC 2616, Section 10.2.2
# ACCEPTED 202 HTTP/1.1, RFC 2616, Section 10.2.3
# NON_AUTHORITATIVE_INFORMATION 203 HTTP/1.1, RFC 2616, Section 10.2.4
# RESET_CONTENT 205 HTTP/1.1, RFC 2616, Section 10.2.6
# PARTIAL_CONTENT 206 HTTP/1.1, RFC 2616, Section 10.2.7
# MULTI_STATUS 207 WEBDAV RFC 2518, Section 10.2
elif re.match('20[0-9]+', str(response.status_code)):
if resp_content_type.find('application/json') >= 0:
res = response.json()
if runlog is True:
logger.debug(self.pp.pformat(res))
else:
logger.debug(truncate(res))
elif resp_content_type.find('application/xml') >= 0:
# res = xmltodict.parse(response.text, dict_constructor=dict)
res = response.text
if runlog is True:
logger.debug(res)
else:
logger.debug(truncate(res))
elif resp_content_type.find('text/xml') >= 0:
# res = xmltodict.parse(response.text, dict_constructor=dict)
res = response.text
else:
res = response.text
if runlog is True:
self.runlogger.info(
'response data: %s' %
truncate(response.text, size=response_size))
if pretty_response is True:
self.runlogger.debug(self.pp.pformat(res))
# validate with swagger schema
validate = self.validate_response(resp_content_type, schema, path,
method, response, runlog)
except:
logger.error('', exc_info=1)
if runlog is True:
self.runlogger.error('', exc_info=1)
raise
logger.debug('Call api elapsed: %s' % (time.time() - start))
self.assertEqual(validate, True)
return res
def get(self,
uri,
query=None,
params=None,
timeout=600,
user=None,
pretty_response=False,
runlog=True):
if user is None:
user = self.users[self.run_test_user]
res = self.call(self.endpoint_service,
uri,
'get',
data='',
query=query,
timeout=timeout,
params=params,
headers=self.custom_headers,
pretty_response=pretty_response,
runlog=runlog,
**user)
return res
def post(self,
uri,
data=None,
query=None,
params=None,
timeout=600,
user=None):
if user is None:
user = self.users[self.run_test_user]
res = self.call(self.endpoint_service,
uri,
'post',
data=data,
query=query,
params=params,
timeout=timeout,
headers=self.custom_headers,
**user)
if res is not None and 'jobid' in res:
self.wait_job(res['jobid'])
if res is not None and 'taskid' in res:
self.wait_task(res['taskid'])
return res
def put(self,
uri,
data=None,
query=None,
params=None,
timeout=600,
user=None):
if user is None:
user = self.users[self.run_test_user]
res = self.call(self.endpoint_service,
uri,
'put',
data=data,
query=query,
params=params,
timeout=timeout,
headers=self.custom_headers,
**user)
if res is not None and 'jobid' in res:
self.wait_job(res['jobid'])
return res
def patch(self,
uri,
data=None,
query=None,
params=None,
timeout=600,
user=None):
if user is None:
user = self.users[self.run_test_user]
res = self.call(self.endpoint_service,
uri,
'patch',
data=data,
query=query,
params=params,
timeout=timeout,
headers=self.custom_headers,
**user)
if res is not None and 'jobid' in res:
self.wait_job(res['jobid'])
return res
def delete(self,
uri,
data=None,
query=None,
params=None,
timeout=600,
user=None):
if user is None:
user = self.users[self.run_test_user]
res = self.call(self.endpoint_service,
uri,
'delete',
data=data,
query=query,
params=params,
timeout=timeout,
headers=self.custom_headers,
**user)
if res is not None and 'jobid' in res:
self.wait_job(res['jobid'])
return res
#
# job
#
def get_job_state(self, jobid):
try:
res = self.call(self.module,
'/v1.0/%s/worker/tasks/{oid}' % self.module_prefix,
'get',
params={'oid': jobid},
runlog=False,
**self.users[self.run_test_user])
job = res.get('task_instance')
state = job.get('status')
logger.debug('Get job %s state: %s' % (jobid, state))
if state == 'FAILURE':
for err in job.get('traceback', []):
self.runlogger.error(err.rstrip())
return state
except (NotFoundException, Exception):
return 'EXPUNGED'
def wait_job(self, jobid, delta=3, accepted_state='SUCCESS', maxtime=600):
"""Wait resource
"""
logger.info('wait for: %s' % jobid)
self.runlogger.info('wait for: %s' % jobid)
state = self.get_job_state(jobid)
elapsed = 0
while state not in ['SUCCESS', 'FAILURE']:
self.runlogger.info('.')
sleep(delta)
state = self.get_job_state(jobid)
if elapsed > maxtime and state != accepted_state:
state = TIMEOUT
self.assertEqual(state, accepted_state)
#
# task
#
def get_task_status(self, taskid):
try:
res = self.call(self.module,
'/v2.0/%s/worker/tasks/{oid}/status' %
self.module_prefix,
'get',
params={'oid': taskid},
runlog=True,
**self.users[self.run_test_user])
job = res.get('task_instance')
state = job.get('status')
logger.debug('Get task %s state: %s' % (taskid, state))
if state == 'FAILURE':
for err in job.get('traceback', []):
self.runlogger.error(err.rstrip())
return state
except (NotFoundException, Exception):
return 'EXPUNGED'
def wait_task(self,
taskid,
delta=3,
accepted_state='SUCCESS',
maxtime=600):
"""Wait resource
"""
logger.info('wait for: %s' % taskid)
self.runlogger.info('wait for: %s' % taskid)
state = self.get_task_status(taskid)
elapsed = 0
while state not in ['SUCCESS', 'FAILURE']:
self.runlogger.info('.')
sleep(delta)
state = self.get_task_status(taskid)
if elapsed > maxtime and state != accepted_state:
state = TIMEOUT
self.assertEqual(state, accepted_state)
def setup_param_for_runner(self, param):
param = '%s-%s' % (param, self.index)
return param
def create_subsystem(self, subsystem_config):
"""Create subsystem.
:param subsystem_config: subsystem configuration file
"""
res = []
# read subsystem config
config = self.read_config(subsystem_config)
subsystem = get_value(config, u'api_subsystem', None, exception=True)
update = get_value(config, u'update', False)
api_config = get_value(config, u'api', {})
self.logger.debug(api_config)
# set operation user
operation.user = (api_config.get(u'user', None), u'localhost', None)
self.set_permissions(classes=self.classes)
# init auth subsytem
if subsystem == u'auth':
res.extend(self.__configure(config, update=update))
res.extend(self.__init_subsystem(config, update=update))
# setup main kombu queue
# init oauth2 subsytem
elif subsystem == u'oauth2':
res.extend(self.__init_subsystem(config, update=update))
# init other subsystem
else:
# create api client instance
client = BeehiveApiClient(api_config[u'endpoint'],
u'keyauth',
api_config[u'user'],
api_config[u'pwd'],
api_config[u'catalog'])
# create super user
user = {u'name':u'%s_admin@local' % config[u'api_subsystem'],
u'pwd':random_password(20),
u'desc':u'%s internal user' % subsystem}
try:
client.add_system_user(user[u'name'],
password=user[u'pwd'],
description=u'User %s' % user[u'name'])
except BeehiveApiClientError as ex:
if ex.code == 409:
client.update_user(user[u'name'], user[u'name'], user[u'pwd'],
u'User %s' % user[u'name'])
else:
raise
if update is False:
# append system user config
config[u'config'].append({u'group':u'api',
u'name':u'user',
u'value':{u'name':user[u'name'],
u'pwd':user[u'pwd']}})
# append catalog config
config[u'config'].append({u'group':u'api',
u'name':u'catalog',
u'value':api_config[u'catalog']})
# append auth endpoints config
config[u'config'].append({u'group':u'api',
u'name':u'endpoints',
u'value':json.dumps(api_config[u'endpoint'])})
res.extend(self.__configure(config, update=update))
res.extend(self.__init_subsystem(config, update=update))
return res
def create_subsystem(self, subsystem_config, update=False):
"""Create subsystem.
:param subsystem_config: subsystem configuration file
:param update: if update is True don't replace database schema [default=False]
"""
res = []
# read subsystem config
config = read_file(subsystem_config)
subsystem = get_value(config, 'api_subsystem', None, exception=True)
api_config = get_value(config, 'api', {})
if update is True:
self.logger.info('Update %s subsystem' % subsystem)
else:
self.logger.info('Create new %s subsystem' % subsystem)
# set operation user
operation.user = (api_config.get('user', None), 'localhost', None)
operation.id = str(uuid4())
self.set_permissions(classes=self.classes)
# init auth subsytem
if subsystem == 'auth':
res.extend(self.__configure(config, update=update))
res.extend(self.__init_subsystem(config, update=update))
# setup main kombu queue
# init oauth2 subsytem
elif subsystem == 'oauth2':
res.extend(self.__init_subsystem(config, update=update))
# init other subsystem
else:
# create api client instance
client = BeehiveApiClient(api_config['endpoint'],
'keyauth',
api_config['user'],
api_config['pwd'],
None,
api_config['catalog'])
if update is False:
# create super user
user = {'name': '%s_admin@local' % config['api_subsystem'],
'pwd': random_password(20),
'desc': '%s internal user' % subsystem}
try:
client.add_system_user(user['name'], password=user['pwd'], desc='User %s' % user['name'])
except BeehiveApiClientError as ex:
if ex.code == 409:
client.update_user(user['name'], user['name'], user['pwd'], 'User %s' % user['name'])
else:
raise
# append system user config
config['config'].append({'group': 'api', 'name': 'user',
'value': {'name': user['name'], 'pwd': user['pwd']}})
# append catalog config
config['config'].append({'group': 'api', 'name': 'catalog', 'value': api_config['catalog']})
# append auth endpoints config
config['config'].append({'group': 'api', 'name': 'endpoints',
'value': json.dumps(api_config['endpoint'])})
res.extend(self.__configure(config, update=update))
res.extend(self.__init_subsystem(config, update=update))
return res
"favcolor": "red",
"ntpserver": "wolf.example.com",
"monitoring": "pack.example.com"
}
}
config = {
"endpoint": ["http://10.102.184.52:6060"],
"user": "******",
"pwd": "testlab",
"catalog": 3,
"provider": 13,
"super-zone": 2992
}
client = BeehiveApiClient(config[u'endpoint'], config[u'user'], config[u'pwd'],
config[u'catalog'])
uri = u'/v1.0/providers/%s/instances/' % config[u'provider']
data = u'super-zone=%s' % (config[u'super-zone'])
instances = client.invoke(u'resource', uri, u'GET', data=data,
parse=True)[u'instances']
inventory = {}
for instance in instances:
ip = instance[u'networks'][0][u'ip']
if ip is not None:
try:
inventory[instance[u'parent_name']][u'hosts'].append(ip)
except:
inventory[instance[u'parent_name']] = {u'hosts': [ip]}
def __init__(self, *args, **kwargs):
JobTask.__init__(self, *args, **kwargs)
self.apiclient = BeehiveApiClient([], None, None, None)
def setUpClass(cls):
logger.log(
60,
'#################### Testplan %s - START ####################' %
cls.__name__)
logging.getLogger('beehive.test.run')\
.log(60, '#################### Testplan %s - START ####################' % cls.__name__)
self = cls
# ssl
path = os.path.dirname(__file__).replace('beehive/common',
'beehive/tests')
pos = path.find('tests')
path = path[:pos + 6]
keyfile = None
certfile = None
# load configs
try:
home = os.path.expanduser('~')
if self.main_config_file is None:
config_file = '%s/beehive.yml' % home
self.main_config_file = config_file
else:
config_file = self.main_config_file
config = self.load_file(config_file)
logger.info('Get beehive test configuration: %s' % config_file)
except Exception as ex:
raise Exception(
'Error loading config file. Search in user home. %s' % ex)
# load configs fernet key
try:
home = os.path.expanduser('~')
if self.main_config_file is None:
config_file = '%s/beehive.fernet' % home
self.main_fernet_file = config_file
else:
config_file = self.main_config_file.replace('yml', 'fernet')
fernet = self.load_file(config_file)
logger.info('Get beehive test fernet key: %s' % config_file)
except Exception as ex:
raise Exception(
'Error loading fernet key file. Search in user home. %s' % ex)
# load specific configs for a set of test
try:
if self.spec_config_file is not None:
config2 = self.load_file(self.spec_config_file)
recursive_update(config, config2)
logger.info('Get beehive test specific configuration: %s' %
self.spec_config_file)
except Exception as ex:
raise Exception(
'Error loading config file. Search in user home. %s' % ex)
logger.info('Validation active: %s' % cls.validation_active)
cfg = config
self.test_config = config.get('configs', {})
if self.test_config.get('resource', None) is not None:
for key in self.test_config.get('resource').keys():
if 'configs' in cfg.keys() and 'resource' in cfg.get(
'configs').keys():
self.test_config.get('resource').get(key).update(
cfg.get('configs').get('resource').get(key, {}))
if 'configs' in cfg.keys() and 'container' in cfg.get(
'configs').keys():
self.test_config.get('container').update(
cfg.get('configs').get('container'))
self.fernet = fernet
# endpoints
self.endpoints = cfg.get('endpoints')
self.swagger_endpoints = cfg.get('swagger')
logger.info('Endpoints: %s' % self.endpoints)
# redis connection
if cfg.get('redis') is not None:
self.redis_uri = cfg.get('redis').get('uri')
if self.redis_uri is not None and self.redis_uri != '':
rhost, rport, db = self.redis_uri.split(';')
self.redis = redis.StrictRedis(host=rhost,
port=int(rport),
db=int(db))
# celery broker
self.worker = cfg.get('worker')
# mysql connection
self.db_uris = cfg.get('db-uris')
# get users
self.users = cfg.get('users')
# create auth client
self.auth_client = BeehiveApiClient([], 'keyauth', None, '', None)
# create api endpoint
self.api = {}
self.schema = {}
for subsystem, endpoint in self.endpoints.items():
self.api[subsystem] = RemoteClient(endpoint,
keyfile=keyfile,
certfile=certfile)
self.load_result()
self.custom_headers = {}
self.endpoit_service = 'auth'
class BeehiveApiClientTestCase(BeehiveTestCase):
"""
"""
def setUp(self):
BeehiveTestCase.setUp(self)
global uid, seckey
endpoints = [self.endpoints.get(self.test_config['default-endpoint'])]
user = self.users.get('test3')
self.user_name = user.get('user')
self.pwd = user.get('pwd')
self.ip = user.get('ip')
self.catalog_id = user.get('catalog')
authtype = user.get('auth')
self.client = BeehiveApiClient(endpoints, authtype, self.user_name,
self.pwd, None, self.catalog_id)
if uid is not None:
self.client.uid = uid
self.client.seckey = seckey
def tearDown(self):
BeehiveTestCase.tearDown(self)
def test_ping_subsystem(self):
self.client.load_catalog()
res = self.client.ping(subsystem='auth')
self.logger.info(self.pp.pformat(res))
def test_ping_endpoint(self):
res = self.client.ping(endpoint={
'proto': 'http',
'host': '10.102.184.69',
'port': 6060
})
self.logger.info(self.pp.pformat(res))
def test_create_keyauth_token(self):
global uid, seckey
res = self.client.create_token(api_user=self.user_name,
api_user_pwd=self.pwd,
login_ip=self.ip)
uid = res['access_token']
seckey = res['seckey']
self.logger.info(self.client.endpoints)
def test_exist(self):
global uid, seckey
res = self.client.exist(uid)
self.logger.info(res)
def test_load_catalog(self):
global uid, seckey
res = self.client.load_catalog()
self.logger.info(res)
#
# catalogs
#
def test_get_catalogs(self):
global uid, seckey
for i in range(1, 20):
time.sleep(2)
res = self.client.get_catalogs()
self.logger.info(self.pp.pformat(res))
def test_get_catalog(self):
global uid, seckey
res = self.client.get_catalog(self.catalog_id)
self.logger.info(self.pp.pformat(res))
def test_create_catalog(self):
res = self.client.create_catalog('prova', 'internal')
self.logger.info(self.pp.pformat(res))
def test_delete_catalog(self):
catalog_id = 7
res = self.client.delete_catalog(catalog_id)
#
# endpoints
#
def test_get_endpoints(self):
global uid, seckey
res = self.client.get_endpoints()
self.logger.info(self.pp.pformat(res))
def test_get_endpoint(self):
global uid, seckey
res = self.client.get_endpoint(17)
self.logger.info(self.pp.pformat(res))
def test_create_endpoint(self):
name = 'prova'
service = 'auth'
uri = 'http://localhost:5000'
res = self.client.create_endpoint(2, name, service, uri)
self.logger.info(self.pp.pformat(res))
def test_delete_endpoint(self):
endpoint_id = 17
res = self.client.delete_endpoint(endpoint_id)
#
# resources
#
def test_list_resources(self):
global uid, seckey
res = self.client.invoke('resource1', '/v1.0/resources/', 'get', '')
def create_main(auth_config, format, args):
"""
"""
pp = PrettyPrinter(width=200)
res = []
# set operation user
operation.user = (auth_config[u'user'], u'localhost', None)
set_operation(classes=classes)
try:
subsystem = args.pop(0)
except:
print(u'ERROR : Specify subsystem')
logger.error(u'Specify subsystem', exc_info=1)
return 1
try:
file_config = args.pop(0)
except:
print(u'ERROR : Specify subsystem config file')
logger.error(u'Specify subsystem config file', exc_info=1)
return 1
# read subsystem config
config = __read_subsytem_config(file_config)
update = config.get(u'update', False)
# init auth subsytem
if subsystem == u'auth':
res.extend(configure(config, update=update))
res.extend(init_subsystem(config, update=update))
# init other subsystem
else:
# create api client instance
client = BeehiveApiClient(auth_config[u'endpoint'],
auth_config[u'user'], auth_config[u'pwd'],
auth_config[u'catalog'])
# create super user
user = {
u'name': u'%s_admin@local' % config[u'api_subsystem'],
u'pwd': random_password(20),
u'desc': u'%s internal user' % subsystem
}
try:
client.add_system_user(user[u'name'],
password=user[u'pwd'],
description=u'User %s' % user[u'name'])
except BeehiveApiClientError as ex:
if ex.code == 409:
client.update_user(user[u'name'], user[u'name'], user[u'pwd'],
u'User %s' % user[u'name'])
else:
raise
if update is False:
# append system user config
config[u'config'].append({
u'group': u'auth',
u'name': u'api_user',
u'value': {
u'name': user[u'name'],
u'pwd': user[u'pwd']
}
})
# append catalog config
config[u'config'].append({
u'group': u'auth',
u'name': u'catalog',
u'value': auth_config[u'catalog']
})
# append auth endpoints config
config[u'config'].append({
u'group':
u'auth',
u'name':
u'endpoints',
u'value':
json.dumps(auth_config[u'endpoint'])
})
res.extend(configure(config, update=update))
res.extend(init_subsystem(config, update=update))
if format == u'text':
pass
else:
pp.pprint(res)
return 0