def start(self):
# Connect to rabbitmq for config update notifications
rabbitmq_qname = self._service_id
self._vnc_amqp = VncAmqpHandle(self._sandesh, self._logger,
self._db_cls, self._reaction_map,
self._service_id, self._rabbitmq_cfg)
self._vnc_amqp.establish()
cassandra_credential = {
'username': self._cassandra_cfg['user'],
'password': self._cassandra_cfg['password']
}
if not all(cassandra_credential.values()):
cassandra_credential = None
try:
self._vnc_db = VncObjectDBClient(self._cassandra_cfg['servers'],
self._cassandra_cfg['cluster_id'],
logger=self._logger.log,
credential=cassandra_credential)
except Exception as e:
template = 'Exception {0} connecting to Config DB. Arguments:\n{1!r}'
msg = template.format(type(e).__name__, e.args)
self._logger.error('%s: %s' % (msg, traceback.format_exc()))
exit()
self._db_cls.init(self, self._logger, self._vnc_db)
self._sync_config_db()
def initialize_db_connection(logger, args):
credential = None
if args.cassandra_user and args.cassandra_password:
credential = {
'username': args.cassandra_user,
'password': args.cassandra_password
}
timeout = int(args.job_manager_db_conn_retry_timeout)
max_retries = int(args.job_manager_db_conn_max_retries)
retry_count = 1
while True:
try:
return VncObjectDBClient(args.cassandra_server_list,
db_prefix=args.cluster_id,
logger=logger.log,
credential=credential,
ssl_enabled=args.cassandra_use_ssl,
ca_certs=args.cassandra_ca_certs)
except Exception as e:
if retry_count >= max_retries:
raise e
logger.error("Error while initializing db connection, "
"retrying: %s" % str(e))
gevent.sleep(timeout)
finally:
retry_count = retry_count + 1
def __init__(self, force, resources_file, cassandra_servers,
cassandra_username, cassandra_password, cassandra_use_ssl,
cassandra_ca_certs, db_prefix, cassandra_batch_size,
zookeeper_servers, rules_per_security_group, keystone_client,
dont_populate_zookeeper):
self._force = force
self._resource_distribution = yaml.load(resources_file)
self._cassandra_batch_size = cassandra_batch_size
self._rules_per_security_group = rules_per_security_group
self._keystone_client = keystone_client
self._dont_populate_zookeeper = dont_populate_zookeeper
# Connect to cassandra database
logger.debug("Initilizing the cassandra connection on %s",
cassandra_servers)
cassandra_credentials = {}
if (cassandra_username is not None and cassandra_password is not None):
cassandra_credentials = {
'username': cassandra_username,
'password': cassandra_password,
}
def vnc_cassandra_client_logger(msg, level=logging.INFO):
logger.log(msg=msg, level=level)
self._object_db = VncObjectDBClient(cassandra_servers,
db_prefix,
self._UUID_KEYSPACE,
None,
vnc_cassandra_client_logger,
credential=cassandra_credentials,
ssl_enabled=cassandra_use_ssl,
ca_certs=cassandra_ca_certs)
self._uuid_cf = self._object_db.get_cf('obj_uuid_table')
self._fqname_cf = self._object_db.get_cf('obj_fq_name_table')
# Initilize zookeeper client
if self._dont_populate_zookeeper:
self._zk_client = DummyZookeeperClient()
else:
self._zk_client = ZookeeperClient(zookeeper_servers)
def _initialize_db_client(self, job_ctx):
db_init_params = job_ctx.get('db_init_params')
cred = None
if (db_init_params.get('cassandra_user') is not None and
db_init_params.get('cassandra_password') is not None):
cred = {'username': db_init_params.get('cassandra_user'),
'password': db_init_params.get('cassandra_password')}
if self.db_client is None:
self.db_client = VncObjectDBClient(
db_init_params.get('cassandra_server_list'),
job_ctx.get('cluster_id'), None, None,
logger=self._db_logger.log, credential=cred,
ssl_enabled=db_init_params.get('cassandra_use_ssl'),
ca_certs=db_init_params.get('cassandra_ca_certs'))
def __init__(self, force, resources_file, cassandra_servers,
cassandra_username, cassandra_password,
cassandra_use_ssl, cassandra_ca_certs, db_prefix,
cassandra_batch_size, zookeeper_servers,
rules_per_security_group, keystone_client,
dont_populate_zookeeper):
self._force = force
self._resource_distribution = yaml.load(resources_file)
self._cassandra_batch_size = cassandra_batch_size
self._rules_per_security_group = rules_per_security_group
self._keystone_client = keystone_client
self._dont_populate_zookeeper = dont_populate_zookeeper
# Connect to cassandra database
logger.debug("Initilizing the cassandra connection on %s",
cassandra_servers)
cassandra_credentials = {}
if (cassandra_username is not None and
cassandra_password is not None):
cassandra_credentials = {
'username': cassandra_username,
'password': cassandra_password,
}
def vnc_cassandra_client_logger(msg, level=logging.INFO):
logger.log(msg=msg, level=level)
self._object_db = VncObjectDBClient(
cassandra_servers,
db_prefix,
self._UUID_KEYSPACE,
None,
vnc_cassandra_client_logger,
credential=cassandra_credentials,
ssl_enabled=cassandra_use_ssl,
ca_certs=cassandra_ca_certs)
self._uuid_cf = self._object_db.get_cf('obj_uuid_table')
self._fqname_cf = self._object_db.get_cf('obj_fq_name_table')
# Initilize zookeeper client
if self._dont_populate_zookeeper:
self._zk_client = DummyZookeeperClient()
else:
self._zk_client = ZookeeperClient(zookeeper_servers)
def start(self):
# Connect to rabbitmq for config update notifications
rabbitmq_qname = self._service_id
self._vnc_amqp = VncAmqpHandle(self._sandesh, self._logger,
self._db_cls, self._reaction_map,
self._service_id, self._rabbitmq_cfg)
self._vnc_amqp.establish()
cassandra_credential = {
'username': self._cassandra_cfg['user'],
'password': self._cassandra_cfg['password']
}
if not all(cassandra_credential.values()):
cassandra_credential = None
self._vnc_db = VncObjectDBClient(self._cassandra_cfg['servers'],
self._cassandra_cfg['cluster_id'],
logger=self._logger.log,
credential=cassandra_credential)
self._db_cls.init(self, self._logger, self._vnc_db)
self._sync_config_db()
class LoadDataBase(object):
_UUID_KEYSPACE_NAME = 'config_db_uuid'
_OBJ_UUID_CF_NAME = 'obj_uuid_table'
_OBJ_FQ_NAME_CF_NAME = 'obj_fq_name_table'
_OBJ_SHARED_CF_NAME = 'obj_shared_table'
_UUID_KEYSPACE = {
_UUID_KEYSPACE_NAME: {
_OBJ_UUID_CF_NAME: {
'cf_args': {
'autopack_names': False,
'autopack_values': False,
},
},
_OBJ_FQ_NAME_CF_NAME: {
'cf_args': {
'autopack_values': False,
},
},
_OBJ_SHARED_CF_NAME: {}
}
}
# Resources supported by that script
# The order of that list is import, that defines the resources
# order creation
_SUPPORTED_RESOURCES = [
'project',
'security-group',
'virtual-network',
'virtual-machine-interface',
]
_PERMS2 = {
'owner': None,
'owner_access': 7,
'global_access': 0,
'share': [],
}
BATCH_QUEUE_SIZE = 1000
RULES_PER_SG = 4
def __init__(self, force, resources_file, cassandra_servers,
cassandra_username, cassandra_password, db_prefix,
cassandra_batch_size, zookeeper_servers,
rules_per_security_group, keystone_client,
dont_populate_zookeeper):
self._force = force
self._resource_distribution = yaml.load(resources_file)
self._cassandra_batch_size = cassandra_batch_size
self._rules_per_security_group = rules_per_security_group
self._keystone_client = keystone_client
self._dont_populate_zookeeper = dont_populate_zookeeper
# Connect to cassandra database
logger.debug("Initilizing the cassandra connection on %s",
cassandra_servers)
cassandra_credentials = {}
if (cassandra_username is not None and
cassandra_password is not None):
cassandra_credentials = {
'username': cassandra_username,
'password': cassandra_password,
}
def vnc_cassandra_client_logger(msg, level=logging.INFO):
logger.log(msg=msg, level=level)
self._object_db = VncObjectDBClient(
cassandra_servers,
db_prefix,
self._UUID_KEYSPACE,
None,
vnc_cassandra_client_logger,
credential=cassandra_credentials)
self._uuid_cf = self._object_db.get_cf('obj_uuid_table')
self._fqname_cf = self._object_db.get_cf('obj_fq_name_table')
# Initilize zookeeper client
if self._dont_populate_zookeeper:
self._zk_client = DummyZookeeperClient()
else:
self._zk_client = ZookeeperClient(zookeeper_servers)
def sanitize_resources(self):
logger.debug("Santizing resources distribution")
self._resource_map = OrderedDict()
for resource_type in self._SUPPORTED_RESOURCES:
object_path = 'contrail_db_loader.resources.%s.%s' %\
(resource_type.replace('-', '_'),
camel_case(resource_type))
kwargs = {
'db_manager': self._object_db,
'batch_size': self._cassandra_batch_size,
'zk_client': self._zk_client,
'project_amount': self._resource_distribution.get('project',
0),
'amount_per_project': self._resource_distribution.get(
resource_type, 0),
}
self._resource_map[resource_type] = import_object(object_path,
**kwargs)
resources_not_supported = (set(self._resource_distribution.keys()) -
set(self._SUPPORTED_RESOURCES))
if resources_not_supported:
logger.warning('Loading resources %s are not supported' %
', '.join(resources_not_supported))
def summarize_resources_to_create(self):
msg = """Will populate %(project)d projects with:
- security groups: %(sg)d
- access control lists: %(acl)d
- virtual networks: %(vn)d
- routing instances: %(ri)d
- route targets: %(rt)d
- virtual machine interface: %(vmi)d
- virtual machine: %(vm)d
- intance ip: %(iip)d
That will load %(sum)d resources into database."""
dict = {
'project': self._resource_map['project'].total_amount,
'sg': self._resource_map['security-group'].amount_per_project + 1,
'acl': (self._resource_map['security-group'].amount_per_project +
1) * 2,
'vn': self._resource_map['virtual-network'].amount_per_project,
'ri': self._resource_map['virtual-network'].amount_per_project,
'rt': self._resource_map['virtual-network'].amount_per_project,
'vmi': self._resource_map['virtual-machine-interface'].
amount_per_project,
'vm': self._resource_map['virtual-machine-interface'].
amount_per_project,
'iip': self._resource_map['virtual-machine-interface'].
amount_per_project,
}
dict['sum'] = 0
for resource in self._resource_map.values():
dict['sum'] += resource.total_amount
logger.warning(msg, dict)
if (not self._force and
not prompt('Do you want to load that amount of resources?')):
exit(0)
def create_resources(self):
self._zk_client.connect()
for resource in self._resource_map.values():
logger.info("Loading '%s' resources into the database...",
resource.type)
if resource.type == 'project':
_, time_elapsed = resource.create_resources(
self._keystone_client)
elif resource.type == 'security-group':
_, time_elapsed = resource.create_resources(
self._rules_per_security_group)
elif resource.type == 'virtual-machine-interface':
_, time_elapsed = resource.create_resources(
self._resource_map['virtual-network'].amount_per_project)
else:
_, time_elapsed = resource.create_resources()
logger.info("%d resources were created to load %d '%s' in "
"%2.2f seconds.", resource.total_amount,
resource.amount_per_project, resource.type,
time_elapsed)
self._zk_client.disconnect()
class LoadDataBase(object):
_UUID_KEYSPACE_NAME = 'config_db_uuid'
_OBJ_UUID_CF_NAME = 'obj_uuid_table'
_OBJ_FQ_NAME_CF_NAME = 'obj_fq_name_table'
_OBJ_SHARED_CF_NAME = 'obj_shared_table'
_UUID_KEYSPACE = {
_UUID_KEYSPACE_NAME: {
_OBJ_UUID_CF_NAME: {
'cf_args': {
'autopack_names': False,
'autopack_values': False,
},
},
_OBJ_FQ_NAME_CF_NAME: {
'cf_args': {
'autopack_values': False,
},
},
_OBJ_SHARED_CF_NAME: {}
}
}
# Resources supported by that script
# The order of that list is import, that defines the resources
# order creation
_SUPPORTED_RESOURCES = [
'project',
'security-group',
'virtual-network',
'virtual-machine-interface',
]
_PERMS2 = {
'owner': None,
'owner_access': 7,
'global_access': 0,
'share': [],
}
BATCH_QUEUE_SIZE = 1000
RULES_PER_SG = 4
def __init__(self, force, resources_file, cassandra_servers,
cassandra_username, cassandra_password, cassandra_use_ssl,
cassandra_ca_certs, db_prefix, cassandra_batch_size,
zookeeper_servers, rules_per_security_group, keystone_client,
dont_populate_zookeeper):
self._force = force
self._resource_distribution = yaml.load(resources_file)
self._cassandra_batch_size = cassandra_batch_size
self._rules_per_security_group = rules_per_security_group
self._keystone_client = keystone_client
self._dont_populate_zookeeper = dont_populate_zookeeper
# Connect to cassandra database
logger.debug("Initilizing the cassandra connection on %s",
cassandra_servers)
cassandra_credentials = {}
if (cassandra_username is not None and cassandra_password is not None):
cassandra_credentials = {
'username': cassandra_username,
'password': cassandra_password,
}
def vnc_cassandra_client_logger(msg, level=logging.INFO):
logger.log(msg=msg, level=level)
self._object_db = VncObjectDBClient(cassandra_servers,
db_prefix,
self._UUID_KEYSPACE,
None,
vnc_cassandra_client_logger,
credential=cassandra_credentials,
ssl_enabled=cassandra_use_ssl,
ca_certs=cassandra_ca_certs)
self._uuid_cf = self._object_db.get_cf('obj_uuid_table')
self._fqname_cf = self._object_db.get_cf('obj_fq_name_table')
# Initilize zookeeper client
if self._dont_populate_zookeeper:
self._zk_client = DummyZookeeperClient()
else:
self._zk_client = ZookeeperClient(zookeeper_servers)
def sanitize_resources(self):
logger.debug("Santizing resources distribution")
self._resource_map = OrderedDict()
for resource_type in self._SUPPORTED_RESOURCES:
object_path = 'contrail_db_loader.resources.%s.%s' %\
(resource_type.replace('-', '_'),
camel_case(resource_type))
kwargs = {
'db_manager':
self._object_db,
'batch_size':
self._cassandra_batch_size,
'zk_client':
self._zk_client,
'project_amount':
self._resource_distribution.get('project', 0),
'amount_per_project':
self._resource_distribution.get(resource_type, 0),
}
self._resource_map[resource_type] = import_object(
object_path, **kwargs)
resources_not_supported = (set(self._resource_distribution.keys()) -
set(self._SUPPORTED_RESOURCES))
if resources_not_supported:
logger.warning('Loading resources %s are not supported' %
', '.join(resources_not_supported))
def summarize_resources_to_create(self):
msg = """Will populate %(project)d projects with:
- security groups: %(sg)d
- access control lists: %(acl)d
- virtual networks: %(vn)d
- routing instances: %(ri)d
- route targets: %(rt)d
- virtual machine interface: %(vmi)d
- virtual machine: %(vm)d
- intance ip: %(iip)d
That will load %(sum)d resources into database."""
dict = {
'project':
self._resource_map['project'].total_amount,
'sg':
self._resource_map['security-group'].amount_per_project + 1,
'acl':
(self._resource_map['security-group'].amount_per_project + 1) * 2,
'vn':
self._resource_map['virtual-network'].amount_per_project,
'ri':
self._resource_map['virtual-network'].amount_per_project,
'rt':
self._resource_map['virtual-network'].amount_per_project,
'vmi':
self._resource_map['virtual-machine-interface'].amount_per_project,
'vm':
self._resource_map['virtual-machine-interface'].amount_per_project,
'iip':
self._resource_map['virtual-machine-interface'].amount_per_project,
}
dict['sum'] = 0
for resource in self._resource_map.values():
dict['sum'] += resource.total_amount
logger.warning(msg, dict)
if (not self._force and
not prompt('Do you want to load that amount of resources?')):
exit(0)
def create_resources(self):
self._zk_client.connect()
for resource in self._resource_map.values():
logger.info("Loading '%s' resources into the database...",
resource.type)
if resource.type == 'project':
_, time_elapsed = resource.create_resources(
self._keystone_client)
elif resource.type == 'security-group':
_, time_elapsed = resource.create_resources(
self._rules_per_security_group)
elif resource.type == 'virtual-machine-interface':
_, time_elapsed = resource.create_resources(
self._resource_map['virtual-network'].amount_per_project,
self._resource_map['security-group'].amount_per_project)
else:
_, time_elapsed = resource.create_resources()
logger.info(
"%d resources were created to load %d '%s' in "
"%2.2f seconds.", resource.total_amount,
resource.amount_per_project, resource.type, time_elapsed)
self._zk_client.disconnect()
class FilterModule(object):
def __init__(self):
self._db_logger = DbLogger()
self._logger = self._db_logger.logger
self.db_client = None
# end __init__
def filters(self):
return {
'read_from_db': self.read_from_db
}
# end filters
def _initialize_db_client(self, job_ctx):
db_init_params = job_ctx.get('db_init_params')
cred = None
if (db_init_params.get('cassandra_user') is not None and
db_init_params.get('cassandra_password') is not None):
cred = {'username': db_init_params.get('cassandra_user'),
'password': db_init_params.get('cassandra_password')}
if self.db_client is None:
self.db_client = VncObjectDBClient(
db_init_params.get('cassandra_server_list'),
job_ctx.get('cluster_id'), None, None,
logger=self._db_logger.log, credential=cred,
ssl_enabled=db_init_params.get('cassandra_use_ssl'),
ca_certs=db_init_params.get('cassandra_ca_certs'))
# end _initialize_db_client
def read_from_db(self, job_ctx, obj_type, obj_id, obj_fields=None,
ret_readonly=False):
if self.db_client is None:
try:
self._initialize_db_client(job_ctx)
except Exception as e:
msg = "Error while initializing the cassandra DB " \
"client. %s " % repr(e)
self._logger.error(msg)
return self._build_result(False, msg)
try:
ok, cassandra_result = self.db_client.object_read(
obj_type, [obj_id], obj_fields, ret_readonly=ret_readonly)
except Exception as e:
self._logger.error("Exception while trying to read %s %s from db"
" %s " % (obj_type, obj_id, repr(e)))
return self._build_result(False, str(e))
return self._build_result(ok, cassandra_result[0])
# end db_read
@staticmethod
def _build_result(is_ok, result):
if is_ok is not None and is_ok:
status = 'success'
else:
status = 'false'
return {
'status': status,
'result': result
}
