def start(self): # Connect to rabbitmq for config update notifications rabbitmq_qname = self._service_id self._vnc_amqp = VncAmqpHandle(self._sandesh, self._logger, self._db_cls, self._reaction_map, self._service_id, self._rabbitmq_cfg) self._vnc_amqp.establish() cassandra_credential = { 'username': self._cassandra_cfg['user'], 'password': self._cassandra_cfg['password'] } if not all(cassandra_credential.values()): cassandra_credential = None try: self._vnc_db = VncObjectDBClient(self._cassandra_cfg['servers'], self._cassandra_cfg['cluster_id'], logger=self._logger.log, credential=cassandra_credential) except Exception as e: template = 'Exception {0} connecting to Config DB. Arguments:\n{1!r}' msg = template.format(type(e).__name__, e.args) self._logger.error('%s: %s' % (msg, traceback.format_exc())) exit() self._db_cls.init(self, self._logger, self._vnc_db) self._sync_config_db()
def initialize_db_connection(logger, args): credential = None if args.cassandra_user and args.cassandra_password: credential = { 'username': args.cassandra_user, 'password': args.cassandra_password } timeout = int(args.job_manager_db_conn_retry_timeout) max_retries = int(args.job_manager_db_conn_max_retries) retry_count = 1 while True: try: return VncObjectDBClient(args.cassandra_server_list, db_prefix=args.cluster_id, logger=logger.log, credential=credential, ssl_enabled=args.cassandra_use_ssl, ca_certs=args.cassandra_ca_certs) except Exception as e: if retry_count >= max_retries: raise e logger.error("Error while initializing db connection, " "retrying: %s" % str(e)) gevent.sleep(timeout) finally: retry_count = retry_count + 1
def __init__(self, force, resources_file, cassandra_servers, cassandra_username, cassandra_password, cassandra_use_ssl, cassandra_ca_certs, db_prefix, cassandra_batch_size, zookeeper_servers, rules_per_security_group, keystone_client, dont_populate_zookeeper): self._force = force self._resource_distribution = yaml.load(resources_file) self._cassandra_batch_size = cassandra_batch_size self._rules_per_security_group = rules_per_security_group self._keystone_client = keystone_client self._dont_populate_zookeeper = dont_populate_zookeeper # Connect to cassandra database logger.debug("Initilizing the cassandra connection on %s", cassandra_servers) cassandra_credentials = {} if (cassandra_username is not None and cassandra_password is not None): cassandra_credentials = { 'username': cassandra_username, 'password': cassandra_password, } def vnc_cassandra_client_logger(msg, level=logging.INFO): logger.log(msg=msg, level=level) self._object_db = VncObjectDBClient(cassandra_servers, db_prefix, self._UUID_KEYSPACE, None, vnc_cassandra_client_logger, credential=cassandra_credentials, ssl_enabled=cassandra_use_ssl, ca_certs=cassandra_ca_certs) self._uuid_cf = self._object_db.get_cf('obj_uuid_table') self._fqname_cf = self._object_db.get_cf('obj_fq_name_table') # Initilize zookeeper client if self._dont_populate_zookeeper: self._zk_client = DummyZookeeperClient() else: self._zk_client = ZookeeperClient(zookeeper_servers)
def _initialize_db_client(self, job_ctx): db_init_params = job_ctx.get('db_init_params') cred = None if (db_init_params.get('cassandra_user') is not None and db_init_params.get('cassandra_password') is not None): cred = {'username': db_init_params.get('cassandra_user'), 'password': db_init_params.get('cassandra_password')} if self.db_client is None: self.db_client = VncObjectDBClient( db_init_params.get('cassandra_server_list'), job_ctx.get('cluster_id'), None, None, logger=self._db_logger.log, credential=cred, ssl_enabled=db_init_params.get('cassandra_use_ssl'), ca_certs=db_init_params.get('cassandra_ca_certs'))
def __init__(self, force, resources_file, cassandra_servers, cassandra_username, cassandra_password, cassandra_use_ssl, cassandra_ca_certs, db_prefix, cassandra_batch_size, zookeeper_servers, rules_per_security_group, keystone_client, dont_populate_zookeeper): self._force = force self._resource_distribution = yaml.load(resources_file) self._cassandra_batch_size = cassandra_batch_size self._rules_per_security_group = rules_per_security_group self._keystone_client = keystone_client self._dont_populate_zookeeper = dont_populate_zookeeper # Connect to cassandra database logger.debug("Initilizing the cassandra connection on %s", cassandra_servers) cassandra_credentials = {} if (cassandra_username is not None and cassandra_password is not None): cassandra_credentials = { 'username': cassandra_username, 'password': cassandra_password, } def vnc_cassandra_client_logger(msg, level=logging.INFO): logger.log(msg=msg, level=level) self._object_db = VncObjectDBClient( cassandra_servers, db_prefix, self._UUID_KEYSPACE, None, vnc_cassandra_client_logger, credential=cassandra_credentials, ssl_enabled=cassandra_use_ssl, ca_certs=cassandra_ca_certs) self._uuid_cf = self._object_db.get_cf('obj_uuid_table') self._fqname_cf = self._object_db.get_cf('obj_fq_name_table') # Initilize zookeeper client if self._dont_populate_zookeeper: self._zk_client = DummyZookeeperClient() else: self._zk_client = ZookeeperClient(zookeeper_servers)
def start(self): # Connect to rabbitmq for config update notifications rabbitmq_qname = self._service_id self._vnc_amqp = VncAmqpHandle(self._sandesh, self._logger, self._db_cls, self._reaction_map, self._service_id, self._rabbitmq_cfg) self._vnc_amqp.establish() cassandra_credential = { 'username': self._cassandra_cfg['user'], 'password': self._cassandra_cfg['password'] } if not all(cassandra_credential.values()): cassandra_credential = None self._vnc_db = VncObjectDBClient(self._cassandra_cfg['servers'], self._cassandra_cfg['cluster_id'], logger=self._logger.log, credential=cassandra_credential) self._db_cls.init(self, self._logger, self._vnc_db) self._sync_config_db()
class LoadDataBase(object): _UUID_KEYSPACE_NAME = 'config_db_uuid' _OBJ_UUID_CF_NAME = 'obj_uuid_table' _OBJ_FQ_NAME_CF_NAME = 'obj_fq_name_table' _OBJ_SHARED_CF_NAME = 'obj_shared_table' _UUID_KEYSPACE = { _UUID_KEYSPACE_NAME: { _OBJ_UUID_CF_NAME: { 'cf_args': { 'autopack_names': False, 'autopack_values': False, }, }, _OBJ_FQ_NAME_CF_NAME: { 'cf_args': { 'autopack_values': False, }, }, _OBJ_SHARED_CF_NAME: {} } } # Resources supported by that script # The order of that list is import, that defines the resources # order creation _SUPPORTED_RESOURCES = [ 'project', 'security-group', 'virtual-network', 'virtual-machine-interface', ] _PERMS2 = { 'owner': None, 'owner_access': 7, 'global_access': 0, 'share': [], } BATCH_QUEUE_SIZE = 1000 RULES_PER_SG = 4 def __init__(self, force, resources_file, cassandra_servers, cassandra_username, cassandra_password, db_prefix, cassandra_batch_size, zookeeper_servers, rules_per_security_group, keystone_client, dont_populate_zookeeper): self._force = force self._resource_distribution = yaml.load(resources_file) self._cassandra_batch_size = cassandra_batch_size self._rules_per_security_group = rules_per_security_group self._keystone_client = keystone_client self._dont_populate_zookeeper = dont_populate_zookeeper # Connect to cassandra database logger.debug("Initilizing the cassandra connection on %s", cassandra_servers) cassandra_credentials = {} if (cassandra_username is not None and cassandra_password is not None): cassandra_credentials = { 'username': cassandra_username, 'password': cassandra_password, } def vnc_cassandra_client_logger(msg, level=logging.INFO): logger.log(msg=msg, level=level) self._object_db = VncObjectDBClient( cassandra_servers, db_prefix, self._UUID_KEYSPACE, None, vnc_cassandra_client_logger, credential=cassandra_credentials) self._uuid_cf = self._object_db.get_cf('obj_uuid_table') self._fqname_cf = self._object_db.get_cf('obj_fq_name_table') # Initilize zookeeper client if self._dont_populate_zookeeper: self._zk_client = DummyZookeeperClient() else: self._zk_client = ZookeeperClient(zookeeper_servers) def sanitize_resources(self): logger.debug("Santizing resources distribution") self._resource_map = OrderedDict() for resource_type in self._SUPPORTED_RESOURCES: object_path = 'contrail_db_loader.resources.%s.%s' %\ (resource_type.replace('-', '_'), camel_case(resource_type)) kwargs = { 'db_manager': self._object_db, 'batch_size': self._cassandra_batch_size, 'zk_client': self._zk_client, 'project_amount': self._resource_distribution.get('project', 0), 'amount_per_project': self._resource_distribution.get( resource_type, 0), } self._resource_map[resource_type] = import_object(object_path, **kwargs) resources_not_supported = (set(self._resource_distribution.keys()) - set(self._SUPPORTED_RESOURCES)) if resources_not_supported: logger.warning('Loading resources %s are not supported' % ', '.join(resources_not_supported)) def summarize_resources_to_create(self): msg = """Will populate %(project)d projects with: - security groups: %(sg)d - access control lists: %(acl)d - virtual networks: %(vn)d - routing instances: %(ri)d - route targets: %(rt)d - virtual machine interface: %(vmi)d - virtual machine: %(vm)d - intance ip: %(iip)d That will load %(sum)d resources into database.""" dict = { 'project': self._resource_map['project'].total_amount, 'sg': self._resource_map['security-group'].amount_per_project + 1, 'acl': (self._resource_map['security-group'].amount_per_project + 1) * 2, 'vn': self._resource_map['virtual-network'].amount_per_project, 'ri': self._resource_map['virtual-network'].amount_per_project, 'rt': self._resource_map['virtual-network'].amount_per_project, 'vmi': self._resource_map['virtual-machine-interface']. amount_per_project, 'vm': self._resource_map['virtual-machine-interface']. amount_per_project, 'iip': self._resource_map['virtual-machine-interface']. amount_per_project, } dict['sum'] = 0 for resource in self._resource_map.values(): dict['sum'] += resource.total_amount logger.warning(msg, dict) if (not self._force and not prompt('Do you want to load that amount of resources?')): exit(0) def create_resources(self): self._zk_client.connect() for resource in self._resource_map.values(): logger.info("Loading '%s' resources into the database...", resource.type) if resource.type == 'project': _, time_elapsed = resource.create_resources( self._keystone_client) elif resource.type == 'security-group': _, time_elapsed = resource.create_resources( self._rules_per_security_group) elif resource.type == 'virtual-machine-interface': _, time_elapsed = resource.create_resources( self._resource_map['virtual-network'].amount_per_project) else: _, time_elapsed = resource.create_resources() logger.info("%d resources were created to load %d '%s' in " "%2.2f seconds.", resource.total_amount, resource.amount_per_project, resource.type, time_elapsed) self._zk_client.disconnect()
class LoadDataBase(object): _UUID_KEYSPACE_NAME = 'config_db_uuid' _OBJ_UUID_CF_NAME = 'obj_uuid_table' _OBJ_FQ_NAME_CF_NAME = 'obj_fq_name_table' _OBJ_SHARED_CF_NAME = 'obj_shared_table' _UUID_KEYSPACE = { _UUID_KEYSPACE_NAME: { _OBJ_UUID_CF_NAME: { 'cf_args': { 'autopack_names': False, 'autopack_values': False, }, }, _OBJ_FQ_NAME_CF_NAME: { 'cf_args': { 'autopack_values': False, }, }, _OBJ_SHARED_CF_NAME: {} } } # Resources supported by that script # The order of that list is import, that defines the resources # order creation _SUPPORTED_RESOURCES = [ 'project', 'security-group', 'virtual-network', 'virtual-machine-interface', ] _PERMS2 = { 'owner': None, 'owner_access': 7, 'global_access': 0, 'share': [], } BATCH_QUEUE_SIZE = 1000 RULES_PER_SG = 4 def __init__(self, force, resources_file, cassandra_servers, cassandra_username, cassandra_password, cassandra_use_ssl, cassandra_ca_certs, db_prefix, cassandra_batch_size, zookeeper_servers, rules_per_security_group, keystone_client, dont_populate_zookeeper): self._force = force self._resource_distribution = yaml.load(resources_file) self._cassandra_batch_size = cassandra_batch_size self._rules_per_security_group = rules_per_security_group self._keystone_client = keystone_client self._dont_populate_zookeeper = dont_populate_zookeeper # Connect to cassandra database logger.debug("Initilizing the cassandra connection on %s", cassandra_servers) cassandra_credentials = {} if (cassandra_username is not None and cassandra_password is not None): cassandra_credentials = { 'username': cassandra_username, 'password': cassandra_password, } def vnc_cassandra_client_logger(msg, level=logging.INFO): logger.log(msg=msg, level=level) self._object_db = VncObjectDBClient(cassandra_servers, db_prefix, self._UUID_KEYSPACE, None, vnc_cassandra_client_logger, credential=cassandra_credentials, ssl_enabled=cassandra_use_ssl, ca_certs=cassandra_ca_certs) self._uuid_cf = self._object_db.get_cf('obj_uuid_table') self._fqname_cf = self._object_db.get_cf('obj_fq_name_table') # Initilize zookeeper client if self._dont_populate_zookeeper: self._zk_client = DummyZookeeperClient() else: self._zk_client = ZookeeperClient(zookeeper_servers) def sanitize_resources(self): logger.debug("Santizing resources distribution") self._resource_map = OrderedDict() for resource_type in self._SUPPORTED_RESOURCES: object_path = 'contrail_db_loader.resources.%s.%s' %\ (resource_type.replace('-', '_'), camel_case(resource_type)) kwargs = { 'db_manager': self._object_db, 'batch_size': self._cassandra_batch_size, 'zk_client': self._zk_client, 'project_amount': self._resource_distribution.get('project', 0), 'amount_per_project': self._resource_distribution.get(resource_type, 0), } self._resource_map[resource_type] = import_object( object_path, **kwargs) resources_not_supported = (set(self._resource_distribution.keys()) - set(self._SUPPORTED_RESOURCES)) if resources_not_supported: logger.warning('Loading resources %s are not supported' % ', '.join(resources_not_supported)) def summarize_resources_to_create(self): msg = """Will populate %(project)d projects with: - security groups: %(sg)d - access control lists: %(acl)d - virtual networks: %(vn)d - routing instances: %(ri)d - route targets: %(rt)d - virtual machine interface: %(vmi)d - virtual machine: %(vm)d - intance ip: %(iip)d That will load %(sum)d resources into database.""" dict = { 'project': self._resource_map['project'].total_amount, 'sg': self._resource_map['security-group'].amount_per_project + 1, 'acl': (self._resource_map['security-group'].amount_per_project + 1) * 2, 'vn': self._resource_map['virtual-network'].amount_per_project, 'ri': self._resource_map['virtual-network'].amount_per_project, 'rt': self._resource_map['virtual-network'].amount_per_project, 'vmi': self._resource_map['virtual-machine-interface'].amount_per_project, 'vm': self._resource_map['virtual-machine-interface'].amount_per_project, 'iip': self._resource_map['virtual-machine-interface'].amount_per_project, } dict['sum'] = 0 for resource in self._resource_map.values(): dict['sum'] += resource.total_amount logger.warning(msg, dict) if (not self._force and not prompt('Do you want to load that amount of resources?')): exit(0) def create_resources(self): self._zk_client.connect() for resource in self._resource_map.values(): logger.info("Loading '%s' resources into the database...", resource.type) if resource.type == 'project': _, time_elapsed = resource.create_resources( self._keystone_client) elif resource.type == 'security-group': _, time_elapsed = resource.create_resources( self._rules_per_security_group) elif resource.type == 'virtual-machine-interface': _, time_elapsed = resource.create_resources( self._resource_map['virtual-network'].amount_per_project, self._resource_map['security-group'].amount_per_project) else: _, time_elapsed = resource.create_resources() logger.info( "%d resources were created to load %d '%s' in " "%2.2f seconds.", resource.total_amount, resource.amount_per_project, resource.type, time_elapsed) self._zk_client.disconnect()
class FilterModule(object): def __init__(self): self._db_logger = DbLogger() self._logger = self._db_logger.logger self.db_client = None # end __init__ def filters(self): return { 'read_from_db': self.read_from_db } # end filters def _initialize_db_client(self, job_ctx): db_init_params = job_ctx.get('db_init_params') cred = None if (db_init_params.get('cassandra_user') is not None and db_init_params.get('cassandra_password') is not None): cred = {'username': db_init_params.get('cassandra_user'), 'password': db_init_params.get('cassandra_password')} if self.db_client is None: self.db_client = VncObjectDBClient( db_init_params.get('cassandra_server_list'), job_ctx.get('cluster_id'), None, None, logger=self._db_logger.log, credential=cred, ssl_enabled=db_init_params.get('cassandra_use_ssl'), ca_certs=db_init_params.get('cassandra_ca_certs')) # end _initialize_db_client def read_from_db(self, job_ctx, obj_type, obj_id, obj_fields=None, ret_readonly=False): if self.db_client is None: try: self._initialize_db_client(job_ctx) except Exception as e: msg = "Error while initializing the cassandra DB " \ "client. %s " % repr(e) self._logger.error(msg) return self._build_result(False, msg) try: ok, cassandra_result = self.db_client.object_read( obj_type, [obj_id], obj_fields, ret_readonly=ret_readonly) except Exception as e: self._logger.error("Exception while trying to read %s %s from db" " %s " % (obj_type, obj_id, repr(e))) return self._build_result(False, str(e)) return self._build_result(ok, cassandra_result[0]) # end db_read @staticmethod def _build_result(is_ok, result): if is_ok is not None and is_ok: status = 'success' else: status = 'false' return { 'status': status, 'result': result }