def main(args_str=None): global _zookeeper_client if not args_str: args_str = ' '.join(sys.argv[1:]) args = parse_args(args_str) if args.cluster_id: client_pfx = args.cluster_id + '-' zk_path_pfx = args.cluster_id + '/' else: client_pfx = '' zk_path_pfx = '' # randomize collector list args.random_collectors = args.collectors if args.collectors: args.random_collectors = random.sample(args.collectors, len(args.collectors)) # Initialize logger without introspect thread dm_logger = DeviceManagerLogger(args, http_server_port=-1) # Initialize AMQP handler then close it to be sure remain queue of a # precedent run is cleaned vnc_amqp = DMAmqpHandle(dm_logger, DeviceManager.REACTION_MAP, args) vnc_amqp.establish() vnc_amqp.close() dm_logger.debug("Removed remained AMQP queue") _zookeeper_client = ZookeeperClient(client_pfx + "device-manager", args.zk_server_ip) dm_logger.notice("Waiting to be elected as master...") _zookeeper_client.master_election(zk_path_pfx + "/device-manager", os.getpid(), run_device_manager, dm_logger, args)
def main(args_str=None): global _zookeeper_client if not args_str: args_str = ' '.join(sys.argv[1:]) args = parse_args(args_str) if args.cluster_id: client_pfx = args.cluster_id + '-' zk_path_pfx = args.cluster_id + '/' else: client_pfx = '' zk_path_pfx = '' # randomize collector list args.random_collectors = args.collectors if args.collectors: args.random_collectors = random.sample(args.collectors, len(args.collectors)) # Initialize logger without introspect thread dm_logger = DeviceManagerLogger(args, http_server_port=-1) # Initialize AMQP handler then close it to be sure remain queue of a # precedent run is cleaned vnc_amqp = DMAmqpHandle(dm_logger, DeviceManager.REACTION_MAP, args) vnc_amqp.establish() vnc_amqp.close() dm_logger.debug("Removed remained AMQP queue") _zookeeper_client = ZookeeperClient(client_pfx+"device-manager", args.zk_server_ip) dm_logger.notice("Waiting to be elected as master...") _zookeeper_client.master_election(zk_path_pfx+"/device-manager", os.getpid(), run_device_manager, dm_logger, args)
def main(args_str=None): global _zookeeper_client if not args_str: args_str = ' '.join(sys.argv[1:]) args = parse_args(args_str) if args.cluster_id: client_pfx = args.cluster_id + '-' zk_path_pfx = args.cluster_id + '/' else: client_pfx = '' zk_path_pfx = '' # Initialize discovery client discovery_client = None if args.disc_server_ip and args.disc_server_port: dss_kwargs = {} if args.disc_server_ssl: if args.disc_server_cert: dss_kwargs.update({'cert': args.disc_server_cert}) if args.disc_server_key: dss_kwargs.update({'key': args.disc_server_key}) if args.disc_server_cacert: dss_kwargs.update({'cacert': args.disc_server_cacert}) discovery_client = client.DiscoveryClient( args.disc_server_ip, args.disc_server_port, ModuleNames[Module.DEVICE_MANAGER], **dss_kwargs) # Initialize logger module = Module.DEVICE_MANAGER module_pkg = "device_manager" dm_logger = ConfigServiceLogger(discovery_client, module, module_pkg, args, http_server_port=-1) # Initialize AMQP handler then close it to be sure remain queue of a # precedent run is cleaned vnc_amqp = DMAmqpHandle(dm_logger, DeviceManager.REACTION_MAP, args) vnc_amqp.establish() vnc_amqp.close() dm_logger.debug("Removed remained AMQP queue") _zookeeper_client = ZookeeperClient(client_pfx + "device-manager", args.zk_server_ip) dm_logger.notice("Waiting to be elected as master...") _zookeeper_client.master_election(zk_path_pfx + "/device-manager", os.getpid(), run_device_manager, dm_logger, args)
def main(args_str=None): global _amqp_client global _zookeeper_client if not args_str: args_str = ' '.join(sys.argv[1:]) args = parse_args(args_str) if args.cluster_id: client_pfx = args.cluster_id + '-' zk_path_pfx = args.cluster_id + '/' else: client_pfx = '' zk_path_pfx = '' # randomize collector list args.random_collectors = args.collectors if args.collectors: args.random_collectors = random.sample(args.collectors, len(args.collectors)) # Initialize logger without introspect thread dm_logger = DeviceManagerLogger(args, http_server_port=-1) # Initialize AMQP handler then close it to be sure remain queue of a # precedent run is cleaned vnc_amqp = DMAmqpHandle(dm_logger, {}, args) vnc_amqp.establish() vnc_amqp.close() dm_logger.debug("Removed remaining AMQP queue from previous run") if 'host_ip' not in args: args.host_ip = socket.gethostbyname(socket.getfqdn()) _amqp_client = initialize_amqp_client(dm_logger, args) _zookeeper_client = ZookeeperClient(client_pfx + "device-manager", args.zk_server_ip, args.host_ip) try: # Initialize the device job manager DeviceJobManager(_amqp_client, _zookeeper_client, args, dm_logger) # Allow kombu client to connect consumers gevent.sleep(0.5) except Exception as e: dm_logger.error("Error while initializing the device job " "manager %s" % str(e)) raise e try: # Initialize the device ztp manager DeviceZtpManager(_amqp_client, args, dm_logger) # Allow kombu client to connect consumers gevent.sleep(0.5) except Exception as e: dm_logger.error("Error while initializing the device ztp " "manager %s" % str(e)) raise e gevent.signal(signal.SIGHUP, sighup_handler) gevent.signal(signal.SIGTERM, sigterm_handler) gevent.signal(signal.SIGINT, sigterm_handler) dm_logger.notice("Waiting to be elected as master...") _zookeeper_client.master_election(zk_path_pfx + "/device-manager", os.getpid(), run_device_manager, dm_logger, args)
def main(args_str=None): global _amqp_client global _zookeeper_client global _object_db if not args_str: args_str = ' '.join(sys.argv[1:]) args = parse_args(args_str) if args.cluster_id: client_pfx = args.cluster_id + '-' zk_path_pfx = args.cluster_id + '/' else: client_pfx = '' zk_path_pfx = '' # randomize collector list args.random_collectors = args.collectors if args.collectors: args.random_collectors = random.sample(args.collectors, len(args.collectors)) # Initialize logger without introspect thread dm_logger = DeviceManagerLogger(args, http_server_port=-1) # Initialize AMQP handler then close it to be sure remain queue of a # precedent run is cleaned vnc_amqp = DMAmqpHandle(dm_logger, {}, args) vnc_amqp.establish() vnc_amqp.close() dm_logger.debug("Removed remaining AMQP queue from previous run") if 'host_ip' in args: host_ip = args.host_ip else: host_ip = socket.gethostbyname(socket.getfqdn()) _amqp_client = initialize_amqp_client(dm_logger, args) _zookeeper_client = ZookeeperClient(client_pfx+"device-manager", args.zk_server_ip, host_ip) _object_db = DMCassandraDB.get_instance(_zookeeper_client, args, dm_logger) try: # Initialize the device job manager DeviceJobManager(_object_db, _amqp_client, _zookeeper_client, args, dm_logger) except Exception as e: dm_logger.error("Error while initializing the device job " "manager %s" % repr(e)) try: # Initialize the device ztp manager DeviceZtpManager(_amqp_client, args, dm_logger) except Exception as e: dm_logger.error("Error while initializing the device ztp " "manager %s" % repr(e)) gevent.signal(signal.SIGHUP, sighup_handler) gevent.signal(signal.SIGTERM, sigterm_handler) gevent.signal(signal.SIGINT, sigterm_handler) dm_logger.notice("Waiting to be elected as master...") _zookeeper_client.master_election(zk_path_pfx+"/device-manager", os.getpid(), run_device_manager, dm_logger, args)
class DeviceManager(object): REACTION_MAP = { 'physical_router': { 'self': ['bgp_router', 'physical_interface', 'logical_interface'], 'bgp_router': [], 'physical_interface': [], 'logical_interface': [], 'virtual_network': [], 'global_system_config': [], }, 'global_system_config': { 'self': ['physical_router'], 'physical_router': [], }, 'bgp_router': { 'self': ['bgp_router', 'physical_router'], 'bgp_router': ['physical_router'], 'physical_router': [], }, 'physical_interface': { 'self': ['physical_router', 'physical_interface', 'logical_interface'], 'physical_router': ['logical_interface'], 'logical_interface': ['physical_router'], 'physical_interface': ['physical_router'], 'virtual_machine_interface': ['physical_interface'], }, 'logical_interface': { 'self': [ 'physical_router', 'physical_interface', 'virtual_machine_interface' ], 'physical_interface': ['virtual_machine_interface'], 'virtual_machine_interface': ['physical_router', 'physical_interface'], 'physical_router': ['virtual_machine_interface'] }, 'virtual_machine_interface': { 'self': [ 'logical_interface', 'physical_interface', 'virtual_network', 'floating_ip', 'instance_ip', 'port_tuple' ], 'logical_interface': ['virtual_network'], 'virtual_network': ['logical_interface'], 'floating_ip': ['virtual_network'], 'instance_ip': ['virtual_network'], 'routing_instance': ['port_tuple', 'physical_interface'], 'port_tuple': ['physical_interface'] }, 'service_instance': { 'self': ['port_tuple'], 'port_tuple': [], }, 'port_tuple': { 'self': ['virtual_machine_interface', 'service_instance'], 'service_instance': ['virtual_machine_interface'], 'virtual_machine_interface': ['service_instance'] }, 'virtual_network': { 'self': ['physical_router', 'virtual_machine_interface'], 'routing_instance': ['physical_router', 'virtual_machine_interface'], 'physical_router': [], 'virtual_machine_interface': ['physical_router'], }, 'routing_instance': { 'self': [ 'routing_instance', 'virtual_network', 'virtual_machine_interface' ], 'routing_instance': ['virtual_network', 'virtual_machine_interface'], 'virtual_network': [] }, 'floating_ip': { 'self': ['virtual_machine_interface'], 'virtual_machine_interface': [], }, 'instance_ip': { 'self': ['virtual_machine_interface'], 'virtual_machine_interface': [], }, } def __init__(self, dm_logger=None, args=None): self._args = args PushConfigState.set_repush_interval(int(self._args.repush_interval)) PushConfigState.set_repush_max_interval( int(self._args.repush_max_interval)) PushConfigState.set_push_delay_per_kb( float(self._args.push_delay_per_kb)) PushConfigState.set_push_delay_max(int(self._args.push_delay_max)) PushConfigState.set_push_delay_enable( bool(self._args.push_delay_enable)) if dm_logger is not None: self.logger = dm_logger else: # Initialize discovery client discovery_client = None if self._args.disc_server_ip and self._args.disc_server_port: dss_kwargs = {} if self._args.disc_server_ssl: if self._args.disc_server_cert: dss_kwargs.update( {'cert': self._args.disc_server_cert}) if self._args.disc_server_key: dss_kwargs.update({'key': self._args.disc_server_key}) if self._args.disc_server_cacert: dss_kwargs.update( {'cacert': self._args.disc_server_cacert}) discovery_client = client.DiscoveryClient( self._args.disc_server_ip, self._args.disc_server_port, ModuleNames[Module.DEVICE_MANAGER], **dss_kwargs) # Initialize logger module = Module.DEVICE_MANAGER module_pkg = "device_manager" self.logger = ConfigServiceLogger(discovery_client, module, module_pkg, args) # Retry till API server is up connected = False self.connection_state_update(ConnectionStatus.INIT) while not connected: try: self._vnc_lib = VncApi( args.admin_user, args.admin_password, args.admin_tenant_name, args.api_server_ip, args.api_server_port, api_server_use_ssl=args.api_server_use_ssl) connected = True self.connection_state_update(ConnectionStatus.UP) except requests.exceptions.ConnectionError as e: # Update connection info self.connection_state_update(ConnectionStatus.DOWN, str(e)) time.sleep(3) except ResourceExhaustionError: # haproxy throws 503 time.sleep(3) # Initialize amqp self._vnc_amqp = DMAmqpHandle(self.logger, self.REACTION_MAP, self._args) self._vnc_amqp.establish() # Initialize cassandra self._cassandra = DMCassandraDB.getInstance(self, _zookeeper_client) DBBaseDM.init(self, self.logger, self._cassandra) DBBaseDM._sandesh = self.logger._sandesh for obj in GlobalSystemConfigDM.list_obj(): GlobalSystemConfigDM.locate(obj['uuid'], obj) for obj in GlobalVRouterConfigDM.list_obj(): GlobalVRouterConfigDM.locate(obj['uuid'], obj) for obj in VirtualNetworkDM.list_obj(): vn = VirtualNetworkDM.locate(obj['uuid'], obj) if vn is not None and vn.routing_instances is not None: for ri_id in vn.routing_instances: ri_obj = RoutingInstanceDM.locate(ri_id) for obj in BgpRouterDM.list_obj(): BgpRouterDM.locate(obj['uuid'], obj) pr_obj_list = PhysicalRouterDM.list_obj() pr_uuid_set = set([pr_obj['uuid'] for pr_obj in pr_obj_list]) self._cassandra.handle_pr_deletes(pr_uuid_set) for obj in PortTupleDM.list_obj(): PortTupleDM.locate(obj['uuid'], obj) for obj in pr_obj_list: pr = PhysicalRouterDM.locate(obj['uuid'], obj) li_set = pr.logical_interfaces vmi_set = set() for pi_id in pr.physical_interfaces: pi = PhysicalInterfaceDM.locate(pi_id) if pi: li_set |= pi.logical_interfaces vmi_set |= pi.virtual_machine_interfaces for li_id in li_set: li = LogicalInterfaceDM.locate(li_id) if li and li.virtual_machine_interface: vmi_set |= set([li.virtual_machine_interface]) for vmi_id in vmi_set: vmi = VirtualMachineInterfaceDM.locate(vmi_id) si_obj_list = ServiceInstanceDM.list_obj() si_uuid_set = set([si_obj['uuid'] for si_obj in si_obj_list]) self._cassandra.handle_pnf_resource_deletes(si_uuid_set) for obj in si_obj_list: ServiceInstanceDM.locate(obj['uuid'], obj) for obj in InstanceIpDM.list_obj(): InstanceIpDM.locate(obj['uuid'], obj) for obj in FloatingIpDM.list_obj(): FloatingIpDM.locate(obj['uuid'], obj) for vn in VirtualNetworkDM.values(): vn.update_instance_ip_map() for pr in PhysicalRouterDM.values(): pr.set_config_state() self._vnc_amqp._db_resync_done.set() try: gevent.joinall(self._vnc_amqp._vnc_kombu.greenlets()) except KeyboardInterrupt: self._vnc_amqp.close() raise # end __init__ def connection_state_update(self, status, message=None): ConnectionState.update( conn_type=ConnType.APISERVER, name='ApiServer', status=status, message=message or 'ApiServer Connection State updated', server_addrs=[ '%s:%s' % (self._args.api_server_ip, self._args.api_server_port) ])