def dbe_obj_read(self, resource_type, resource_id, obj_fields):
resource_oper = None
try:
if "all" in obj_fields:
ok, obj_dict = self._db_conn.dbe_read(
resource_type,
resource_id,
)
else:
obj_fields = list(set(obj_fields))
ok, obj_dict = self._db_conn.dbe_read(resource_type,
resource_id,
obj_fields=obj_fields)
if not ok:
resource_oper = "STOP"
obj_dict = {"error": "dbe_read failure: %s" % obj_dict}
except NoIdError:
err_msg = "Resource with id %s already deleted at the \
point of dbe_read" % resource_id
err_msg += detailed_traceback()
self.config_log(err_msg, level=SandeshLevel.SYS_NOTICE)
resource_oper = OP_DELETE
obj_dict = {"uuid": resource_id}
except Exception as e:
err_msg = "Exception %s while performing dbe_read" % e
err_msg += detailed_traceback()
self.config_log(err_msg, level=SandeshLevel.SYS_NOTICE)
resource_oper = "STOP"
obj_dict = {"error": str(e)}
return resource_oper, obj_dict
def wrapper(*args, **kwargs):
if ((sys._getframe(1).f_code.co_name not in
['_Multiget', 'multiget']) and
func.__name__ in ['get', 'multiget']):
msg = ("It is not recommended to use 'get' or 'multiget' "
"pycassa methods. It's better to use 'xget' or "
"'get_range' methods due to thrift limitations")
self.options.logger(msg, level=SandeshLevel.SYS_WARN)
try:
if self.get_status() != ConnectionStatus.UP:
# will set conn_state to UP if successful
self._cassandra_init_conn_pools()
self.start_time = datetime.datetime.now()
return func(*args, **kwargs)
except (pycassa.pool.AllServersUnavailable, pycassa.pool.MaximumRetryException) as e:
if self.get_status() != ConnectionStatus.DOWN:
self.report_status_down()
msg = 'Cassandra connection down. Exception in %s' % (
str(func))
self.options.logger(msg, level=SandeshLevel.SYS_ERR)
raise DatabaseUnavailableError(
'Error, %s: %s' % (str(e), utils.detailed_traceback()))
finally:
if ((self.log_response_time) and (oper)):
self.end_time = datetime.datetime.now()
self.log_response_time(self.end_time - self.start_time, oper)
def _ifmap_dequeue_task(self):
while True:
try:
self._publish_to_ifmap_dequeue()
except Exception as e:
tb = detailed_traceback()
self.config_log(tb, level=SandeshLevel.SYS_ERR)
def _ifmap_dequeue_task(self):
while True:
try:
self._publish_to_ifmap_dequeue()
except Exception as e:
tb = detailed_traceback()
self.config_log(tb, level=SandeshLevel.SYS_ERR)
def dispatch(self):
while True:
notification = self._notification_queue.get()
resource_type = notification.get("type")
resource_oper = notification.get("oper")
resource_id = notification.get("uuid")
resource_data = {}
if not self._client_queues.get(resource_type, None):
# No clients subscribed for this resource type
continue
if resource_oper == "DELETE":
resource_data = {"uuid": resource_id}
else:
try:
ok, resource_data = self._db_conn.dbe_read(
resource_type, resource_id)
if not ok:
resource_oper = "STOP"
resource_data = {
"error": "dbe_read failure: %s" % resource_data
}
except NoIdError:
err_msg = "Resource with id %s already deleted at the \
point of dbe_read" % resource_id
err_msg += detailed_traceback()
self.config_log(err_msg, level=SandeshLevel.SYS_NOTICE)
resource_oper = "DELETE"
resource_data = {"uuid": resource_id}
except Exception as e:
err_msg = "Exception %s while performing dbe_read" % e
err_msg += detailed_traceback()
self.config_log(err_msg, level=SandeshLevel.SYS_NOTICE)
resource_oper = "STOP"
resource_data = {"error": str(e)}
event = self.pack(event=resource_oper,
data={resource_type: resource_data})
for client_queue in self._client_queues.get(resource_type, []):
client_queue.put_nowait(event)
def wrapper(*args, **kwargs):
try:
if self.get_status() != ConnectionStatus.UP:
self._Init_Cluster()
self.start_time = datetime.datetime.now()
return func(*args, **kwargs)
except (connector.InvalidRequest,
connector.cluster.NoHostAvailable,
connector.cqlengine.CQLEngineException) as e:
if self.get_status() != ConnectionStatus.DOWN:
self.report_status_down()
self.options.logger(
"Cassandra connection down. Exception in {}".format(
func),
level=SandeshLevel.SYS_ERR)
raise DatabaseUnavailableError("error, {}: {}".format(
e, utils.detailed_traceback()))
finally:
if ((self.log_response_time) and (oper)):
self.end_time = datetime.datetime.now()
self.log_response_time(self.end_time - self.start_time,
oper)
def _load_init_data(self):
"""
Load init data for job playbooks.
This function loads init data from a data file specified by the
argument '--fabric_ansible_dir' to the database. The data file
must be in JSON format and follow the format below
"my payload": {
"object_type": "tag"
"objects": [
{
"fq_name": [
"fabric=management_ip"
],
"name": "fabric=management_ip",
"tag_type_name": "fabric",
"tag_value": "management_ip"
}
]
}
"""
try:
json_data = self._load_json_data()
if json_data is None:
self._logger.error('Unable to load init data')
return
for item in json_data.get("data"):
object_type = item.get("object_type")
# Get the class name from object type
cls_name = CamelCase(object_type)
# Get the class object
cls_ob = str_to_class(cls_name, resource_client.__name__)
# saving the objects to the database
for obj in item.get("objects"):
instance_obj = cls_ob.from_dict(**obj)
# create/update the object
fq_name = instance_obj.get_fq_name()
try:
uuid = self._vnc_api.fq_name_to_id(
object_type, fq_name)
if object_type == "tag":
continue
instance_obj.set_uuid(uuid)
# Update config json inside role-config object
if object_type == 'role-config':
role_config_obj = self._vnc_api.\
role_config_read(id=uuid)
cur_config_json = json.loads(
role_config_obj.get_role_config_config())
def_config_json = json.loads(
instance_obj.get_role_config_config())
def_config_json.update(cur_config_json)
instance_obj.set_role_config_config(
json.dumps(def_config_json))
if object_type != 'telemetry-profile' and \
object_type != 'sflow-profile':
self._vnc_api._object_update(
object_type, instance_obj)
except NoIdError:
self._vnc_api._object_create(object_type, instance_obj)
for item in json_data.get("refs"):
from_type = item.get("from_type")
from_fq_name = item.get("from_fq_name")
from_uuid = self._vnc_api.fq_name_to_id(
from_type, from_fq_name)
to_type = item.get("to_type")
to_fq_name = item.get("to_fq_name")
to_uuid = self._vnc_api.fq_name_to_id(to_type, to_fq_name)
self._vnc_api.ref_update(from_type, from_uuid, to_type,
to_uuid, to_fq_name, 'ADD')
except Exception as e:
err_msg = 'error while loading init data: %s\n' % str(e)
err_msg += detailed_traceback()
self._logger.error(err_msg)
# - fetch list of all the physical routers
# - check physical and overlay role associated with each PR
# - create ref between physical_role and physical_router object,
# if PR is assigned with a specific physical role
# - create ref between overlay_roles and physical_router object,
# if PR is assigned with specific overlay roles
obj_list = self._vnc_api._objects_list('physical-router')
pr_list = obj_list.get('physical-routers')
for pr in pr_list or []:
try:
pr_obj = self._vnc_api.\
physical_router_read(id=pr.get('uuid'))
physical_role = pr_obj.get_physical_router_role()
overlay_roles = pr_obj.get_routing_bridging_roles()
if overlay_roles is not None:
overlay_roles = overlay_roles.get_rb_roles()
if physical_role:
try:
physical_role_uuid = self._vnc_api.\
fq_name_to_id('physical_role',
['default-global-system-config',
physical_role])
if physical_role_uuid:
self._vnc_api.ref_update('physical-router',
pr.get('uuid'),
'physical-role',
physical_role_uuid, None,
'ADD')
except NoIdError:
pass
if overlay_roles:
for overlay_role in overlay_roles or []:
try:
overlay_role_uuid = self._vnc_api.\
fq_name_to_id('overlay_role',
['default-global-system-config',
overlay_role.lower()])
if overlay_role_uuid:
self._vnc_api.ref_update(
'physical-router', pr.get('uuid'),
'overlay-role', overlay_role_uuid, None,
'ADD')
except NoIdError:
pass
except NoIdError:
pass
# handle deleted job_templates as part of in-place cluster update
to_be_del_jt_names = [
'show_interfaces_template', 'show_config_interfaces_template',
'show_interfaces_by_names_template'
]
for jt_name in to_be_del_jt_names:
try:
self._vnc_api.job_template_delete(
fq_name=['default-global-system-config', jt_name])
except NoIdError:
pass
def _load_init_data(self):
"""
Load init data for job playbooks.
This function loads init data from a data file specified by the
argument '--fabric_ansible_dir' to the database. The data file
must be in JSON format and follow the format below
"my payload": {
"object_type": "tag"
"objects": [
{
"fq_name": [
"fabric=management_ip"
],
"name": "fabric=management_ip",
"tag_type_name": "fabric",
"tag_value": "management_ip"
}
]
}
"""
try:
json_data = self._load_json_data()
if json_data is None:
self._logger.error('Unable to load init data')
return
for item in json_data.get("data"):
object_type = item.get("object_type")
# Get the class name from object type
cls_name = CamelCase(object_type)
# Get the class object
cls_ob = str_to_class(cls_name, resource_client.__name__)
# saving the objects to the database
for obj in item.get("objects"):
instance_obj = cls_ob.from_dict(**obj)
# create/update the object
fq_name = instance_obj.get_fq_name()
try:
uuid_id = self._vnc_api.fq_name_to_id(
object_type, fq_name)
if object_type == "tag":
continue
instance_obj.set_uuid(uuid_id)
# Update config json inside role-config object
if object_type == 'role-config':
role_config_obj = self._vnc_api.\
role_config_read(id=uuid_id)
cur_config_json = json.loads(
role_config_obj.get_role_config_config())
def_config_json = json.loads(
instance_obj.get_role_config_config())
def_config_json.update(cur_config_json)
instance_obj.set_role_config_config(
json.dumps(def_config_json))
if object_type != 'telemetry-profile' and \
object_type != 'sflow-profile' and \
object_type != 'device-functional-group':
self._vnc_api._object_update(
object_type, instance_obj)
except NoIdError:
self._vnc_api._object_create(object_type, instance_obj)
for item in json_data.get("refs"):
from_type = item.get("from_type")
from_fq_name = item.get("from_fq_name")
from_uuid = self._vnc_api.fq_name_to_id(
from_type, from_fq_name)
to_type = item.get("to_type")
to_fq_name = item.get("to_fq_name")
to_uuid = self._vnc_api.fq_name_to_id(to_type, to_fq_name)
self._vnc_api.ref_update(from_type, from_uuid, to_type,
to_uuid, to_fq_name, 'ADD')
except Exception as e:
err_msg = 'error while loading init data: %s\n' % str(e)
err_msg += detailed_traceback()
self._logger.error(err_msg)
# create VN and IPAM for IPV6 link-local addresses
ipv6_link_local_nw = '_internal_vn_ipv6_link_local'
self._create_ipv6_ll_ipam_and_vn(self._vnc_api, ipv6_link_local_nw)
# - fetch list of all the physical routers
# - check physical and overlay role associated with each PR
# - create ref between physical_role and physical_router object,
# if PR is assigned with a specific physical role
# - create ref between overlay_roles and physical_router object,
# if PR is assigned with specific overlay roles
obj_list = self._vnc_api._objects_list('physical-router')
pr_list = obj_list.get('physical-routers')
for pr in pr_list or []:
try:
pr_obj = self._vnc_api.\
physical_router_read(id=pr.get('uuid'))
physical_role = pr_obj.get_physical_router_role()
overlay_roles = pr_obj.get_routing_bridging_roles()
if overlay_roles is not None:
overlay_roles = overlay_roles.get_rb_roles()
if physical_role:
try:
physical_role_uuid = self._vnc_api.\
fq_name_to_id('physical_role',
['default-global-system-config',
physical_role])
if physical_role_uuid:
self._vnc_api.ref_update('physical-router',
pr.get('uuid'),
'physical-role',
physical_role_uuid, None,
'ADD')
except NoIdError:
pass
if overlay_roles:
for overlay_role in overlay_roles or []:
try:
overlay_role_uuid = self._vnc_api.\
fq_name_to_id('overlay_role',
['default-global-system-config',
overlay_role.lower()])
if overlay_role_uuid:
self._vnc_api.ref_update(
'physical-router', pr.get('uuid'),
'overlay-role', overlay_role_uuid, None,
'ADD')
except NoIdError:
pass
except NoIdError:
pass
# handle replacing master-LR as <fab_name>-master-LR here
# as part of in-place cluster update. Copy the master-LR
# and also its associated vns and their annotations here
master_lr_obj = None
try:
master_lr_obj = self._vnc_api.logical_router_read(
fq_name=['default-domain', 'default-project', 'master-LR'])
except NoIdError:
try:
master_lr_obj = self._vnc_api.logical_router_read(
fq_name=['default-domain', 'admin', 'master-LR'])
except NoIdError:
pass
if master_lr_obj:
vmi_refs = master_lr_obj.get_virtual_machine_interface_refs() or []
# get existing pr refs
pr_refs = master_lr_obj.get_physical_router_refs() or []
fabric_refs = master_lr_obj.get_fabric_refs() or []
perms2 = master_lr_obj.get_perms2()
fab_fq_name = None
try:
# This has to happen before creating fab-master-LR as
# otherwise it will fail creation
# of fab-master-lr with annotations having master-lr uuid
# Now delete master-LR object
# this will delete lr annotations from fabric in
# corresponding VNs if they exist
self._vnc_api.logical_router_delete(
id=master_lr_obj.get_uuid())
# try to obtain the fabric refs either by fabric ref if one
# is available or from pr_refs if available
if pr_refs and not fabric_refs:
# this is assuming that even though there can be
# multiple pr refs, a LR cannot have more than
# one fabric refs. So a random pr chosen in the pr
# refs list will have the same fabric name as the other
# prs in the list
pr_ref = pr_refs[-1]
pr_obj = self._vnc_api.physical_router_read(id=pr_ref.get(
'uuid', self._vnc_api.fq_name_to_id(pr_ref.get('to'))))
fabric_refs = pr_obj.get_fabric_refs() or []
if fabric_refs:
fabric_ref = fabric_refs[-1]
fab_fq_name = fabric_ref.get(
'to',
self._vnc_api.id_to_fq_name(fabric_ref.get('uuid')))
# if fab_fq_name is not derivable or was not present, then
# skip creating fab_name-master-LR as fabric information
# is not available
# if fab_fq_name is available, copy necessary refs from prev.
# master LR, create new fab_name-master-LR and this will update
# VN annotations accordingly.
if fab_fq_name:
def_project = self._vnc_api.project_read(
['default-domain', 'default-project'])
fab_name = fab_fq_name[-1]
lr_fq_name = [
'default-domain', 'default-project',
fab_name + '-master-LR'
]
fab_master_lr_obj = LogicalRouter(
name=lr_fq_name[-1],
fq_name=lr_fq_name,
logical_router_gateway_external=False,
logical_router_type='vxlan-routing',
parent_obj=def_project)
perms2.set_global_access(PERMS_RWX)
fab_master_lr_obj.set_perms2(perms2)
fab_master_lr_obj.set_virtual_machine_interface_list(
vmi_refs)
fab_master_lr_obj.set_physical_router_list(pr_refs)
fab_master_lr_obj.set_fabric_list(fabric_refs)
self._vnc_api.logical_router_create(fab_master_lr_obj)
except NoIdError:
pass
except Exception as exc:
err_msg = "An exception occurred while attempting to " \
"create fabric master-LR: %s " % exc.message
self._logger.warning(err_msg)
# handle deleted job_templates as part of in-place cluster update
to_be_del_jt_names = [
'show_interfaces_template', 'show_config_interfaces_template',
'show_interfaces_by_names_template'
]
for jt_name in to_be_del_jt_names:
try:
self._vnc_api.job_template_delete(
fq_name=['default-global-system-config', jt_name])
except NoIdError:
pass
def _load_init_data(self):
"""
Load init data for job playbooks.
This function loads init data from a data file specified by the
argument '--fabric_ansible_dir' to the database. The data file
must be in JSON format and follow the format below
"my payload": {
"object_type": "tag"
"objects": [
{
"fq_name": [
"fabric=management_ip"
],
"name": "fabric=management_ip",
"tag_type_name": "fabric",
"tag_value": "management_ip"
}
]
}
"""
try:
json_data = self._load_json_data()
if json_data is None:
self._logger.error('Unable to load init data')
return
for item in json_data.get("data"):
object_type = item.get("object_type")
# Get the class name from object type
cls_name = CamelCase(object_type)
# Get the class object
cls_ob = str_to_class(cls_name, resource_client.__name__)
# saving the objects to the database
for obj in item.get("objects"):
instance_obj = cls_ob.from_dict(**obj)
# create/update the object
fq_name = instance_obj.get_fq_name()
try:
uuid = self._vnc_api.fq_name_to_id(
object_type, fq_name)
if object_type == "tag":
continue
instance_obj.set_uuid(uuid)
self._vnc_api._object_update(object_type, instance_obj)
except NoIdError:
self._vnc_api._object_create(object_type, instance_obj)
for item in json_data.get("refs"):
from_type = item.get("from_type")
from_fq_name = item.get("from_fq_name")
from_uuid = self._vnc_api.fq_name_to_id(
from_type, from_fq_name)
to_type = item.get("to_type")
to_fq_name = item.get("to_fq_name")
to_uuid = self._vnc_api.fq_name_to_id(to_type, to_fq_name)
self._vnc_api.ref_update(from_type, from_uuid, to_type,
to_uuid, to_fq_name, 'ADD')
except Exception as e:
err_msg = 'error while loading init data: %s\n' % str(e)
err_msg += detailed_traceback()
self._logger.error(err_msg)
def _Init_Cluster(self):
self.report_status_init()
self._cql_select = self._handle_exceptions(self._cql_select, "SELECT")
self._Insert = self._handle_exceptions(self._Insert, "INSERT")
self._Remove = self._handle_exceptions(self._Remove, "REMOVE")
self._Get_Range = self._handle_exceptions(self._Get_Range, "GET_RANGE")
self._Get_Count = self._handle_exceptions(self._Get_Count, "GET_COUNT")
# Authentication related options
auth_provider = None
if self.options.credential:
auth_provider = connector.auth.PlainTextAuthProvider(
username=self.options.credential.get('username'),
password=self.options.credential.get('password'))
# SSL related options
ssl_options, ssl_context = None, None
if self.options.ssl_enabled:
ssl_context = ssl.SSLContext(SSL_VERSION)
ssl_context.load_verify_locations(self.options.ca_certs)
ssl_context.verify_mode = ssl.CERT_REQUIRED
ssl_context.check_hostname = False
ssl_options = {}
# Profiles related features
ExecutionProfile = connector.cluster.ExecutionProfile
profiles = {
connector.cluster.EXEC_PROFILE_DEFAULT:
ExecutionProfile(
# TODO(sahid): Do we really want QUORUM when counting?
consistency_level=self.ConsistencyLevel,
row_factory=self.RowFactory),
}
# Addresses, ports related options
endpoints = []
for address in self._server_list:
try:
server, port = address.split(':', 1)
endpoints.append((server, int(port)))
except ValueError:
endpoints.append(address)
connector.ProtocolVersion.SUPPORTED_VERSIONS = self.ProtocolVersions
try:
self._cluster = connector.cluster.Cluster(
endpoints,
ssl_options=ssl_options,
ssl_context=ssl_context,
auth_provider=auth_provider,
execution_profiles=profiles,
cql_version=self.CqlVersion)
self._cluster.connect()
except Exception as error:
raise DatabaseUnavailableError("error, {}: {}".format(
error, utils.detailed_traceback()))
# Initializes RW keyspaces
for ks, cf_dict in self.options.rw_keyspaces.items():
keyspace = self.keyspace(ks)
if self.options.reset_config:
self.safe_drop_keyspace(keyspace)
self.safe_create_keyspace(keyspace)
self.ensure_keyspace_replication(keyspace)
# Ensures RO keyspaces are initialized
while not self.are_keyspaces_ready(self.options.ro_keyspaces):
self.options.logger("waiting for keyspaces '{}' to be ready "
"before to continue...".format(
self.options.ro_keyspaces),
level=SandeshLevel.SYS_INFO)
# Let's a chance to an other greenthread to be scheduled.
gevent.sleep(1)
# The CFs are flatten in a dict with the keyspaces' session
# related.
for ks, cf_dict in itertools.chain(self.options.rw_keyspaces.items(),
self.options.ro_keyspaces.items()):
for cf_name in cf_dict:
self.create_session(self.keyspace(ks), cf_name)
# Now we create the tables/CFs if not already alive.
for cf_name in self._cf_dict:
self.safe_create_table(cf_name)
self.ensure_table_properties(cf_name)
self.report_status_up()
def _load_init_data(self):
"""
This function loads init data from a data file specified by the
argument '--fabric_ansible_dir' to the database. The data file
must be in JSON format and follow the format below:
{
"data": [
{
"object_type": "<vnc object type name>",
"objects": [
{
<vnc object payload>
},
...
]
},
...
]
}
Here is an example:
{
"data": [
{
"object_type": "tag",
"objects": [
{
"fq_name": [
"fabric=management_ip"
],
"name": "fabric=management_ip",
"tag_type_name": "fabric",
"tag_value": "management_ip"
}
]
}
]
}
"""
try:
json_data = self._load_json_data()
if json_data is None:
self._logger.error('Unable to load init data')
return
for item in json_data.get("data"):
object_type = item.get("object_type")
# Get the class name from object type
cls_name = CamelCase(object_type)
# Get the class object
cls_ob = str_to_class(cls_name, resource_client.__name__)
# saving the objects to the database
for obj in item.get("objects"):
instance_obj = cls_ob.from_dict(**obj)
# create/update the object
fq_name = instance_obj.get_fq_name()
try:
uuid = self._vnc_api.fq_name_to_id(object_type, fq_name)
if object_type == "tag":
continue
instance_obj.set_uuid(uuid)
self._vnc_api._object_update(object_type, instance_obj)
except NoIdError:
self._vnc_api._object_create(object_type, instance_obj)
for item in json_data.get("refs"):
from_type = item.get("from_type")
from_fq_name = item.get("from_fq_name")
from_uuid = self._vnc_api.fq_name_to_id(from_type, from_fq_name)
to_type = item.get("to_type")
to_fq_name = item.get("to_fq_name")
to_uuid = self._vnc_api.fq_name_to_id(to_type, to_fq_name)
self._vnc_api.ref_update(from_type, from_uuid, to_type,
to_uuid, to_fq_name, 'ADD')
except Exception as e:
err_msg = 'error while loading init data: %s\n' % str(e)
err_msg += detailed_traceback()
self._logger.error(err_msg)