def update_supported_managed_entities(self, device_id, managed_entities):
"""
Update the supported OMCI Managed Entities for this device
:param device_id: (str) ONU Device ID
:param managed_entities: (set) Managed Entity class IDs
"""
try:
me_list = [
ManagedEntity(class_id=class_id,
name=self._managed_entity_to_name(
device_id, class_id))
for class_id in managed_entities
]
data = MibDeviceData()
device_path = self._get_device_path(device_id)
query_data = self._kv_store.get(device_path)
data.ParseFromString(query_data)
now = datetime.utcnow()
data.managed_entities.extend(me_list)
# Update
self._kv_store.set(device_path, data.SerializeToString())
self._modified = now
self.log.debug('save-me-list-complete', device_id=device_id)
except Exception as e:
self.log.exception('add-me-failure', e=e, me_list=managed_entities)
raise
def update_supported_message_types(self, device_id, msg_types):
"""
Update the supported OMCI Managed Entities for this device
:param device_id: (str) ONU Device ID
:param msg_types: (set) Message Type values (ints)
"""
try:
now = datetime.utcnow()
msg_type_list = [
MessageType(message_type=msg_type.value)
for msg_type in msg_types
]
data = MibDeviceData()
device_path = self._get_device_path(device_id)
query_data = self._kv_store.get(device_path)
data.ParseFromString(query_data)
data.message_types.extend(msg_type_list)
# Update
self._kv_store.set(device_path, data.SerializeToString())
self._modified = now
self.log.debug('save-msg-types-complete', device_id=device_id)
except Exception as e:
self.log.exception('add-msg-types-failure',
e=e,
msg_types=msg_types)
raise
def save_mib_data_sync(self, device_id, value):
"""
Save the MIB Data Sync to the database in an easy location to access
:param device_id: (str) ONU Device ID
:param value: (int) Value to save
"""
self.log.debug('save-mds', device_id=device_id, value=value)
try:
if not isinstance(value, int):
raise TypeError('MIB Data Sync is an integer')
if not 0 <= value <= 255:
raise ValueError(
'Invalid MIB-data-sync value {}. Must be 0..255'.format(
value))
data = MibDeviceData()
path = self._get_device_path(device_id)
query_data = self._kv_store.get(path)
data.ParseFromString(query_data)
now = datetime.utcnow()
data.mib_data_sync = value
# Update
self._kv_store.set(path, data.SerializeToString())
self._modified = now
self.log.debug('save-mds-complete', device_id=device_id)
except Exception as e:
self.log.exception('save-mds-exception', device_id=device_id, e=e)
raise
def save_last_sync(self, device_id, value):
"""
Save the Last Sync time to the database in an easy location to access
:param device_id: (str) ONU Device ID
:param value: (DateTime) Value to save
"""
self.log.debug('save-last-sync', device_id=device_id, time=str(value))
try:
if not isinstance(value, datetime):
raise TypeError('Expected a datetime object, got {}'.format(
type(datetime)))
data = MibDeviceData()
path = self._get_device_path(device_id)
query_data = self._kv_store.get(path)
data.ParseFromString(query_data)
now = datetime.utcnow()
data.last_sync_time = self._time_to_string(value)
# Update
self._kv_store.set(path, data.SerializeToString())
self._modified = now
self.log.debug('save-mds-complete', device_id=device_id)
except Exception as e:
self.log.exception('save-last-sync-exception',
device_id=device_id,
e=e)
raise
def add(self, device_id, overwrite=False):
"""
Add a new ONU to database
:param device_id: (str) Device ID of ONU to add
:param overwrite: (bool) Overwrite existing entry if found.
:raises KeyError: If device already exists and 'overwrite' is False
"""
self.log.debug('add-device', device_id=device_id, overwrite=overwrite)
now = datetime.utcnow()
found = False
data = MibDeviceData(device_id=device_id,
created=self._time_to_string(now),
last_sync_time='',
mib_data_sync=0,
version=MibDbExternal.CURRENT_VERSION)
path = self._get_device_path(device_id)
self.log.debug("add-device-path", device_id=device_id, path=path)
try:
#raises KeyError if not found
device = self._kv_store[path]
#device is found at this point
found = True
if not overwrite:
# Device already exists
raise KeyError(
'Device with ID {} already exists in MIB database'.format(
device_id))
self._kv_store[path] = data.SerializeToString()
self._modified = now
except KeyError:
#KeyError after finding the device should be raised
if found:
raise
# Did not exist, add it now
self._kv_store[path] = data.SerializeToString()
self._created = now
self._modified = now
def get_last_sync(self, device_id):
"""
Get the Last Sync Time saved to the database for a device
:param device_id: (str) ONU Device ID
:return: (int) The Value or None if not found
"""
self.log.debug('get-last-sync', device_id=device_id)
try:
data = MibDeviceData()
path = self._get_device_path(device_id)
data.ParseFromString(self.kv_store[path])
return self._string_to_time(data.last_sync_time)
except KeyError:
return None # OMCI MIB_DB entry has not yet been created
except Exception as e:
self.log.exception('get-last-sync-exception', e=e)
raise
def get_mib_data_sync(self, device_id):
"""
Get the MIB Data Sync value last saved to the database for a device
:param device_id: (str) ONU Device ID
:return: (int) The Value or None if not found
"""
self.log.debug('get-mds', device_id=device_id)
try:
data = MibDeviceData()
path = self._get_device_path(device_id)
data.ParseFromString(self._kv_store[path])
return int(data.mib_data_sync)
except KeyError:
return None # OMCI MIB_DB entry has not yet been created
except Exception as e:
self.log.exception('get-mds-exception', device_id=device_id, e=e)
raise
def _create_new_device(self, device_id):
"""
Create an entry for new device object returning device proto object
:param device_id: (str) ONU Device ID
:returns: (MibDeviceData) The new populated device object
"""
now = self._time_to_string(datetime.utcnow())
device_data = MibDeviceData(device_id=device_id,
created=now,
last_sync_time='',
mib_data_sync=0,
version=MibDbExternal.CURRENT_VERSION)
return device_data
def add(self, device_id, overwrite=False):
"""
Add a new ONU to database
:param device_id: (str) Device ID of ONU to add
:param overwrite: (bool) Overwrite existing entry if found.
:raises KeyError: If device already exists and 'overwrite' is False
"""
self.log.debug('add-device', device_id=device_id, overwrite=overwrite)
now = datetime.utcnow()
found = False
root_proxy = self._core.get_proxy('/')
data = MibDeviceData(device_id=device_id,
created=self._time_to_string(now),
last_sync_time='',
mib_data_sync=0,
version=MibDbExternal.CURRENT_VERSION)
try:
dev_proxy = self._device_proxy(device_id)
found = True
if not overwrite:
# Device already exists
raise KeyError('Device with ID {} already exists in MIB database'.
format(device_id))
# Overwrite with new data
data = dev_proxy.get('/', depth=0)
self._root_proxy.update(MibDbExternal.DEVICE_PATH.format(device_id), data)
self._modified = now
except KeyError:
if found:
raise
# Did not exist, add it now
root_proxy.add(MibDbExternal.MIB_PATH, data)
self._created = now
self._modified = now
def on_mib_reset(self, device_id):
"""
Reset/clear the database for a specific Device
:param device_id: (str) ONU Device ID
:raises DatabaseStateError: If the database is not enabled
:raises KeyError: If the device does not exist in the database
"""
self.log.debug('on-mib-reset', device_id=device_id)
data = MibDeviceData()
try:
path = self._get_device_path(device_id)
query_data = self._kv_store.get(path)
if query_data is not None:
data.ParseFromString(query_data)
# data = MibDeviceData(Wipe out any existing class IDs
class_ids = [c.class_id for c in data.classes]
if len(class_ids):
for class_id in class_ids:
class_path = self._get_class_path(device_id, class_id)
# Delete detailed classes and instances
self._kv_store.delete(class_path)
# Reset MIB Data Sync to zero
now = datetime.utcnow()
new_data = MibDeviceData(device_id=device_id,
created=data.created,
last_sync_time=data.last_sync_time,
mib_data_sync=0,
version=MibDbExternal.CURRENT_VERSION)
# Update with blanked out device object
self._kv_store.set(path, new_data.SerializeToString())
self._modified = now
self.log.debug('mib-reset-complete', device_id=device_id)
else:
self.log.warn("mib-reset-no-data-to-reset",
device_id=device_id)
except Exception as e:
self.log.exception('mib-reset-exception', device_id=device_id, e=e)
raise
def on_mib_reset(self, device_id):
"""
Reset/clear the database for a specific Device
:param device_id: (str) ONU Device ID
:raises DatabaseStateError: If the database is not enabled
:raises KeyError: If the device does not exist in the database
"""
self.log.debug('on-mib-reset', device_id=device_id)
try:
device_proxy = self._device_proxy(device_id)
data = device_proxy.get(depth=2)
# Wipe out any existing class IDs
class_ids = [c.class_id for c in data.classes]
if len(class_ids):
for class_id in class_ids:
device_proxy.remove(MibDbExternal.CLASS_PATH.format(class_id))
# Reset MIB Data Sync to zero
now = datetime.utcnow()
data = MibDeviceData(device_id=device_id,
created=data.created,
last_sync_time=data.last_sync_time,
mib_data_sync=0,
version=MibDbExternal.CURRENT_VERSION)
# Update
self._root_proxy.update(MibDbExternal.DEVICE_PATH.format(device_id),
data)
self._modified = now
self.log.debug('mib-reset-complete', device_id=device_id)
except Exception as e:
self.log.exception('mib-reset-exception', device_id=device_id, e=e)
raise
def on_mib_reset(self, device_id):
"""
Reset/clear the database for a specific Device
:param device_id: (str) ONU Device ID
:raises DatabaseStateError: If the database is not enabled
:raises KeyError: If the device does not exist in the database
"""
self.log.debug('on-mib-reset', device_id=device_id)
data = MibDeviceData()
try:
path = self._get_device_path(device_id)
data.ParseFromString(self._kv_store[path])
# data = MibDeviceData(Wipe out any existing class IDs
class_ids = [c.class_id for c in data.classes]
if len(class_ids):
for class_id in class_ids:
classpath = MibDbExternal.CLASS_PATH.format(
device_id, class_id)
del self._kv_store[classpath]
# Reset MIB Data Sync to zero
now = datetime.utcnow()
data = MibDeviceData(device_id=device_id,
created=data.created,
last_sync_time=data.last_sync_time,
mib_data_sync=0,
version=MibDbExternal.CURRENT_VERSION)
# Update
self._kv_store[path] = data.SerializeToString()
self._modified = now
self.log.debug('mib-reset-complete', device_id=device_id)
except Exception as e:
self.log.exception('mib-reset-exception', device_id=device_id, e=e)
raise
except KeyError as e:
self.log.debug("mib-reset-no-data-to-reset", device_id=device_id)
def query(self,
device_id,
class_id=None,
instance_id=None,
attributes=None):
"""
Get database information.
This method can be used to request information from the database to the detailed
level requested
:param device_id: (str) ONU Device ID
:param class_id: (int) Managed Entity class ID
:param instance_id: (int) Managed Entity instance
:param attributes: (list/set or str) Managed Entity instance's attributes
:return: (dict) The value(s) requested. If class/inst/attribute is
not found, an empty dictionary is returned
:raises KeyError: If the requested device does not exist
:raises DatabaseStateError: If the database is not enabled
"""
self.log.debug('query',
device_id=device_id,
class_id=class_id,
entity_id=instance_id,
attributes=attributes)
start_time = datetime.utcnow()
end_time = None
try:
if class_id is None:
# Get full device info. This is painful given the recursive lookups involved!
dev_data = MibDeviceData()
device_path = self._get_device_path(device_id)
query_data = self._kv_store.get(device_path)
if query_data is not None:
dev_data.ParseFromString(query_data)
class_ids = [c.class_id for c in dev_data.classes]
class_data_dict = dict()
if len(class_ids):
for class_id in class_ids:
# Recursively call query with the class_id passed, so below can do what it already does
class_data_dict[class_id] = self.query(
device_id, class_id)
end_time = datetime.utcnow()
data = self._device_to_dict(dev_data, class_data_dict)
else:
self.log.debug('query-no-device', device_id=device_id)
data = dict()
elif instance_id is None:
# Get all instances of the class
class_data = MibClassData()
class_path = self._get_class_path(device_id, class_id)
query_data = self._kv_store.get(class_path)
if query_data is not None:
class_data.ParseFromString(query_data)
end_time = datetime.utcnow()
data = self._class_to_dict(device_id, class_data)
else:
self.log.debug('query-no-class',
device_id=device_id,
class_id=class_id)
data = dict()
else:
# Get all attributes of a specific ME
class_data = MibClassData()
instance_data = None
class_path = self._get_class_path(device_id, class_id)
query_data = self._kv_store.get(class_path)
if query_data is not None:
class_data.ParseFromString(query_data)
end_time = datetime.utcnow()
for inst in class_data.instances:
if inst.instance_id == instance_id:
instance_data = inst
if instance_data is not None:
if attributes is None:
# All Attributes
data = self._instance_to_dict(
device_id, class_id, instance_data)
else:
# Specific attribute(s)
if isinstance(attributes, six.string_types):
attributes = {attributes}
data = {
attr.name: self._string_to_attribute(
device_id, class_id, attr.name, attr.value)
for attr in instance_data.attributes
if attr.name in attributes
}
else:
self.log.debug('query-no-instance',
device_id=device_id,
class_id=class_id,
entity_id=instance_id)
data = dict()
else:
self.log.debug('query-no-class',
device_id=device_id,
class_id=class_id)
data = dict()
return data
except Exception as e:
self.log.exception('query-exception', device_id=device_id, e=e)
raise
finally:
if end_time is not None:
diff = end_time.utcnow() - start_time
# NOTE: Change to 'debug' when checked in, manually change to 'info'
# for development testing.
self.log.debug('db-get-time',
milliseconds=diff.microseconds / 1000,
class_id=class_id,
entity_id=instance_id)
self._statistics['get'].increment(diff.microseconds / 1000)
def delete(self, device_id, class_id, entity_id):
"""
Delete an entity from the database if it exists. If all instances
of a class are deleted, the class is deleted as well.
:param device_id: (str) ONU Device ID
:param class_id: (int) ME Class ID
:param entity_id: (int) ME Entity ID
:returns: (bool) True if the instance was found and deleted. False
if it did not exist.
:raises KeyError: If device does not exist
:raises DatabaseStateError: If the database is not enabled
"""
self.log.debug('delete',
device_id=device_id,
class_id=class_id,
entity_id=entity_id)
if not self._started:
raise DatabaseStateError('The Database is not currently active')
if not isinstance(device_id, six.string_types):
raise TypeError('Device ID should be an string')
if not 0 <= class_id <= 0xFFFF:
raise ValueError('class-id is 0..0xFFFF')
if not 0 <= entity_id <= 0xFFFF:
raise ValueError('instance-id is 0..0xFFFF')
start_time = datetime.utcnow()
try:
now = datetime.utcnow()
class_path = self._get_class_path(device_id, class_id)
class_data = MibClassData()
query_data = self._kv_store.get(class_path)
if query_data is not None:
class_data.ParseFromString(query_data)
inst_index = next(
(index for index in range(len(class_data.instances))
if class_data.instances[index].instance_id == entity_id),
None)
# Remove instance
if inst_index is not None:
del class_data.instances[inst_index]
self._kv_store.set(class_path,
class_data.SerializeToString())
# If resulting class has no instance, remove it as well
if len(class_data.instances) == 0:
self._kv_store.delete(class_path)
# Clean up Device class pointer
dev_data = MibDeviceData()
device_path = self._get_device_path(device_id)
query_data = self._kv_store.get(device_path)
dev_data.ParseFromString(query_data)
class_index = next(
(index for index in range(len(dev_data.classes))
if dev_data.classes[index].class_id == class_id),
None)
if class_index is not None:
del dev_data.classes[class_index]
self._kv_store.set(device_path,
dev_data.SerializeToString())
self._modified = now
return True
else:
self.log.warn('delete-key-not-found',
device_id=device_id,
class_id=class_id,
entity_id=entity_id)
return False # Not found
except Exception as e:
self.log.exception('delete-exception',
device_id=device_id,
class_id=class_id,
entity_id=entity_id,
e=e)
raise
finally:
diff = datetime.utcnow() - start_time
# NOTE: Change to 'debug' when checked in, manually change to 'info'
# for development testing.
self.log.debug('db-delete-time',
milliseconds=diff.microseconds / 1000)
self._statistics['delete'].increment(diff.microseconds / 1000)
def set(self, device_id, class_id, entity_id, attributes):
"""
Set a database value. This should only be called by the MIB synchronizer
and its related tasks
:param device_id: (str) ONU Device ID
:param class_id: (int) ME Class ID
:param entity_id: (int) ME Entity ID
:param attributes: (dict) Attribute dictionary
:returns: (bool) True if the value was saved to the database. False if the
value was identical to the current instance
:raises KeyError: If device does not exist
:raises DatabaseStateError: If the database is not enabled
"""
self.log.debug('set',
device_id=device_id,
class_id=class_id,
entity_id=entity_id,
attributes=attributes)
operation = 'set'
start_time = datetime.utcnow()
try:
if not isinstance(device_id, six.string_types):
raise TypeError('Device ID should be a string')
if not 0 <= class_id <= 0xFFFF:
raise ValueError(
"Invalid Class ID: {}, should be 0..65535".format(
class_id))
if not 0 <= entity_id <= 0xFFFF:
raise ValueError(
"Invalid Instance ID: {}, should be 0..65535".format(
entity_id))
if not isinstance(attributes, dict):
raise TypeError("Attributes should be a dictionary")
if not self._started:
raise DatabaseStateError(
'The Database is not currently active')
class_path = self._get_class_path(device_id, class_id)
class_data = MibClassData()
query_data = self._kv_store.get(class_path)
if query_data is None:
# Here if the class-id does not yet exist in the database
# This is needed to create a "slimmed down" reference to the class in the device object.
# This would be used later if querying the entire device and needed to pull all the classes and instances
new_class_data_ptr = self._create_new_class(
device_id, class_id)
dev_data = MibDeviceData()
device_path = self._get_device_path(device_id)
start_time = datetime.utcnow()
query_data = self._kv_store.get(device_path)
dev_data.ParseFromString(query_data)
dev_data.classes.extend([new_class_data_ptr])
self._kv_store.set(device_path, dev_data.SerializeToString())
# Create fully populated class/entity instance data in its own place in the KV store
new_class_data = self._create_new_class(
device_id, class_id, entity_id, attributes)
self._kv_store.set(class_path,
new_class_data.SerializeToString())
return True
else:
# Here if the class-id exists in the database and we are updating instances or attributes
class_data.ParseFromString(query_data)
inst_data = next((inst for inst in class_data.instances
if inst.instance_id == entity_id), None)
modified = False
new_data = None
if inst_data is None:
# Creating a new instance
operation = 'create'
new_data = self._create_new_instance(
device_id, class_id, entity_id, attributes)
modified = True
else:
# Possibly adding to or updating an existing instance
new_data = self._update_existing_instance(
device_id, class_id, entity_id, attributes, inst_data)
if new_data is not None:
modified = True
if modified:
inst_index = next(
(index for index in range(len(class_data.instances)) if
class_data.instances[index].instance_id == entity_id),
None)
# Delete the old instance
if inst_index is not None:
del class_data.instances[inst_index]
# Add the new/updated instance
class_data.instances.extend([new_data])
self._kv_store.set(class_path,
class_data.SerializeToString())
return modified
except Exception as e:
self.log.exception('set-exception',
device_id=device_id,
class_id=class_id,
entity_id=entity_id,
attributes=attributes,
e=e)
raise
finally:
if start_time is not None:
diff = datetime.utcnow() - start_time
# NOTE: Change to 'debug' when checked in, manually change to 'info'
# for development testing.
self.log.debug('db-{}-time'.format(operation),
milliseconds=diff.microseconds / 1000)
self._statistics[operation].increment(diff.microseconds / 1000)
def query(self,
device_id,
class_id=None,
instance_id=None,
attributes=None):
"""
Get database information.
This method can be used to request information from the database to the detailed
level requested
:param device_id: (str) ONU Device ID
:param class_id: (int) Managed Entity class ID
:param instance_id: (int) Managed Entity instance
:param attributes: (list/set or str) Managed Entity instance's attributes
:return: (dict) The value(s) requested. If class/inst/attribute is
not found, an empty dictionary is returned
:raises KeyError: If the requested device does not exist
:raises DatabaseStateError: If the database is not enabled
"""
self.log.debug('query',
device_id=device_id,
class_id=class_id,
instance_id=instance_id,
attributes=attributes)
start_time = datetime.utcnow()
end_time = None
try:
if class_id is None:
# Get full device info
dev_data = MibDeviceData()
device_path = self._get_device_path(device_id)
dev_data.ParseFromString(self._kv_store[device_path])
end_time = datetime.utcnow()
data = self._device_to_dict(dev_data)
elif instance_id is None:
# Get all instances of the class
try:
class_data = MibClassData()
class_path = self._get_class_path(device_id, class_id)
class_data.ParseFromString(self._kv_store[class_path])
end_time = datetime.utcnow()
data = self._class_to_dict(device_id, class_data)
except KeyError:
data = dict()
else:
# Get all attributes of a specific ME
try:
class_data = MibClassData()
instance_data = None
class_path = self._get_class_path(device_id, class_id)
class_data.ParseFromString(self._kv_store[class_path])
end_time = datetime.utcnow()
for inst in class_data.instances:
if inst.instance_id == instance_id:
instance_data = inst
if instance_data == None:
raise KeyError
if attributes is None:
# All Attributes
data = self._instance_to_dict(device_id, class_id,
instance_data)
else:
# Specific attribute(s)
if isinstance(attributes, six.string_types):
attributes = {attributes}
data = {
attr.name:
self._string_to_attribute(device_id, class_id,
attr.name, attr.value)
for attr in instance_data.attributes
if attr.name in attributes
}
except KeyError:
data = dict()
return data
except KeyError:
self.log.warn('query-no-device', device_id=device_id)
raise
except Exception as e:
self.log.exception('get-last-sync-exception',
device_id=device_id,
e=e)
raise
finally:
if end_time is not None:
diff = end_time.utcnow() - start_time
# NOTE: Change to 'debug' when checked in, manually change to 'info'
# for development testing.
self.log.debug('db-get-time',
milliseconds=diff.microseconds / 1000,
class_id=class_id,
instance_id=instance_id)
self._statistics['get'].increment(diff.microseconds / 1000)