def list(self, vpoolguid=None, storagerouterguid=None, query=None):
"""
Overview of all vDisks
:param vpoolguid: Guid of the vPool to retrieve its disks
:type vpoolguid: str
:param storagerouterguid: Guid of the StorageRouter to retrieve its disks
:type storagerouterguid: str
:param query: A query to be executed if required
:type query: DataQuery
"""
if vpoolguid is not None:
vpool = VPool(vpoolguid)
vdisks = vpool.vdisks
elif storagerouterguid is not None:
storagerouter = StorageRouter(storagerouterguid)
vdisks = DataList(
VDisk, {
'type':
DataList.where_operator.AND,
'items': [('guid', DataList.operator.IN,
storagerouter.vdisks_guids)]
})
else:
vdisks = VDiskList.get_vdisks()
if query is not None:
query_vdisk_guids = DataList(VDisk, query).guids
vdisks = [
vdisk for vdisk in vdisks if vdisk.guid in query_vdisk_guids
]
return vdisks
def list(self, discover=False, ip=None, node_id=None):
"""
Lists all available ALBA Nodes
:param discover: If True and IP provided, return list of single ALBA node, If True and no IP provided, return all ALBA nodes else return modeled ALBA nodes
:param ip: IP of ALBA node to retrieve
:param node_id: ID of the ALBA node
"""
if discover is False and (ip is not None or node_id is not None):
raise RuntimeError('Discover is mutually exclusive with IP and nodeID')
if (ip is None and node_id is not None) or (ip is not None and node_id is None):
raise RuntimeError('Both IP and nodeID need to be specified')
if discover is False:
return AlbaNodeList.get_albanodes()
if ip is not None:
node = AlbaNode(volatile=True)
node.ip = ip
node.type = 'ASD'
node.node_id = node_id
node.port = EtcdConfiguration.get('/ovs/alba/asdnodes/{0}/config/main|port'.format(node_id))
node.username = EtcdConfiguration.get('/ovs/alba/asdnodes/{0}/config/main|username'.format(node_id))
node.password = EtcdConfiguration.get('/ovs/alba/asdnodes/{0}/config/main|password'.format(node_id))
data = node.client.get_metadata()
if data['_success'] is False and data['_error'] == 'Invalid credentials':
raise RuntimeError('Invalid credentials')
if data['node_id'] != node_id:
raise RuntimeError('Unexpected node identifier. {0} vs {1}'.format(data['node_id'], node_id))
node_list = DataList(AlbaNode, {})
node_list._executed = True
node_list._guids = [node.guid]
node_list._objects = {node.guid: node}
node_list._data = {node.guid: {'guid': node.guid, 'data': node._data}}
return node_list
nodes = {}
model_node_ids = [node.node_id for node in AlbaNodeList.get_albanodes()]
found_node_ids = []
asd_node_ids = []
if EtcdConfiguration.dir_exists('/ovs/alba/asdnodes'):
asd_node_ids = EtcdConfiguration.list('/ovs/alba/asdnodes')
for node_id in asd_node_ids:
node = AlbaNode(volatile=True)
node.type = 'ASD'
node.node_id = node_id
node.ip = EtcdConfiguration.get('/ovs/alba/asdnodes/{0}/config/main|ip'.format(node_id))
node.port = EtcdConfiguration.get('/ovs/alba/asdnodes/{0}/config/main|port'.format(node_id))
node.username = EtcdConfiguration.get('/ovs/alba/asdnodes/{0}/config/main|username'.format(node_id))
node.password = EtcdConfiguration.get('/ovs/alba/asdnodes/{0}/config/main|password'.format(node_id))
if node.node_id not in model_node_ids and node.node_id not in found_node_ids:
nodes[node.guid] = node
found_node_ids.append(node.node_id)
node_list = DataList(AlbaNode, {})
node_list._executed = True
node_list._guids = nodes.keys()
node_list._objects = nodes
node_list._data = dict([(node.guid, {'guid': node.guid, 'data': node._data}) for node in nodes.values()])
return node_list
def test_primarykeys(self):
"""
Validates whether the primary keys are kept in sync
"""
disk = TestDisk()
disk.name = 'disk'
keys = DataList.get_pks(disk._namespace, disk._name)
self.assertEqual(len(keys), 0, 'There should be no primary keys ({0})'.format(len(keys)))
disk.save()
keys = DataList.get_pks(disk._namespace, disk._name)
self.assertEqual(len(keys), 1, 'There should be one primary key ({0})'.format(len(keys)))
disk.delete()
keys = DataList.get_pks(disk._namespace, disk._name)
self.assertEqual(len(keys), 0, 'There should be no primary keys ({0})'.format(len(keys)))
def _get_list_property(self, attribute):
"""
Getter for the list property
It will execute the related query every time to return a list of hybrid objects that
refer to this object. The resulting data will be stored or merged into the cached list
preserving as much already loaded objects as possible
"""
info = self._objects[attribute]['info']
remote_class = Descriptor().load(info['class']).get_object()
remote_key = info['key']
datalist = DataList.get_relation_set(remote_class, remote_key,
self.__class__, attribute,
self.guid)
if self._objects[attribute]['data'] is None:
self._objects[attribute]['data'] = DataObjectList(
datalist.data, remote_class)
else:
self._objects[attribute]['data'].update(datalist.data)
if info['list'] is True:
return self._objects[attribute]['data']
else:
data = self._objects[attribute]['data']
if len(data) > 1:
raise InvalidRelationException(
'More than one element found in {0}'.format(attribute))
return data[0] if len(data) == 1 else None
def list(self, vpoolguid=None, storagerouterguid=None):
"""
Overview of all vDisks
:param vpoolguid: Guid of the vPool to retrieve its disks
:type vpoolguid: str
:param storagerouterguid: Guid of the StorageRouter to retrieve its disks
:type storagerouterguid: str
:return: List of vDisks matching the parameters specified
:rtype: list[ovs.dal.hybrids.vdisk.VDisk]
"""
if vpoolguid is not None:
vpool = VPool(vpoolguid)
vdisks = vpool.vdisks
elif storagerouterguid is not None:
storagerouter = StorageRouter(storagerouterguid)
vdisks = DataList(
VDisk, {
'type':
DataList.where_operator.AND,
'items': [('guid', DataList.operator.IN,
storagerouter.vdisks_guids)]
})
else:
vdisks = VDiskList.get_vdisks()
return vdisks
def get_vpools():
"""
Returns a list of all VPools
"""
return DataList(VPool, {
'type': DataList.where_operator.AND,
'items': []
})
def list(self, query=None):
"""
Overview of all backend types
"""
if query is not None:
query = json.loads(query)
return DataList(BackendType, query)
return BackendTypeList.get_backend_types()
def get_licenses():
"""
Returns a list of all Licenses
"""
return DataList(License, {
'type': DataList.where_operator.AND,
'items': []
})
def get_albanodes():
"""
Returns a list of all AlbaNodes
"""
return DataList(AlbaNode, {
'type': DataList.where_operator.AND,
'items': []
})
def get_services():
"""
Get all services of all types
"""
return DataList(Service, {
'type': DataList.where_operator.AND,
'items': []
})
def get_users():
"""
Returns a list of all Users
"""
return DataList(User, {
'type': DataList.where_operator.AND,
'items': []
})
def get_partitions():
"""
Returns a list of all Partitions
"""
return DataList(DiskPartition, {
'type': DataList.where_operator.AND,
'items': []
})
def get_mgmtcenters():
"""
Returns a list of MgmtCenters
"""
return DataList(MgmtCenter, {
'type': DataList.where_operator.AND,
'items': []
})
def get_vdisks():
"""
Returns a list of all VDisks
"""
return DataList(VDisk, {
'type': DataList.where_operator.AND,
'items': []
})
def get_backends():
"""
Returns a list of all Backends
"""
return DataList(Backend, {
'type': DataList.where_operator.AND,
'items': []
})
def get_pmachines():
"""
Returns a list of all PMachines
"""
return DataList(PMachine, {
'type': DataList.where_operator.AND,
'items': []
})
def get_servicetypes():
"""
Returns a list of all ServiceTypes
"""
return DataList(ServiceType, {
'type': DataList.where_operator.AND,
'items': []
})
def get_failure_domains():
"""
Returns a list of all failure domains
"""
return DataList(FailureDomain, {
'type': DataList.where_operator.AND,
'items': []
})
def get_storagerouters():
"""
Returns a list of all StorageRouters
"""
return DataList(StorageRouter, {
'type': DataList.where_operator.AND,
'items': []
})
def get_storagedrivers_by_storagerouter(machineguid):
"""
Returns a list of all StorageDrivers for Storage Router
"""
storagedrivers = DataList({'object': StorageDriver,
'data': DataList.select.GUIDS,
'query': {'type': DataList.where_operator.AND,
'items': [('storagerouter_guid', DataList.operator.EQUALS, machineguid)]}}).data
return DataObjectList(storagedrivers, StorageDriver)
def get_storagedrivers():
"""
Returns a list of all StorageDrivers
"""
storagedrivers = DataList({'object': StorageDriver,
'data': DataList.select.GUIDS,
'query': {'type': DataList.where_operator.AND,
'items': []}}).data
return DataObjectList(storagedrivers, StorageDriver)
def get_vtemplates():
"""
Returns vTemplates
"""
return DataList(
VMachine, {
'type': DataList.where_operator.AND,
'items': [('is_vtemplate', DataList.operator.EQUALS, True)]
})
def get_vtemplates():
"""
Returns vTemplates
"""
vmachines = DataList({'object': VMachine,
'data': DataList.select.GUIDS,
'query': {'type': DataList.where_operator.AND,
'items': [('is_vtemplate', DataList.operator.EQUALS, True)]}}).data
return DataObjectList(vmachines, VMachine)
def get_alba_node_clusters():
"""
Returns a list of all ALBABackends
:rtype: DataList[ovs.dal.hybrids.albanodecluster.AlbaNodeCluster]
"""
return DataList(AlbaNodeCluster, {
'type': DataList.where_operator.AND,
'items': []
})
def get_groups():
"""
Returns a list of all Groups
"""
groups = DataList({'object': Group,
'data': DataList.select.GUIDS,
'query': {'type': DataList.where_operator.AND,
'items': []}}).data
return DataObjectList(groups, Group)
def get_without_vmachine():
"""
Gets all vDisks without a vMachine
"""
return DataList(
VDisk, {
'type': DataList.where_operator.AND,
'items': [('vmachine_guid', DataList.operator.EQUALS, None)]
})
def get_in_volume_ids(volume_ids):
"""
Returns all vDisks which volume_id is in the given list
"""
return DataList(
VDisk, {
'type': DataList.where_operator.AND,
'items': [('volume_id', DataList.operator.IN, volume_ids)]
})
def get_masters():
"""
Get all MASTER StorageRouters
"""
return DataList(
StorageRouter, {
'type': DataList.where_operator.AND,
'items': [('node_type', DataList.operator.EQUALS, 'MASTER')]
})
def get_partitions():
"""
Returns a list of all Partitions
"""
partitions = DataList({'object': DiskPartition,
'data': DataList.select.GUIDS,
'query': {'type': DataList.where_operator.AND,
'items': []}}).data
return DataObjectList(partitions, DiskPartition)
def get_slaves():
"""
Get all SLAVE StorageRouters
"""
return DataList(
StorageRouter, {
'type': DataList.where_operator.AND,
'items': [('node_type', DataList.operator.EQUALS, 'EXTRA')]
})
def get_vmachines():
"""
Returns a list of all VMachines
"""
vmachines = DataList({'object': VMachine,
'data': DataList.select.GUIDS,
'query': {'type': DataList.where_operator.AND,
'items': []}}).data
return DataObjectList(vmachines, VMachine)
def get_brandings():
"""
Returns a list of all brandings
"""
brandings = DataList({'object': Branding,
'data': DataList.select.GUIDS,
'query': {'type': DataList.where_operator.AND,
'items': []}}).data
return DataObjectList(brandings, Branding)
def _clean_all(self):
"""
Cleans all disks and machines
"""
machine = TestMachine()
keys = DataList.get_pks(machine._namespace, machine._classname)
for guid in keys:
try:
machine = TestMachine(guid)
for disk in machine.disks:
disk.delete()
machine.delete()
except (ObjectNotFoundException, ValueError):
pass
disk = TestDisk()
keys = DataList.get_pks(disk._namespace, disk._classname)
for guid in keys:
try:
disk = TestDisk(guid)
disk.delete()
except (ObjectNotFoundException, ValueError):
pass
def _get_list_property(self, attribute):
"""
Getter for the list property
It will execute the related query every time to return a list of hybrid objects that
refer to this object. The resulting data will be stored or merged into the cached list
preserving as much already loaded objects as possible
"""
info = self._objects[attribute]['info']
remote_class = Descriptor().load(info['class']).get_object()
remote_key = info['key']
datalist = DataList.get_relation_set(remote_class, remote_key, self.__class__, attribute, self.guid)
if self._objects[attribute]['data'] is None:
self._objects[attribute]['data'] = DataObjectList(datalist.data, remote_class)
else:
self._objects[attribute]['data'].merge(datalist.data)
if info['list'] is True:
return self._objects[attribute]['data']
else:
data = self._objects[attribute]['data']
return data[0] if len(data) == 1 else None
def test_lotsofobjects(self):
"""
A test creating, linking and querying a lot of objects
"""
print ''
print 'cleaning up'
self._clean_all()
print 'start test'
tstart = time.time()
if getattr(LotsOfObjects, 'amount_of_machines', None) is None:
LotsOfObjects.amount_of_machines = 50
if getattr(LotsOfObjects, 'amount_of_disks', None) is None:
LotsOfObjects.amount_of_disks = 5
load_data = True
mguids = []
if load_data:
print '\nstart loading data'
start = time.time()
runtimes = []
for i in xrange(0, int(LotsOfObjects.amount_of_machines)):
mstart = time.time()
machine = TestMachine()
machine.name = 'machine_{0}'.format(i)
machine.save()
mguids.append(machine.guid)
for ii in xrange(0, int(LotsOfObjects.amount_of_disks)):
disk = TestDisk()
disk.name = 'disk_{0}_{1}'.format(i, ii)
disk.size = ii * 100
disk.machine = machine
disk.save()
avgitemspersec = ((i + 1) * LotsOfObjects.amount_of_disks) / (time.time() - start)
itemspersec = LotsOfObjects.amount_of_disks / (time.time() - mstart)
runtimes.append(itemspersec)
LotsOfObjects._print_progress('* machine {0}/{1} (run: {2} dps, avg: {3} dps)'.format(i + 1, int(LotsOfObjects.amount_of_machines), round(itemspersec, 2), round(avgitemspersec, 2)))
runtimes.sort()
print '\nloading done ({0}s). min: {1} dps, max: {2} dps'.format(round(time.time() - tstart, 2), round(runtimes[1], 2), round(runtimes[-2], 2))
test_queries = True
if test_queries:
print '\nstart queries'
start = time.time()
runtimes = []
for i in xrange(0, int(LotsOfObjects.amount_of_machines)):
mstart = time.time()
machine = TestMachine(mguids[i])
assert len(machine.disks) == LotsOfObjects.amount_of_disks, 'Not all disks were retrieved ({0})'.format(len(machine.disks))
avgitemspersec = ((i + 1) * LotsOfObjects.amount_of_disks) / (time.time() - start)
itemspersec = LotsOfObjects.amount_of_disks / (time.time() - mstart)
runtimes.append(itemspersec)
LotsOfObjects._print_progress('* machine {0}/{1} (run: {2} dps, avg: {3} dps)'.format(i + 1, int(LotsOfObjects.amount_of_machines), round(itemspersec, 2), round(avgitemspersec, 2)))
runtimes.sort()
print '\ncompleted ({0}s). min: {1} dps, max: {2} dps'.format(round(time.time() - tstart, 2), round(runtimes[1], 2), round(runtimes[-2], 2))
print '\nstart full query on disk property'
start = time.time()
dlist = DataList(TestDisk, {'type': DataList.where_operator.AND,
'items': [('size', DataList.operator.GT, 100),
('size', DataList.operator.LT, (LotsOfObjects.amount_of_disks - 1) * 100)]})
amount = len(dlist)
assert amount == (LotsOfObjects.amount_of_disks - 3) * LotsOfObjects.amount_of_machines, 'Incorrect amount of disks. Found {0} instead of {1}'.format(amount, int((LotsOfObjects.amount_of_disks - 3) * LotsOfObjects.amount_of_machines))
seconds_passed = (time.time() - start)
print 'completed ({0}s) in {1} seconds (avg: {2} dps)'.format(round(time.time() - tstart, 2), round(seconds_passed, 2), round(LotsOfObjects.amount_of_machines * LotsOfObjects.amount_of_disks / seconds_passed, 2))
print '\nloading disks (all)'
start = time.time()
for i in xrange(0, int(LotsOfObjects.amount_of_machines)):
machine = TestMachine(mguids[i])
_ = [_ for _ in machine.disks]
seconds_passed = (time.time() - start)
print 'completed ({0}s) in {1} seconds (avg: {2} dps)'.format(round(time.time() - tstart, 2), round(seconds_passed, 2), round(LotsOfObjects.amount_of_machines * LotsOfObjects.amount_of_disks / seconds_passed, 2))
print '\nstart full query on disk property (using cached objects)'
dlist._volatile.delete(dlist._key)
start = time.time()
dlist = DataList(TestDisk, {'type': DataList.where_operator.AND,
'items': [('size', DataList.operator.GT, 100),
('size', DataList.operator.LT, (LotsOfObjects.amount_of_disks - 1) * 100)]})
amount = len(dlist)
assert amount == (LotsOfObjects.amount_of_disks - 3) * LotsOfObjects.amount_of_machines, 'Incorrect amount of disks. Found {0} instead of {1}'.format(amount, int((LotsOfObjects.amount_of_disks - 3) * LotsOfObjects.amount_of_machines))
seconds_passed = (time.time() - start)
print 'completed ({0}s) in {1} seconds (avg: {2} dps)'.format(round(time.time() - tstart, 2), round(seconds_passed, 2), round(LotsOfObjects.amount_of_machines * LotsOfObjects.amount_of_disks / seconds_passed, 2))
print '\nstart property sort'
dlist = DataList(TestDisk, {'type': DataList.where_operator.AND,
'items': []})
start = time.time()
dlist.sort(key=lambda a: Toolbox.extract_key(a, 'size'))
seconds_passed = (time.time() - start)
print 'completed ({0}s) in {1} seconds (avg: {2} dps)'.format(round(time.time() - tstart, 2), round(seconds_passed, 2), round(LotsOfObjects.amount_of_machines * LotsOfObjects.amount_of_disks / seconds_passed, 2))
print '\nstart dynamic sort'
dlist._volatile.delete(dlist._key)
dlist = DataList(TestDisk, {'type': DataList.where_operator.AND,
'items': []})
start = time.time()
dlist.sort(key=lambda a: Toolbox.extract_key(a, 'predictable'))
seconds_passed = (time.time() - start)
print 'completed ({0}s) in {1} seconds (avg: {2} dps)'.format(round(time.time() - tstart, 2), round(seconds_passed, 2), round(LotsOfObjects.amount_of_machines * LotsOfObjects.amount_of_disks / seconds_passed, 2))
clean_data = True
if clean_data:
print '\ncleaning up'
start = time.time()
runtimes = []
for i in xrange(0, int(LotsOfObjects.amount_of_machines)):
mstart = time.time()
machine = TestMachine(mguids[i])
for disk in machine.disks:
disk.delete()
machine.delete()
avgitemspersec = ((i + 1) * LotsOfObjects.amount_of_disks) / (time.time() - start)
itemspersec = LotsOfObjects.amount_of_disks / (time.time() - mstart)
runtimes.append(itemspersec)
LotsOfObjects._print_progress('* machine {0}/{1} (run: {2} dps, avg: {3} dps)'.format(i + 1, int(LotsOfObjects.amount_of_machines), round(itemspersec, 2), round(avgitemspersec, 2)))
runtimes.sort()
print '\ncompleted ({0}s). min: {1} dps, max: {2} dps'.format(round(time.time() - tstart, 2), round(runtimes[1], 2), round(runtimes[-2], 2))
def test_pk_stretching(self):
"""
Validates whether the primary key lists scale correctly.
* X entries will be added (e.g. 10000)
* X/2 random entries will be deleted (5000)
* X/2 entries will be added again (5000)
* X entries will be removed (10000)
No entries should be remaining
"""
print ''
print 'starting test'
amount_of_objects = 10000 # Must be an even number!
machine = TestMachine()
runtimes = []
# First fill
start = time.time()
keys = DataList._get_pks(machine._namespace, machine._name)
self.assertEqual(len(list(keys)), 0, 'There should be no primary keys yet ({0})'.format(len(list(keys))))
guids = []
mstart = time.time()
for i in xrange(0, amount_of_objects):
guid = str(uuid.uuid4())
guids.append(guid)
DataList.add_pk(machine._namespace, machine._name, guid)
keys = DataList._get_pks(machine._namespace, machine._name)
self.assertEqual(len(list(keys)), len(guids), 'There should be {0} primary keys instead of {1}'.format(len(guids), len(list(keys))))
if i % 100 == 99:
avgitemspersec = (i + 1) / (time.time() - start)
itemspersec = 100 / (time.time() - mstart)
runtimes.append(itemspersec)
self._print_progress('* adding object {0}/{1} (run: {2} ops, avg: {3} ops)'.format(i + 1, int(amount_of_objects), round(itemspersec, 2), round(avgitemspersec, 2)))
mstart = time.time()
print ''
# First delete
amount_of_objects /= 2
shuffle(guids) # Make the test a bit more realistic
guids_copy = guids[:]
dstart = time.time()
mstart = time.time()
for i in xrange(0, amount_of_objects):
guid = guids_copy[i]
guids.remove(guid)
DataList.delete_pk(machine._namespace, machine._name, guid)
keys = DataList._get_pks(machine._namespace, machine._name)
self.assertEqual(len(list(keys)), len(guids), 'There should be {0} primary keys instead of {1}'.format(len(guids), len(list(keys))))
if i % 100 == 99:
avgitemspersec = (i + 1) / (time.time() - dstart)
itemspersec = 100 / (time.time() - mstart)
runtimes.append(itemspersec)
self._print_progress('* delete object {0}/{1} (run: {2} ops, avg: {3} ops)'.format(i + 1, int(amount_of_objects), round(itemspersec, 2), round(avgitemspersec, 2)))
mstart = time.time()
keys = DataList._get_pks(machine._namespace, machine._name)
self.assertEqual(len(list(keys)), amount_of_objects, 'There should be {0} primary keys ({1})'.format(amount_of_objects, len(list(keys))))
print ''
# Second round
sstart = time.time()
mstart = time.time()
for i in xrange(0, amount_of_objects):
guid = str(uuid.uuid4())
guids.append(guid)
DataList.add_pk(machine._namespace, machine._name, guid)
keys = DataList._get_pks(machine._namespace, machine._name)
self.assertEqual(len(list(keys)), len(guids), 'There should be {0} primary keys instead of {1}'.format(len(guids), len(list(keys))))
if i % 100 == 99:
avgitemspersec = (i + 1) / (time.time() - sstart)
itemspersec = 100 / (time.time() - mstart)
runtimes.append(itemspersec)
self._print_progress('* adding object {0}/{1} (run: {2} ops, avg: {3} ops)'.format(i + 1, int(amount_of_objects), round(itemspersec, 2), round(avgitemspersec, 2)))
mstart = time.time()
print ''
# Second delete
amount_of_objects *= 2
shuffle(guids) # Make the test a bit more realistic
guids_copy = guids[:]
dstart = time.time()
mstart = time.time()
for i in xrange(0, amount_of_objects):
guid = guids_copy[i]
guids.remove(guid)
DataList.delete_pk(machine._namespace, machine._name, guid)
keys = DataList._get_pks(machine._namespace, machine._name)
self.assertEqual(len(list(keys)), len(guids), 'There should be {0} primary keys instead of {1}'.format(len(guids), len(list(keys))))
if i % 100 == 99:
avgitemspersec = (i + 1) / (time.time() - dstart)
itemspersec = 100 / (time.time() - mstart)
runtimes.append(itemspersec)
self._print_progress('* delete object {0}/{1} (run: {2} ops, avg: {3} ops)'.format(i + 1, int(amount_of_objects), round(itemspersec, 2), round(avgitemspersec, 2)))
mstart = time.time()
keys = DataList._get_pks(machine._namespace, machine._name)
self.assertEqual(len(guids), 0, 'All guids should be removed. {0} left'.format(len(guids)))
self.assertEqual(len(list(keys)), 0, 'There should be no primary keys ({0})'.format(len(list(keys))))
seconds_passed = (time.time() - start)
runtimes.sort()
print '\ncompleted in {0} seconds (avg: {1} ops, min: {2} ops, max: {3} ops)'.format(round(seconds_passed, 2), round((amount_of_objects * 3) / seconds_passed, 2), round(runtimes[1], 2), round(runtimes[-2], 2))
def save(self, recursive=False, skip=None):
"""
Save the object to the persistent backend and clear cache, making use
of the specified conflict resolve settings.
It will also invalidate certain caches if required. For example lists pointing towards this
object
"""
invalid_fields = []
for prop in self._properties:
if prop.mandatory is True and self._data[prop.name] is None:
invalid_fields.append(prop.name)
for relation in self._relations:
if relation.mandatory is True and self._data[relation.name]['guid'] is None:
invalid_fields.append(relation.name)
if len(invalid_fields) > 0:
raise MissingMandatoryFieldsException('Missing fields on {0}: {1}'.format(self._name, ', '.join(invalid_fields)))
if recursive:
# Save objects that point to us (e.g. disk.vmachine - if this is disk)
for relation in self._relations:
if relation.name != skip: # disks will be skipped
item = getattr(self, relation.name)
if item is not None:
item.save(recursive=True, skip=relation.foreign_key)
# Save object we point at (e.g. machine.disks - if this is machine)
relations = RelationMapper.load_foreign_relations(self.__class__)
if relations is not None:
for key, info in relations.iteritems():
if key != skip: # machine will be skipped
if info['list'] is True:
for item in getattr(self, key).iterloaded():
item.save(recursive=True, skip=info['key'])
else:
item = getattr(self, key)
if item is not None:
item.save(recursive=True, skip=info['key'])
try:
data = self._persistent.get(self._key)
except KeyNotFoundException:
if self._new:
data = {}
else:
raise ObjectNotFoundException('{0} with guid \'{1}\' was deleted'.format(
self.__class__.__name__, self._guid
))
changed_fields = []
data_conflicts = []
for attribute in self._data.keys():
if self._data[attribute] != self._original[attribute]:
# We changed this value
changed_fields.append(attribute)
if attribute in data and self._original[attribute] != data[attribute]:
# Some other process also wrote to the database
if self._datastore_wins is None:
# In case we didn't set a policy, we raise the conflicts
data_conflicts.append(attribute)
elif self._datastore_wins is False:
# If the datastore should not win, we just overwrite the data
data[attribute] = self._data[attribute]
# If the datastore should win, we discard/ignore our change
else:
# Normal scenario, saving data
data[attribute] = self._data[attribute]
elif attribute not in data:
data[attribute] = self._data[attribute]
if data_conflicts:
raise ConcurrencyException('Got field conflicts while saving {0}. Conflicts: {1}'.format(
self._name, ', '.join(data_conflicts)
))
# Refresh internal data structure
self._data = copy.deepcopy(data)
# First, update reverse index
try:
self._mutex_reverseindex.acquire(60)
for relation in self._relations:
key = relation.name
original_guid = self._original[key]['guid']
new_guid = self._data[key]['guid']
if original_guid != new_guid:
if relation.foreign_type is None:
classname = self.__class__.__name__.lower()
else:
classname = relation.foreign_type.__name__.lower()
if original_guid is not None:
reverse_key = 'ovs_reverseindex_{0}_{1}'.format(classname, original_guid)
reverse_index = self._volatile.get(reverse_key)
if reverse_index is not None:
if relation.foreign_key in reverse_index:
entries = reverse_index[relation.foreign_key]
if self.guid in entries:
entries.remove(self.guid)
reverse_index[relation.foreign_key] = entries
self._volatile.set(reverse_key, reverse_index)
if new_guid is not None:
reverse_key = 'ovs_reverseindex_{0}_{1}'.format(classname, new_guid)
reverse_index = self._volatile.get(reverse_key)
if reverse_index is not None:
if relation.foreign_key in reverse_index:
entries = reverse_index[relation.foreign_key]
if self.guid not in entries:
entries.append(self.guid)
reverse_index[relation.foreign_key] = entries
self._volatile.set(reverse_key, reverse_index)
else:
reverse_index[relation.foreign_key] = [self.guid]
self._volatile.set(reverse_key, reverse_index)
else:
reverse_index = {relation.foreign_key: [self.guid]}
self._volatile.set(reverse_key, reverse_index)
reverse_key = 'ovs_reverseindex_{0}_{1}'.format(self._name, self.guid)
reverse_index = self._volatile.get(reverse_key)
if reverse_index is None:
reverse_index = {}
relations = RelationMapper.load_foreign_relations(self.__class__)
if relations is not None:
for key, _ in relations.iteritems():
reverse_index[key] = []
self._volatile.set(reverse_key, reverse_index)
finally:
self._mutex_reverseindex.release()
# Second, invalidate property lists
try:
self._mutex_listcache.acquire(60)
cache_key = '{0}_{1}'.format(DataList.cachelink, self._name)
cache_list = Toolbox.try_get(cache_key, {})
change = False
for list_key in cache_list.keys():
fields = cache_list[list_key]
if ('__all' in fields and self._new) or list(set(fields) & set(changed_fields)):
change = True
self._volatile.delete(list_key)
del cache_list[list_key]
if change is True:
self._volatile.set(cache_key, cache_list)
self._persistent.set(cache_key, cache_list)
finally:
self._mutex_listcache.release()
# Save the data
self._persistent.set(self._key, self._data)
DataList.add_pk(self._namespace, self._name, self._guid)
# Invalidate the cache
self._volatile.delete(self._key)
self._original = copy.deepcopy(self._data)
self.dirty = False
self._new = False
def delete(self, abandon=False):
"""
Delete the given object. It also invalidates certain lists
"""
# Check foreign relations
relations = RelationMapper.load_foreign_relations(self.__class__)
if relations is not None:
for key, info in relations.iteritems():
items = getattr(self, key)
if info['list'] is True:
if len(items) > 0:
if abandon is True:
for item in items.itersafe():
setattr(item, info['key'], None)
try:
item.save()
except ObjectNotFoundException:
pass
else:
raise LinkedObjectException('There are {0} items left in self.{1}'.format(len(items), key))
elif items is not None:
# No list (so a 1-to-1 relation), so there should be an object, or None
item = items # More clear naming
if abandon is True:
setattr(item, info['key'], None)
try:
item.save()
except ObjectNotFoundException:
pass
else:
raise LinkedObjectException('There is still an item linked in self.{0}'.format(key))
# First, update reverse index
try:
self._mutex_reverseindex.acquire(60)
for relation in self._relations:
key = relation.name
original_guid = self._original[key]['guid']
if original_guid is not None:
if relation.foreign_type is None:
classname = self.__class__.__name__.lower()
else:
classname = relation.foreign_type.__name__.lower()
reverse_key = 'ovs_reverseindex_{0}_{1}'.format(classname, original_guid)
reverse_index = self._volatile.get(reverse_key)
if reverse_index is not None:
if relation.foreign_key in reverse_index:
entries = reverse_index[relation.foreign_key]
if self.guid in entries:
entries.remove(self.guid)
reverse_index[relation.foreign_key] = entries
self._volatile.set(reverse_key, reverse_index)
self._volatile.delete('ovs_reverseindex_{0}_{1}'.format(self._name, self.guid))
finally:
self._mutex_reverseindex.release()
# Second, invalidate property lists
try:
self._mutex_listcache.acquire(60)
cache_key = '{0}_{1}'.format(DataList.cachelink, self._name)
cache_list = Toolbox.try_get(cache_key, {})
change = False
for list_key in cache_list.keys():
fields = cache_list[list_key]
if '__all' in fields:
change = True
self._volatile.delete(list_key)
del cache_list[list_key]
if change is True:
self._volatile.set(cache_key, cache_list)
self._persistent.set(cache_key, cache_list)
finally:
self._mutex_listcache.release()
# Delete the object out of the persistent store
try:
self._persistent.delete(self._key)
except KeyNotFoundException:
pass
# Delete the object and its properties out of the volatile store
self.invalidate_dynamics()
self._volatile.delete(self._key)
DataList.delete_pk(self._namespace, self._name, self._guid)
