def setUp(self):
d1 = {
'ahhhh': ['111', '222', ['jajja', 1], 3],
'dict': {
'hhhh': 11,
'a': 2,
'dict': {
'a': 1
}
}
}
d2 = {
'ahhhh': ['111', '222', ['jajja', 1, 2]],
'dict': {
'h': 2,
'a': [2, 3],
'dict': {
'a': 2
}
}
}
self.d1 = {'shit': d1, 'tu': d2, 'uuu': d1}
self.d2 = {'shat': d1, 'tu': d1}
self.differ_DictDiffer = DictDiffer().diff
self.differ_dictdiffer = dictdiffer.diff
def __init__(self,
old_obj,
new_obj,
previous_path=(),
previous_new_path=(),
patch_extractors=[],
key_limits=KeyLimit(),
find_moved_patches=False,
moved_patches_similarity=0.8):
super(DictPatchExtractor,
self).__init__(old_obj, new_obj, previous_path,
previous_new_path, patch_extractors, key_limits,
moved_patches_similarity)
dict_diff = DictDiffer(new_obj, old_obj)
for addition_key in dict_diff.added():
if not self._try_patch_extractors_for_ungrouping(addition_key):
self._add_patch('add', addition_key, None,
new_obj[addition_key])
for removal_key in dict_diff.removed():
self._add_patch('remove', removal_key, old_obj[removal_key], None)
for change_key in dict_diff.changed():
if not self._try_patch_extractors(change_key, change_key):
self._add_patch('change', change_key, old_obj[change_key],
new_obj[change_key])
if find_moved_patches:
self.patches = self._find_moved_parts()
def onSaveButtonClicked(self):
# Set up some variables.
rowcount = self.ui.evTableWidget.rowCount()
evdict = {}
winEnv = windowsev.Win32Environment('system')
# Grab all of the values from the tablewidget and save them.
for i in range(0, rowcount):
evKey = self.ui.evTableWidget.item(i, 0).text()
evValue = self.ui.evTableWidget.item(i, 1).text()
# Save in the dictionary.
evdict[evKey] = evValue
self.newenvs = evdict
differ = DictDiffer(self.newenvs, self.oldenvs)
changed = differ.changed()
for k in changed:
print('Key: ' + k + ' -> Value: ' + self.newenvs[k])
def get_new_connections_with_interfaces_since_last_distance_matrix_change(self):
""" Get only new connections since distance matrix changed.
Returns
-------
OrderedDict<EmulationNodes, tuple<Interfaces>>>
"""
active_connections = singletons.network_manager.connection_store.get_active_interfaces_per_connection()
# first step -> return all connections
if singletons.simulation_manager.current_step == 0:
return active_connections
# return only the diff
else:
# NOTE: we only have a look at the new connections, not changed ones etc.
dd = DictDiffer(active_connections,
self.active_interfaces_per_connection_before_distance_matrix_changed)
return OrderedDict([(k, active_connections[k]) for k in dd.added()])
def print_report(docs):
if not docs or len(docs) < 2: return 0
name = ''
for doc in docs:
if 'name' in doc:
name = doc['name'].encode('utf-8')
break
output = [
'<h2>Differences for declaration %s, %s</h2>' %
(docs[0]['decl_id'], name)
]
past, current = get_pastcurrent(docs)
differ = DictDiffer(current, past)
output += report_changed(current, past, differ.changed())
output += report_added(current, differ.added())
output += report_removed(past, differ.removed())
output.append('<hr />')
print "\n".join(output)
return 1
from testscenarios.scenarios import generate_scenarios
import unittest
from io import BytesIO
import dictdifferold
from dictdiffer import DictDiffer
from testscenarios import TestWithScenarios
scenario_dictdiffer = ('old', {'differ': dictdifferold})
scenario_dictdiffernew = ('new', {'differ': DictDiffer()})
class DictDifferTests(TestWithScenarios):
scenarios = [scenario_dictdiffer, scenario_dictdiffernew]
def test_addition(self):
""" Test diff with an additional key """
first = {}
second = {'a': 'b'}
diffed = next(self.differ.diff(first, second))
assert ('add', '', [('a', 'b')]) == diffed
def test_deletion(self):
""" Test diff with a removed key """
first = {'a': 'b'}
second = {}
diffed = next(self.differ.diff(first, second))
assert ('remove', '', [('a', 'b')]) == diffed
def test_change(self):
def setUp(self):
a = {'a': 1, 'b': 1, 'c': 0}
b = {'a': 1, 'b': 2, 'd': 0}
self.d = DictDiffer(b, a)
def checkDelta(self, letter):
if letter == 'A':
delta = self.deltaA
# Refer to the opposite MHL to access and perform searches on it
oppositeMHL = self.B
listLetter = 'A'
listLabel = '1st'
listLabelOpposite = '2nd'
listColor = LOG_COLOR_MHL_A
listColorOpposite = LOG_COLOR_MHL_B
elif letter == 'B':
delta = self.deltaB
oppositeMHL = self.A
listLetter = 'B'
listLabel = '2nd'
listLabelOpposite = '1st'
listColor = LOG_COLOR_MHL_B
listColorOpposite = LOG_COLOR_MHL_A
else:
raise Exception(
"INTERNAL: Couldn't check deltas, none were specified. Specify one"
)
return
# Quickly clean Nonexistent objects out if they exist
deltaClean = [h for h in delta if not isinstance(h, HashNonexistent)]
deltaClean.sort()
for hash in deltaClean:
# Debug
# print(color('DEBUG >>>', 'yellow'), hash.identifier, color(hash.filename, 'green'))
# print('rh', hash.recordedHashes)
foundHashPossible = None
beenCounted = False # If this hash has been counted yet
# Look for a match by other hash
# E.g., if XXHASH and MD5 present, search by MD5
for otherHashType, otherHashValue in hash.recordedHashes.items():
if otherHashType == hash.identifierType:
pass # Skip the hash type we are already using
hashPossible = oppositeMHL.findByOtherHash(
otherHashType, otherHashValue)
if isinstance(hashPossible, HashNonexistent):
# No result found, move on
foundHashPossible = False
pass
else:
# Found it
# And because we found it by another hash...
# Let's update the IDENTIFIER. Risky?
hash.identifier = otherHashValue
hash.identifierType = otherHashType
hashPossible.identifier = otherHashValue
hashPossible.identifierType = otherHashType
foundHashPossible = True
break
if foundHashPossible is False:
# Searched but no matches by other hash.
# Look for a match by filename
hashPossible = oppositeMHL.findHashByAttribute(
'filename', hash.filename)
if isinstance(hashPossible, HashNonexistent):
# Definitely missing. No other matches by name or hash.
foundHashPossible = False
else:
foundHashPossible = True
if foundHashPossible is True:
# Compare the hash and the possible hash.
diff = DictDiffer(hash.__dict__, hashPossible.__dict__)
dAdded = diff.added()
dRemoved = diff.removed()
dUnchanged = diff.unchanged()
dChanged = diff.changed()
# First print a filename so everything fits underneath it.
logDetail(' ' +
color(hash.filename, None, attrs=LOG_COLOR_BOLD))
# Then begin testing.
if hash.identifierType == hashPossible.identifierType:
# Hash type is the same
if hash.identifier == hashPossible.identifier:
# And so are the hashes
# But check if it's a duplicate first
if hash.isDuplicate is True:
logDetail(
' This file is a duplicate. Another file exists in this MHL with the same hash.'
)
if not beenCounted:
self.COUNT['DUPLICATE'] += 1
beenCounted = True
logDetail(
' Hash ({}):'.format(listLabel),
colored(
hash.identifier +
' ({})'.format(hash.identifierType),
listColor))
else:
if not beenCounted:
self.COUNT['PERFECT'] += 1
beenCounted = True
logDetail(' Hash: identical.')
else:
# But the hashes are different. File has changed?
if not beenCounted:
self.COUNT['HASH_CHANGED'] += 1
beenCounted = True
logDetail(
color(
' Hash: These hashes are different from each other. It is likely the files were different between the time the MHLs were generated.',
LOG_COLOR_WARNING))
else:
# Hash type is not the same. Unlikely to be comparable.
if not beenCounted:
self.COUNT['HASH_TYPE_DIFFERENT'] += 1
beenCounted = True
logDetail(
color(
" Hash: These hashes are of different types. It's not possible to compare them.",
LOG_COLOR_INFORMATION))
if hash.isDuplicate is False:
logDetail(
' Hash ({}):'.format(listLabel),
color(
'{} ({})'.format(hash.identifier,
hash.identifierType), listColor))
logDetail(
' Hash ({}):'.format(listLabelOpposite),
color(
'{} ({})'.format(hashPossible.identifier,
hashPossible.identifierType),
listColorOpposite))
if {'filename', 'directory', 'size'}.issubset(dUnchanged):
# If neither of these variables have changed, then we have a perfect match.
# EVEN THOUGH we used a slightly different preferred hash.
if not beenCounted:
self.COUNT['PERFECT'] += 1
beenCounted = True
continue
else:
if 'filename' in dChanged:
if not beenCounted:
self.COUNT['MINOR'] += 1
beenCounted = True
logDetail(' Filename: different (1st):',
color(hash.filename, LOG_COLOR_MHL_A))
logDetail(
' (2nd):',
color(hashPossible.filename, LOG_COLOR_MHL_B))
else:
# If the filename is the same, it has already been declared closer to the top.
pass
if 'directory' in dChanged:
if not beenCounted:
self.COUNT['MINOR'] += 1
beenCounted = True
logDetail(' Path: different (1st):',
color(hash.directory, LOG_COLOR_MHL_A))
logDetail(
' (2nd):',
color(hashPossible.directory, LOG_COLOR_MHL_B))
else:
logDetail(' Path: identical:', hash.directory)
if 'size' in dChanged:
# First, check if the Size is simply "Not specified"
# This is not an anomaly if so.
if hash.sizeDefined == False:
# If we have come this far (hash match, name, directory) but size can't be compared
# That is as good as we are gonna get.
self.COUNT['PERFECT'] += 1
beenCounted = True
else:
# It is an anomaly if the size has changed while the hash has not.
# Report it as impossible, but also print it to the user anyway.
if not beenCounted:
self.COUNT['IMPOSSIBLE'] += 1
beenCounted = True
logDetail(' Size: different (1st):',
color(hash.sizeHuman, LOG_COLOR_MHL_A))
logDetail(
' (2nd):',
color(hashPossible.sizeHuman, LOG_COLOR_MHL_B))
else:
logDetail(' ' + 'Size: identical: ' +
hashPossible.sizeHuman)
if 'lastmodificationdate' in dChanged:
if LOG_SHOW_DATES:
if not beenCounted:
self.COUNT['MINOR'] += 1
beenCounted = True
hModDate = showDate(hash.lastmodificationdate)
hPModDate = showDate(
hashPossible.lastmodificationdate)
logDetail(' Modified date: different (1st):',
color(hModDate, LOG_COLOR_MHL_A))
logDetail(' (2nd):',
color(hPModDate, LOG_COLOR_MHL_B))
else:
# Don't count date changes unless user wants it (LOG_SHOW_DATES is true)
pass
# Briefly explain to the user what attributes were added/removed
if LOG_SHOW_DATES == False:
dAddedFiltered = [
i for i in dAdded
if i not in LIST_OF_DATE_ATTRIBUTES
]
dRemovedFiltered = [
i for i in dRemoved
if i not in LIST_OF_DATE_ATTRIBUTES
]
else:
dAddedFiltered = dAdded
dRemovedFiltered = dRemoved
if len(dAddedFiltered) > 0:
dAddedString = ', '.join(
str(i) for i in dAddedFiltered)
logDetail(' These attributes exist in 1st only:',
color(dAddedString, LOG_COLOR_MHL_A))
if len(dRemovedFiltered) > 0:
dRemovedString = ', '.join(
str(i) for i in dRemovedFiltered)
logDetail(' These attributes exist in 2nd only:',
color(dRemovedString, LOG_COLOR_MHL_B))
pass
else:
# Else if foundHashPossible was False.
self.COUNT['MISSING'] += 1
logDetail(
' ' +
color(hash.filename, listColor, attrs=LOG_COLOR_BOLD))
logDetail(' This file only exists in',
color(listLabel + ' MHL', listColor) + '.')
logDetail(' ' + 'Path:', hash.directory)
logDetail(' ' + 'Size:', hash.sizeHuman)
logDetail(' ' + 'Hash:', hash.identifier,
'({})'.format(hash.identifierType))
def checkCommon(self):
for hashA, hashB in self.common:
beenCounted = False
diff = DictDiffer(hashA.__dict__, hashB.__dict__)
dAdded = diff.added()
dRemoved = diff.removed()
dChanged = diff.changed()
dUnchanged = diff.unchanged()
if {'filename', 'directory', 'size'}.issubset(dUnchanged):
# If neither of these variables have changed, then we have a perfect match.
# Report it and move on.
if not beenCounted:
self.COUNT['PERFECT'] += 1
beenCounted = True
continue
if 'filename' in dChanged:
if not beenCounted:
self.COUNT['MINOR'] += 1
beenCounted = True
logDetail(' ' +
color(hashA.filename, 'green', attrs=LOG_COLOR_BOLD))
logDetail(' Filename: different (1st):',
color(hashA.filename, LOG_COLOR_MHL_A))
logDetail(' (2nd):',
color(hashB.filename, LOG_COLOR_MHL_B))
else:
logDetail(' ' +
color(hashA.filename, None, attrs=LOG_COLOR_BOLD))
if 'directory' in dChanged:
if not beenCounted:
self.COUNT['MINOR'] += 1
beenCounted = True
logDetail(' Path: different (1st):',
color(hashA.directory, LOG_COLOR_MHL_A))
logDetail(' (2nd):',
color(hashB.directory, LOG_COLOR_MHL_B))
else:
logDetail(' Path: identical: ' + hashA.directory)
# Straight up print the hash, don't check it.
# At this stage, it's not possible for the hash to be different.
# A check has already been performed for the pair to even be included in this group.
logDetail(' Hash: identical: {} ({})'.format(
hashA.identifier, hashA.identifierType))
if 'size' in dChanged:
# First, check if the Size is simply "Not specified"
if hashA.sizeDefined == False or hashB.sizeDefined == False:
self.COUNT['PERFECT'] += 1
beenCounted = True
# It is an anomaly if the size has changed, but not the hash.
# Report it as impossible, but also print it to the user anyway.
if not beenCounted:
self.COUNT['IMPOSSIBLE'] += 1
beenCounted = True
logDetail(' Size: different (1st):',
color(hashA.sizeHuman, LOG_COLOR_MHL_A))
logDetail(' (2nd):',
color(hashB.sizeHuman, LOG_COLOR_MHL_B))
else:
logDetail(' ' + 'Size: identical: ' + hashA.sizeHuman)
if 'lastmodificationdate' in dChanged:
if LOG_SHOW_DATES:
if not beenCounted:
self.COUNT['MINOR'] += 1
beenCounted = True
logDetail(
' Modified date: different (1st):',
color(hashA.lastmodificationdate, LOG_COLOR_MHL_A))
logDetail(
' (2nd):',
color(hashB.lastmodificationdate, LOG_COLOR_MHL_B))
else:
# Don't count date changes unless user wants it (LOG_SHOW_DATES is true)
pass
# Briefly explain to the user what attributes were added/removed
if LOG_SHOW_DATES == False:
dAddedFiltered = [
i for i in dAdded if i not in LIST_OF_DATE_ATTRIBUTES
]
dRemovedFiltered = [
i for i in dRemoved if i not in LIST_OF_DATE_ATTRIBUTES
]
else:
dAddedFiltered = dAdded
dRemovedFiltered = dRemoved
if len(dAddedFiltered) > 0:
dAddedString = ', '.join(str(i) for i in dAddedFiltered)
logDetail(' These attributes exist in 1st only:',
color(dAddedString, LOG_COLOR_MHL_A))
if len(dRemovedFiltered) > 0:
dRemovedString = ', '.join(str(i) for i in dRemovedFiltered)
logDetail(' These attributes exist in 2nd only:',
color(dRemovedString, LOG_COLOR_MHL_B))
def updateStatus(s):
# Init new database session
s._sess = s.Session()
# Parse OpenVPN status file
status = OpenVPNStatusParser(OPENVPN_STATUS_FILE)
# Parse DHCP daemon leases
status.concat_dhcp(DHCP_LEASES_FILE)
s.log.debug('Connected clients: {0}'.format(
status.connected_clients.keys()))
# Filter clients where doesn not have IP address
clients = s.empty_va_filter(status.connected_clients.values())
# Filter clients where robot does not exist
clients = s.robot_exist_filter(clients)
# Conver client list to dictionary
clients = s.list_to_dict(clients)
s.log.debug('Filtered clients: {0}'.format(clients.keys()))
# Get difference between old and new client lists
diff = DictDiffer(clients, s.clients)
s.log.debug('Clients added: {0}'.format(diff.added))
s.log.debug('Clients removed: {0}'.format(diff.removed))
# Create new firewall rules
s.f.createRules(diff.added)
# Delete old firewall rules
s.f.deleteRules(diff.removed)
# Create new DNS records
s.dns.append(diff.added)
# Delete old DNS records
s.dns.delete(diff.removed)
# Update database records
s.connections.append(diff.added)
s.connections.delete(diff.removed)
s.connections.update(diff.changed)
s.connections.commit()
# Start connected LXC-containers
for name in diff.added:
# Spawn lxc-start for added connections
s.log.info('Start container: {0}'.format(name))
Popen(['lxc-start', '--name', name, '-d']).wait()
# Stop disconnected LXC-containers
for name in diff.removed:
# Spawn lxc-stop for removed connections
Popen(['lxc-stop', '--name', name]).wait()
s.log.info('Stop container: {0}'.format(name))
# Restart DNS daemon (after all veth.* in status UP)
s.dns.restart()
# Replace status by new
s.clients = clients
s._sess.close()
