def __init__(self,
old_obj, new_obj,
previous_path=(), previous_new_path=(),
patch_extractors=[],
key_limits=KeyLimit(),
find_moved_patches=False,
moved_patches_similarity=0.8):
super(DictPatchExtractor, self).__init__(old_obj, new_obj,
previous_path,
previous_new_path,
patch_extractors,
key_limits,
moved_patches_similarity)
dict_diff = DictDiffer(new_obj, old_obj)
for addition_key in dict_diff.added():
if not self._try_patch_extractors_for_ungrouping(addition_key):
self._add_patch('add', addition_key, None, new_obj[addition_key])
for removal_key in dict_diff.removed():
self._add_patch('remove', removal_key, old_obj[removal_key], None)
for change_key in dict_diff.changed():
if not self._try_patch_extractors(change_key, change_key):
self._add_patch('change', change_key,
old_obj[change_key], new_obj[change_key])
if find_moved_patches:
self.patches = self._find_moved_parts()
def __init__(self,
old_obj,
new_obj,
previous_path=(),
previous_new_path=(),
patch_extractors=[],
key_limits=KeyLimit(),
find_moved_patches=False,
moved_patches_similarity=0.8):
super(DictPatchExtractor,
self).__init__(old_obj, new_obj, previous_path,
previous_new_path, patch_extractors, key_limits,
moved_patches_similarity)
dict_diff = DictDiffer(new_obj, old_obj)
for addition_key in dict_diff.added():
if not self._try_patch_extractors_for_ungrouping(addition_key):
self._add_patch('add', addition_key, None,
new_obj[addition_key])
for removal_key in dict_diff.removed():
self._add_patch('remove', removal_key, old_obj[removal_key], None)
for change_key in dict_diff.changed():
if not self._try_patch_extractors(change_key, change_key):
self._add_patch('change', change_key, old_obj[change_key],
new_obj[change_key])
if find_moved_patches:
self.patches = self._find_moved_parts()
def test_haschanges_no(self):
"""Dict are the same
"""
a = {'a': 1, 'b': 1, 'c': 0}
b = {'a': 1, 'b': 1, 'c': 0}
tdf = DictDiffer(b, a)
self.assertEqual(tdf.has_changes(), False)
def test_nb_changes_full(self):
"""Number of changes
"""
a = {'a': 1, 'b': 1}
b = {'d': 1, 'f': 1}
tdf = DictDiffer(b, a)
self.assertEqual(tdf.nb_changes(), 4)
def test_nb_changes(self):
"""Number of changes
"""
a = {'a': 1, 'b': 1, 'c': 0}
b = {'a': 1, 'b': 2}
tdf = DictDiffer(b, a)
self.assertEqual(tdf.nb_changes(), 2)
def test_changes_same(self):
"""Dict are same
"""
a = {'a': 1, 'b': 1, 'c': 0}
b = {'a': 1, 'b': 1, 'c': 0}
tdf = DictDiffer(b, a)
self.assertEqual(tdf.changes(), {'added': 0,
'removed': 0,
'changed': 0})
def test_fulldiff(self):
"""
"""
a = {'a': 1, 'b': 1, 'c': 4}
b = {'d': 1, 'f': 1, 'c': 5}
tdf = DictDiffer(b, a)
diff = tdf.fulldiff()
result = {'added': [{'d': 1}, {'f': 1}],
'changed': [{'key': 'c', 'old': 4, 'new': 5}],
'removed': [{'a': 1}, {'b': 1}]}
self.assertEqual(diff, result)
def compare_sets(file_list):
comparison_list = comparison_tuples(file_list)
for tuple in comparison_list:
print tuple
past_xml, current_xml = open_xml(tuple)
past_dict = json.loads(xml2json(past_xml, 0, 1))
current_dict = json.loads(xml2json(current_xml, 0, 1))
differ = DictDiffer(current_dict['Ableton']['LiveSet']['Tracks'], past_dict['Ableton']['LiveSet']['Tracks'])
print "ADDED"
print json.dumps(differ.added(), indent=4)
print "REMOVED"
print json.dumps(differ.removed(), indent=4)
print "CHANGED"
print json.dumps(differ.changed(), indent=4)
def login(self, config):
server = config['FTP_Server']['Server']
username = config['FTP_Server']['Username']
password = config['FTP_Server']['Password']
self.serverpath = config['FTP_Server']['ServerFolder']
self.download_path = config['DEFAULT']['DownloadPath']
try:
self.host = ftputil.FTPHost(server, username, password)
print("Conection Started")
self.serverHashDict = hashgen.openHashDict(self.downloadFile(config['FTP_Server']['HashPath'], self.updaterDir))
self.ui.statusLabel2.setText("")
self.fileDiffer = DictDiffer(self.serverHashDict, self.localHashDict)
self.host.chdir(self.serverpath)
except ftputil.ftp_error.PermanentError:
print("Error!")
print("Login authentication failed.")
self.ui.statusLabel.setText("Error!")
self.ui.statusLabel2.setText("Login authentication failed.")
return 0
except ftputil.ftp_error.FTPOSError as error:
print("Error!")
print("No Internet connection.")
self.ui.statusLabel.setText("Error!")
self.ui.statusLabel2.setText("No Internet connection.")
# raise(error)
return 0
self.pBar1Value.set(0)
self.pBar2Value.set(0)
def setUp(self):
d1 = {
'ahhhh': ['111', '222', ['jajja', 1], 3],
'dict': {
'hhhh': 11,
'a': 2,
'dict': {
'a': 1
}
}
}
d2 = {
'ahhhh': ['111', '222', ['jajja', 1, 2]],
'dict': {
'h': 2,
'a': [2, 3],
'dict': {
'a': 2
}
}
}
self.d1 = {'shit': d1, 'tu': d2, 'uuu': d1}
self.d2 = {'shat': d1, 'tu': d1}
self.differ_DictDiffer = DictDiffer().diff
self.differ_dictdiffer = dictdiffer.diff
class TestDictDiffer(TestCase):
def setUp(self):
a = {'a': 1, 'b': 1, 'c': 0}
b = {'a': 1, 'b': 2, 'd': 0}
self.d = DictDiffer(b, a)
def test_added(self):
self.assertEqual(self.d.added(), set(['d']))
def test_removed(self):
self.assertEqual(self.d.removed(), set(['c']))
def test_changed(self):
self.assertEqual(self.d.changed(), set(['b']))
def test_unchanged(self):
self.assertEqual(self.d.unchanged(), set(['a']))
def get_new_connections_with_interfaces_since_last_distance_matrix_change(self):
""" Get only new connections since distance matrix changed.
Returns
-------
OrderedDict<EmulationNodes, tuple<Interfaces>>>
"""
active_connections = singletons.network_manager.connection_store.get_active_interfaces_per_connection()
# first step -> return all connections
if singletons.simulation_manager.current_step == 0:
return active_connections
# return only the diff
else:
# NOTE: we only have a look at the new connections, not changed ones etc.
dd = DictDiffer(active_connections,
self.active_interfaces_per_connection_before_distance_matrix_changed)
return OrderedDict([(k, active_connections[k]) for k in dd.added()])
def onSaveButtonClicked(self):
# Set up some variables.
rowcount = self.ui.evTableWidget.rowCount()
evdict = {}
winEnv = windowsev.Win32Environment('system')
# Grab all of the values from the tablewidget and save them.
for i in range(0, rowcount):
evKey = self.ui.evTableWidget.item(i, 0).text()
evValue = self.ui.evTableWidget.item(i, 1).text()
# Save in the dictionary.
evdict[evKey] = evValue
self.newenvs = evdict
differ = DictDiffer(self.newenvs, self.oldenvs)
changed = differ.changed()
for k in changed:
print('Key: ' + k + ' -> Value: ' + self.newenvs[k])
def print_report (docs):
if not docs or len(docs) < 2: return 0
name = ''
for doc in docs:
if 'name' in doc:
name = doc['name'].encode('utf-8')
break
output = ['<h2>Differences for declaration %s, %s</h2>' % (docs[0]['decl_id'], name)]
past, current = get_pastcurrent(docs)
differ = DictDiffer(current, past)
output += report_changed(current, past, differ.changed())
output += report_added(current, differ.added())
output += report_removed(past, differ.removed())
output.append('<hr />')
print "\n".join(output)
return 1
def print_report(docs):
if not docs or len(docs) < 2: return 0
name = ''
for doc in docs:
if 'name' in doc:
name = doc['name'].encode('utf-8')
break
output = [
'<h2>Differences for declaration %s, %s</h2>' %
(docs[0]['decl_id'], name)
]
past, current = get_pastcurrent(docs)
differ = DictDiffer(current, past)
output += report_changed(current, past, differ.changed())
output += report_added(current, differ.added())
output += report_removed(past, differ.removed())
output.append('<hr />')
print "\n".join(output)
return 1
from testscenarios.scenarios import generate_scenarios
import unittest
from io import BytesIO
import dictdifferold
from dictdiffer import DictDiffer
from testscenarios import TestWithScenarios
scenario_dictdiffer = ('old', {'differ': dictdifferold})
scenario_dictdiffernew = ('new', {'differ': DictDiffer()})
class DictDifferTests(TestWithScenarios):
scenarios = [scenario_dictdiffer, scenario_dictdiffernew]
def test_addition(self):
""" Test diff with an additional key """
first = {}
second = {'a': 'b'}
diffed = next(self.differ.diff(first, second))
assert ('add', '', [('a', 'b')]) == diffed
def test_deletion(self):
""" Test diff with a removed key """
first = {'a': 'b'}
second = {}
diffed = next(self.differ.diff(first, second))
assert ('remove', '', [('a', 'b')]) == diffed
def test_change(self):
def setUp(self):
a = {'a': 1, 'b': 1, 'c': 0}
b = {'a': 1, 'b': 2, 'd': 0}
self.d = DictDiffer(b, a)
def checkDelta(self, letter):
if letter == 'A':
delta = self.deltaA
# Refer to the opposite MHL to access and perform searches on it
oppositeMHL = self.B
listLetter = 'A'
listLabel = '1st'
listLabelOpposite = '2nd'
listColor = LOG_COLOR_MHL_A
listColorOpposite = LOG_COLOR_MHL_B
elif letter == 'B':
delta = self.deltaB
oppositeMHL = self.A
listLetter = 'B'
listLabel = '2nd'
listLabelOpposite = '1st'
listColor = LOG_COLOR_MHL_B
listColorOpposite = LOG_COLOR_MHL_A
else:
raise Exception(
"INTERNAL: Couldn't check deltas, none were specified. Specify one"
)
return
# Quickly clean Nonexistent objects out if they exist
deltaClean = [h for h in delta if not isinstance(h, HashNonexistent)]
deltaClean.sort()
for hash in deltaClean:
# Debug
# print(color('DEBUG >>>', 'yellow'), hash.identifier, color(hash.filename, 'green'))
# print('rh', hash.recordedHashes)
foundHashPossible = None
beenCounted = False # If this hash has been counted yet
# Look for a match by other hash
# E.g., if XXHASH and MD5 present, search by MD5
for otherHashType, otherHashValue in hash.recordedHashes.items():
if otherHashType == hash.identifierType:
pass # Skip the hash type we are already using
hashPossible = oppositeMHL.findByOtherHash(
otherHashType, otherHashValue)
if isinstance(hashPossible, HashNonexistent):
# No result found, move on
foundHashPossible = False
pass
else:
# Found it
# And because we found it by another hash...
# Let's update the IDENTIFIER. Risky?
hash.identifier = otherHashValue
hash.identifierType = otherHashType
hashPossible.identifier = otherHashValue
hashPossible.identifierType = otherHashType
foundHashPossible = True
break
if foundHashPossible is False:
# Searched but no matches by other hash.
# Look for a match by filename
hashPossible = oppositeMHL.findHashByAttribute(
'filename', hash.filename)
if isinstance(hashPossible, HashNonexistent):
# Definitely missing. No other matches by name or hash.
foundHashPossible = False
else:
foundHashPossible = True
if foundHashPossible is True:
# Compare the hash and the possible hash.
diff = DictDiffer(hash.__dict__, hashPossible.__dict__)
dAdded = diff.added()
dRemoved = diff.removed()
dUnchanged = diff.unchanged()
dChanged = diff.changed()
# First print a filename so everything fits underneath it.
logDetail(' ' +
color(hash.filename, None, attrs=LOG_COLOR_BOLD))
# Then begin testing.
if hash.identifierType == hashPossible.identifierType:
# Hash type is the same
if hash.identifier == hashPossible.identifier:
# And so are the hashes
# But check if it's a duplicate first
if hash.isDuplicate is True:
logDetail(
' This file is a duplicate. Another file exists in this MHL with the same hash.'
)
if not beenCounted:
self.COUNT['DUPLICATE'] += 1
beenCounted = True
logDetail(
' Hash ({}):'.format(listLabel),
colored(
hash.identifier +
' ({})'.format(hash.identifierType),
listColor))
else:
if not beenCounted:
self.COUNT['PERFECT'] += 1
beenCounted = True
logDetail(' Hash: identical.')
else:
# But the hashes are different. File has changed?
if not beenCounted:
self.COUNT['HASH_CHANGED'] += 1
beenCounted = True
logDetail(
color(
' Hash: These hashes are different from each other. It is likely the files were different between the time the MHLs were generated.',
LOG_COLOR_WARNING))
else:
# Hash type is not the same. Unlikely to be comparable.
if not beenCounted:
self.COUNT['HASH_TYPE_DIFFERENT'] += 1
beenCounted = True
logDetail(
color(
" Hash: These hashes are of different types. It's not possible to compare them.",
LOG_COLOR_INFORMATION))
if hash.isDuplicate is False:
logDetail(
' Hash ({}):'.format(listLabel),
color(
'{} ({})'.format(hash.identifier,
hash.identifierType), listColor))
logDetail(
' Hash ({}):'.format(listLabelOpposite),
color(
'{} ({})'.format(hashPossible.identifier,
hashPossible.identifierType),
listColorOpposite))
if {'filename', 'directory', 'size'}.issubset(dUnchanged):
# If neither of these variables have changed, then we have a perfect match.
# EVEN THOUGH we used a slightly different preferred hash.
if not beenCounted:
self.COUNT['PERFECT'] += 1
beenCounted = True
continue
else:
if 'filename' in dChanged:
if not beenCounted:
self.COUNT['MINOR'] += 1
beenCounted = True
logDetail(' Filename: different (1st):',
color(hash.filename, LOG_COLOR_MHL_A))
logDetail(
' (2nd):',
color(hashPossible.filename, LOG_COLOR_MHL_B))
else:
# If the filename is the same, it has already been declared closer to the top.
pass
if 'directory' in dChanged:
if not beenCounted:
self.COUNT['MINOR'] += 1
beenCounted = True
logDetail(' Path: different (1st):',
color(hash.directory, LOG_COLOR_MHL_A))
logDetail(
' (2nd):',
color(hashPossible.directory, LOG_COLOR_MHL_B))
else:
logDetail(' Path: identical:', hash.directory)
if 'size' in dChanged:
# First, check if the Size is simply "Not specified"
# This is not an anomaly if so.
if hash.sizeDefined == False:
# If we have come this far (hash match, name, directory) but size can't be compared
# That is as good as we are gonna get.
self.COUNT['PERFECT'] += 1
beenCounted = True
else:
# It is an anomaly if the size has changed while the hash has not.
# Report it as impossible, but also print it to the user anyway.
if not beenCounted:
self.COUNT['IMPOSSIBLE'] += 1
beenCounted = True
logDetail(' Size: different (1st):',
color(hash.sizeHuman, LOG_COLOR_MHL_A))
logDetail(
' (2nd):',
color(hashPossible.sizeHuman, LOG_COLOR_MHL_B))
else:
logDetail(' ' + 'Size: identical: ' +
hashPossible.sizeHuman)
if 'lastmodificationdate' in dChanged:
if LOG_SHOW_DATES:
if not beenCounted:
self.COUNT['MINOR'] += 1
beenCounted = True
hModDate = showDate(hash.lastmodificationdate)
hPModDate = showDate(
hashPossible.lastmodificationdate)
logDetail(' Modified date: different (1st):',
color(hModDate, LOG_COLOR_MHL_A))
logDetail(' (2nd):',
color(hPModDate, LOG_COLOR_MHL_B))
else:
# Don't count date changes unless user wants it (LOG_SHOW_DATES is true)
pass
# Briefly explain to the user what attributes were added/removed
if LOG_SHOW_DATES == False:
dAddedFiltered = [
i for i in dAdded
if i not in LIST_OF_DATE_ATTRIBUTES
]
dRemovedFiltered = [
i for i in dRemoved
if i not in LIST_OF_DATE_ATTRIBUTES
]
else:
dAddedFiltered = dAdded
dRemovedFiltered = dRemoved
if len(dAddedFiltered) > 0:
dAddedString = ', '.join(
str(i) for i in dAddedFiltered)
logDetail(' These attributes exist in 1st only:',
color(dAddedString, LOG_COLOR_MHL_A))
if len(dRemovedFiltered) > 0:
dRemovedString = ', '.join(
str(i) for i in dRemovedFiltered)
logDetail(' These attributes exist in 2nd only:',
color(dRemovedString, LOG_COLOR_MHL_B))
pass
else:
# Else if foundHashPossible was False.
self.COUNT['MISSING'] += 1
logDetail(
' ' +
color(hash.filename, listColor, attrs=LOG_COLOR_BOLD))
logDetail(' This file only exists in',
color(listLabel + ' MHL', listColor) + '.')
logDetail(' ' + 'Path:', hash.directory)
logDetail(' ' + 'Size:', hash.sizeHuman)
logDetail(' ' + 'Hash:', hash.identifier,
'({})'.format(hash.identifierType))
def checkCommon(self):
for hashA, hashB in self.common:
beenCounted = False
diff = DictDiffer(hashA.__dict__, hashB.__dict__)
dAdded = diff.added()
dRemoved = diff.removed()
dChanged = diff.changed()
dUnchanged = diff.unchanged()
if {'filename', 'directory', 'size'}.issubset(dUnchanged):
# If neither of these variables have changed, then we have a perfect match.
# Report it and move on.
if not beenCounted:
self.COUNT['PERFECT'] += 1
beenCounted = True
continue
if 'filename' in dChanged:
if not beenCounted:
self.COUNT['MINOR'] += 1
beenCounted = True
logDetail(' ' +
color(hashA.filename, 'green', attrs=LOG_COLOR_BOLD))
logDetail(' Filename: different (1st):',
color(hashA.filename, LOG_COLOR_MHL_A))
logDetail(' (2nd):',
color(hashB.filename, LOG_COLOR_MHL_B))
else:
logDetail(' ' +
color(hashA.filename, None, attrs=LOG_COLOR_BOLD))
if 'directory' in dChanged:
if not beenCounted:
self.COUNT['MINOR'] += 1
beenCounted = True
logDetail(' Path: different (1st):',
color(hashA.directory, LOG_COLOR_MHL_A))
logDetail(' (2nd):',
color(hashB.directory, LOG_COLOR_MHL_B))
else:
logDetail(' Path: identical: ' + hashA.directory)
# Straight up print the hash, don't check it.
# At this stage, it's not possible for the hash to be different.
# A check has already been performed for the pair to even be included in this group.
logDetail(' Hash: identical: {} ({})'.format(
hashA.identifier, hashA.identifierType))
if 'size' in dChanged:
# First, check if the Size is simply "Not specified"
if hashA.sizeDefined == False or hashB.sizeDefined == False:
self.COUNT['PERFECT'] += 1
beenCounted = True
# It is an anomaly if the size has changed, but not the hash.
# Report it as impossible, but also print it to the user anyway.
if not beenCounted:
self.COUNT['IMPOSSIBLE'] += 1
beenCounted = True
logDetail(' Size: different (1st):',
color(hashA.sizeHuman, LOG_COLOR_MHL_A))
logDetail(' (2nd):',
color(hashB.sizeHuman, LOG_COLOR_MHL_B))
else:
logDetail(' ' + 'Size: identical: ' + hashA.sizeHuman)
if 'lastmodificationdate' in dChanged:
if LOG_SHOW_DATES:
if not beenCounted:
self.COUNT['MINOR'] += 1
beenCounted = True
logDetail(
' Modified date: different (1st):',
color(hashA.lastmodificationdate, LOG_COLOR_MHL_A))
logDetail(
' (2nd):',
color(hashB.lastmodificationdate, LOG_COLOR_MHL_B))
else:
# Don't count date changes unless user wants it (LOG_SHOW_DATES is true)
pass
# Briefly explain to the user what attributes were added/removed
if LOG_SHOW_DATES == False:
dAddedFiltered = [
i for i in dAdded if i not in LIST_OF_DATE_ATTRIBUTES
]
dRemovedFiltered = [
i for i in dRemoved if i not in LIST_OF_DATE_ATTRIBUTES
]
else:
dAddedFiltered = dAdded
dRemovedFiltered = dRemoved
if len(dAddedFiltered) > 0:
dAddedString = ', '.join(str(i) for i in dAddedFiltered)
logDetail(' These attributes exist in 1st only:',
color(dAddedString, LOG_COLOR_MHL_A))
if len(dRemovedFiltered) > 0:
dRemovedString = ', '.join(str(i) for i in dRemovedFiltered)
logDetail(' These attributes exist in 2nd only:',
color(dRemovedString, LOG_COLOR_MHL_B))
class updater():
def __init__(self, ui):
super(updater, self).__init__()
self.pBar1Value = Reference(0)
self.pBar2Value = Reference(0)
self.totalDownSize = 0
self.ui = ui
self.updaterDir = os.getcwd()
try:
self.localHashDict = hashgen.openHashDict(os.path.join(self.updaterDir, 'updater.dat'))
except IOError as exc:
# print(exc)
self.localHashDict = {}
def login(self, config):
server = config['FTP_Server']['Server']
username = config['FTP_Server']['Username']
password = config['FTP_Server']['Password']
self.serverpath = config['FTP_Server']['ServerFolder']
self.download_path = config['DEFAULT']['DownloadPath']
try:
self.host = ftputil.FTPHost(server, username, password)
print("Conection Started")
self.serverHashDict = hashgen.openHashDict(self.downloadFile(config['FTP_Server']['HashPath'], self.updaterDir))
self.ui.statusLabel2.setText("")
self.fileDiffer = DictDiffer(self.serverHashDict, self.localHashDict)
self.host.chdir(self.serverpath)
except ftputil.ftp_error.PermanentError:
print("Error!")
print("Login authentication failed.")
self.ui.statusLabel.setText("Error!")
self.ui.statusLabel2.setText("Login authentication failed.")
return 0
except ftputil.ftp_error.FTPOSError as error:
print("Error!")
print("No Internet connection.")
self.ui.statusLabel.setText("Error!")
self.ui.statusLabel2.setText("No Internet connection.")
# raise(error)
return 0
self.pBar1Value.set(0)
self.pBar2Value.set(0)
def close(self):
try:
self.host.close()
print("\nConection Closed")
return 0
except:
pass
def checkDownloadPath(self, download_path):
if not os.path.exists(download_path):
try:
os.makedirs(download_path)
except Exception as exc:
print(exc)
def downloadEntirePath(self):
self.totalSize = self.getFullSize()
try:
for (ftp_curpath, ftp_dirs, ftp_files) in self.host.walk(self.host.curdir):
self.downloadFolderFiles(ftp_curpath)
hashgen.saveHashDict(self.localHashDict, self.updaterDir)
except Exception as exc:
raise(exc)
def checkDiffer(self):
if len(self.fileDiffer.added()) != 0 or len(self.fileDiffer.changed()) != 0 or len(self.fileDiffer.removed()) != 0 or not os.path.exists(self.download_path):
return True
else:
return False
def calculateDiffer(self):
if os.path.exists(os.path.join(self.updaterDir, 'updater.dat')) and os.path.exists(self.download_path):
self.downloadDiffer(self.fileDiffer.added())
self.downloadDiffer(self.fileDiffer.changed())
self.removeDiffer(self.fileDiffer.removed())
hashgen.saveHashDict(self.localHashDict, self.updaterDir)
else:
try:
try:
os.remove(os.path.join(self.updaterDir, 'updater.dat'))
except:
pass
self.downloadEntirePath()
except Exception as exc:
raise(exc)
os.remove(os.path.join(self.updaterDir, 'updater.dat'))
def downloadDiffer(self, differ_set):
for files in differ_set:
self.downloadFile(files, self.download_path)
self.localHashDict[files] = self.serverHashDict[files]
def removeDiffer(self, differ_set):
for files in differ_set:
file_path = os.path.join(self.download_path, files)
os.remove(file_path)
del self.localHashDict[files]
print("File", os.path.split(file_path)[1], "removed.")
def getFullSize(self):
totalSize = 0
for (ftp_curpath, ftp_dirs, ftp_files) in self.host.walk(self.host.curdir):
for files in ftp_files:
totalSize += self.host.path.getsize(self.host.path.join(ftp_curpath, files))
return(totalSize)
def downloadFile(self, ftp_filepath, download_path):
self.ui.statusText = "Downloading"
self.dl_file_size = 0
self.pBar1Value.set(0)
self.file_size_bytes = self.host.path.getsize(ftp_filepath)
file_size = self.__size(self.file_size_bytes)
ftp_path, filename=self.host.path.split(ftp_filepath)
print("\nDownloading " + filename + " - Size: " + file_size)
self.ui.statusLabel2.setText(filename + " - Size:" + file_size)
try:
self.checkDownloadPath(os.path.join(download_path, ftp_path))
self.host.download(ftp_filepath, os.path.join(download_path, ftp_filepath), mode='b', callback=self.__downloadBuffer)
print(end="\n\n")
except OSError as exc:
raise(exc)
return os.path.join(download_path, filename)
def downloadFolderFiles(self, ftp_path=''):
self.ui.startPBars(self.pBar1Value, self.pBar2Value)
for filename in self.host.listdir(ftp_path):
ftp_filepath = self.host.path.join(ftp_path, filename)
# print(ftp_filepath)
if self.host.path.isfile(ftp_filepath):
self.ui.updateStatus = True
self.ui.statusText = "Checking files"
self.ui.statusLabel2.setText("Checking " + filename)
print("Checking", filename)
try:
localFileHash = hashgen.getFileHash(os.path.join(self.download_path, ftp_filepath))
except Exception as exc:
localFileHash = None
if self.serverHashDict[ftp_filepath] != localFileHash:
self.downloadFile(ftp_filepath, self.download_path)
else:
self.totalDownSize += self.host.path.getsize(ftp_filepath)
self.pBar2Value.set(int(self.totalDownSize * 100 / self.totalSize))
self.localHashDict[ftp_filepath] = self.serverHashDict[ftp_filepath]
self.ui.stopPBars()
def __downloadBuffer(self, buffer):
buffer_len = len(buffer)
self.dl_file_size += buffer_len
self.totalDownSize += buffer_len
p = int(self.dl_file_size * 100 / self.file_size_bytes)
try:
r = int(self.totalDownSize * 100 / self.totalSize)
except:
r = int(self.totalDownSize * 100 / self.file_size_bytes)
print("Status: ", p, "%", end="\r")
self.pBar1Value.set(p)
self.pBar2Value.set(r)
def __size(self, size):
if size < 1024:
file_size = str("{0:.2f}".format(size)) + " Bytes"
elif size/1024 < 1024:
file_size = str("{0:.2f}".format(size/1024)) + " KB"
elif size/1024**2 < 1024:
file_size = str("{0:.2f}".format(size/1024**2)) + " MB"
elif size/1024**3 < 1024:
file_size = str("{0:.2f}".format(size/1024**3)) + " GB"
return file_size
def updateStatus(s):
# Init new database session
s._sess = s.Session()
# Parse OpenVPN status file
status = OpenVPNStatusParser(OPENVPN_STATUS_FILE)
# Parse DHCP daemon leases
status.concat_dhcp(DHCP_LEASES_FILE)
s.log.debug('Connected clients: {0}'.format(
status.connected_clients.keys()))
# Filter clients where doesn not have IP address
clients = s.empty_va_filter(status.connected_clients.values())
# Filter clients where robot does not exist
clients = s.robot_exist_filter(clients)
# Conver client list to dictionary
clients = s.list_to_dict(clients)
s.log.debug('Filtered clients: {0}'.format(clients.keys()))
# Get difference between old and new client lists
diff = DictDiffer(clients, s.clients)
s.log.debug('Clients added: {0}'.format(diff.added))
s.log.debug('Clients removed: {0}'.format(diff.removed))
# Create new firewall rules
s.f.createRules(diff.added)
# Delete old firewall rules
s.f.deleteRules(diff.removed)
# Create new DNS records
s.dns.append(diff.added)
# Delete old DNS records
s.dns.delete(diff.removed)
# Update database records
s.connections.append(diff.added)
s.connections.delete(diff.removed)
s.connections.update(diff.changed)
s.connections.commit()
# Start connected LXC-containers
for name in diff.added:
# Spawn lxc-start for added connections
s.log.info('Start container: {0}'.format(name))
Popen(['lxc-start', '--name', name, '-d']).wait()
# Stop disconnected LXC-containers
for name in diff.removed:
# Spawn lxc-stop for removed connections
Popen(['lxc-stop', '--name', name]).wait()
s.log.info('Stop container: {0}'.format(name))
# Restart DNS daemon (after all veth.* in status UP)
s.dns.restart()
# Replace status by new
s.clients = clients
s._sess.close()
def setUp(self):
a = {'a': 1, 'b': 1, 'c': 0}
b = {'a': 1, 'b': 2, 'd': 0}
self.d = DictDiffer(b, a)
def test_haschanges_empty(self):
"""Dict are empty
"""
tdf = DictDiffer({}, {})
self.assertFalse(tdf.has_changes())
class TestDictDiffer(TestCase):
def setUp(self):
a = {'a': 1, 'b': 1, 'c': 0}
b = {'a': 1, 'b': 2, 'd': 0}
self.d = DictDiffer(b, a)
def test_added(self):
self.assertEqual(self.d.added(), set(['d']))
def test_removed(self):
self.assertEqual(self.d.removed(), set(['c']))
def test_changed(self):
self.assertEqual(self.d.changed(), set(['b']))
def test_unchanged(self):
self.assertEqual(self.d.unchanged(), set(['a']))
def test_changes(self):
self.assertEqual(self.d.changes(), {'added': 1,
'removed': 1,
'changed': 1})
def test_changes_same(self):
"""Dict are same
"""
a = {'a': 1, 'b': 1, 'c': 0}
b = {'a': 1, 'b': 1, 'c': 0}
tdf = DictDiffer(b, a)
self.assertEqual(tdf.changes(), {'added': 0,
'removed': 0,
'changed': 0})
def test_haschanges(self):
self.assertEqual(self.d.has_changes(), True)
def test_haschanges_no(self):
"""Dict are the same
"""
a = {'a': 1, 'b': 1, 'c': 0}
b = {'a': 1, 'b': 1, 'c': 0}
tdf = DictDiffer(b, a)
self.assertEqual(tdf.has_changes(), False)
def test_haschanges_empty(self):
"""Dict are empty
"""
tdf = DictDiffer({}, {})
self.assertFalse(tdf.has_changes())
def test_nb_changes(self):
"""Number of changes
"""
a = {'a': 1, 'b': 1, 'c': 0}
b = {'a': 1, 'b': 2}
tdf = DictDiffer(b, a)
self.assertEqual(tdf.nb_changes(), 2)
def test_nb_changes_same(self):
"""Number of changes
"""
a = {'a': 1, 'b': 1}
b = {'a': 1, 'b': 1}
tdf = DictDiffer(b, a)
self.assertEqual(tdf.nb_changes(), 0)
def test_nb_changes_full(self):
"""Number of changes
"""
a = {'a': 1, 'b': 1}
b = {'d': 1, 'f': 1}
tdf = DictDiffer(b, a)
self.assertEqual(tdf.nb_changes(), 4)
def test_fulldiff(self):
"""
"""
a = {'a': 1, 'b': 1, 'c': 4}
b = {'d': 1, 'f': 1, 'c': 5}
tdf = DictDiffer(b, a)
diff = tdf.fulldiff()
result = {'added': [{'d': 1}, {'f': 1}],
'changed': [{'key': 'c', 'old': 4, 'new': 5}],
'removed': [{'a': 1}, {'b': 1}]}
self.assertEqual(diff, result)
