def testDel(tarDir, isFolder): #{0
testTarDir = []
testContent = "testFile"
testTarDir.append(func.buildPath(tarDir, "testDir", delim))
tStr = "testDir" + delim + "subDir"
testTarDir.append(func.buildPath(tarDir, tStr, delim))
tStr = "testDir" + delim + "testFile.txt"
testTarFile = func.buildPath(tarDir, tStr, delim)
for tDir in testTarDir: #{1
if not os.path.isdir(tDir): #{2
os.mkdir(tDir)
#}2
#}1
tFile = open(testTarFile, 'a', encoding='utf8')
tFile.write(testContent)
tFile.close()
if isFolder: #{1
delDirFile(tarDir, "subDir", isFolder, '')
delete(isFolder)
if os.path.isfile(testTarFile): #{2
print("Test successful")
shutil.rmtree(testTarDir[0], onerror=func.remove_readonly)
return True
#}2
else: #{2
print("Test failed")
shutil.rmtree(testTarDir[0], onerror=func.remove_readonly)
return False
#}2
#}1
else: #{1
delDirFile(tarDir, "testFile.txt", isFolder, '')
delete(isFolder)
if os.path.isdir(testTarDir[1]): #{2
print("Test successful")
shutil.rmtree(testTarDir[0], onerror=func.remove_readonly)
return True
#}2
else: #{2
print("Test failed")
shutil.rmtree(testTarDir[0], onerror=func.remove_readonly)
return False
def checkNcopy(srcDir, dstDir, srcName, dstName, delim): #{0
try: #{1
cpCounter = 0
while os.path.isfile(func.buildPath(dstDir, dstName, delim)): #{2
#modift fileName assign to new string var
cpCounter = cpCounter + 1
dstName = dstName.split('.')
dstName.insert(len(dstName) - 1, str(cpCounter))
dstName[0:len(dstName) -
1] = ['-'.join(dstName[0:len(dstName) - 1])]
dstName = '.'.join(dstName)
#}2
src = func.buildPath(srcDir, srcName, delim)
dst = func.buildPath(dstDir, dstName, delim)
shutil.copy2(src, dst)
return True
#}1
except OSError: #{1
print('OSError during file copying')
return False
def testDup(tarDir, mode, cmpMode): #{0
testTarDir = []
testTarFile = []
testContent = "testFile"
testTarDir.append(func.buildPath(tarDir, "testDir", delim))
tStr = "testDir" + delim + "subDir1"
testTarDir.append(func.buildPath(tarDir, tStr, delim))
tStr = "testDir" + delim + "subDir2"
testTarDir.append(func.buildPath(tarDir, tStr, delim))
tStr = "testDir" + delim + "subDir1" + delim + "testFile1.txt"
testTarFile.append(func.buildPath(tarDir, tStr, delim))
tStr = "testDir" + delim + "subDir2" + delim + "testFile2.txt"
testTarFile.append(func.buildPath(tarDir, tStr, delim))
tStr = "testDir" + delim + "subDir2" + delim + "testFile3.txt"
testTarFile.append(func.buildPath(tarDir, tStr, delim))
for tDir in testTarDir: #{1
if not os.path.isdir(tDir): #{2
os.mkdir(tDir)
#}2
#}1
for i in range(len(testTarFile)): #{1
if not os.path.isfile(testTarFile[i]): #{2
tFile = open(testTarFile[i], 'a', encoding='utf8')
tFile.write(testContent)
tFile.close()
#}2
#}1
if mode == 1: #{1
globalCompare(testTarDir[0], cmpMode, '')
dup()
result1 = os.listdir(testTarDir[1])
result2 = os.listdir(testTarDir[2])
if (len(result1) == 0 and len(result2) == 1) or \
(len(result1) == 1 and len(result2) == 0):#{2
print("Test successful")
shutil.rmtree(testTarDir[0], onerror=func.remove_readonly)
return True
#}2
else: #{2
print("Test failed")
shutil.rmtree(testTarDir[0], onerror=func.remove_readonly)
return False
#}2
#}1
elif mode == 2: #{1
localCompare(testTarDir[0], cmpMode, '')
dup()
result1 = os.listdir(testTarDir[1])
result2 = os.listdir(testTarDir[2])
if len(result1) == 1 and len(result2) == 1: #{2
print("Test successful")
shutil.rmtree(testTarDir[0], onerror=func.remove_readonly)
return True
#}2
else: #{2
print("Test failed")
shutil.rmtree(testTarDir[0], onerror=func.remove_readonly)
return False
def validate(srcDir, dstDir, exlSrcDir, exlDstDir): #{0
srcList = []
dstList = []
print('Checksum...')
if type(srcDir) is list: #{1
for i in srcDir: #{2
p1 = func.buildPath(i[0], i[2], delim)
p2 = func.buildPath(i[1], i[3], delim)
fileHash1 = func.hasher(p1, False, hash_Mode)
fileHash2 = func.hasher(p2, False, hash_Mode)
if fileHash1 != fileHash2: #{3
print('Checksum: File checksum failed')
return 0
#}3
#}2
print('Checksum: File checksum passed')
return 1
#}1
else: #{1
entity0 = entityc.entity(srcDir, dstDir, delim)
srcDir = entity0.srcDir
dstDir = entity0.dstDir
esList = func.listExl(exlSrcDir, delim)
edList = func.listExl(exlDstDir, delim)
print('Source directories to exclude:')
print(esList)
print('Destination directories to exclude:')
print(edList)
isExlSrc = False
isExlDst = False
for pf, sf, fn in os.walk(srcDir): #{2
for esDir in esList: #{3
if esDir in pf: #{4
isExlSrc = True
#}4
#}3
if isExlSrc == True: #{3
isExlSrc = False
continue
#}3
else: #{3
for f in fn: #{4
tempStr = func.buildPath(pf, f, delim)
srcList.append(tempStr)
#}4
#}3
#}2
for pf, sf, fn in os.walk(dstDir): #{2
for edDir in edList: #{3
if edDir in pf: #{4
isExlDst = True
#}4
#}3
if isExlDst == True: #{3
isExlDst = False
continue
#}3
else: #{3
for f in fn: #{4
tempStr = func.buildPath(pf, f, delim)
dstList.append(tempStr)
#}4
#}3
#}2
if len(srcList) != len(dstList): #{2
print('Checksum: directory checksum failed')
return 0
#}2
else: #{2
for i in range(len(srcList)): #{3
p1 = srcList[i]
p2 = dstList[i]
tStr1 = p1.replace(srcDir, '')
strHash1 = func.hasher(tStr1, True, hash_Mode)
fileHash1 = func.hasher(p1, False, hash_Mode)
tStr2 = p2.replace(dstDir, '')
strHash2 = func.hasher(tStr2, True, hash_Mode)
fileHash2 = func.hasher(p2, False, hash_Mode)
if strHash1 != strHash2 or fileHash1 != fileHash2: #{4
print('Checksum: directory checksum failed')
return 0
#}4
#}3
#}2
print('Checksum: directory checksum passed')
return 1
def compareNcopy(srcDir, dstDir): #{0
entity0 = entityc.entity(srcDir, dstDir, delim)
srcDir = entity0.srcDir
dstDir = entity0.dstDir
if not os.path.isdir(srcDir) or not os.path.isdir(
dstDir) or srcDir == dstDir: #{1
return 0
#}1
else: #{1
sameFileSrc = []
srcFileList = []
srcSizeList = []
srcToCopy = []
dstFileList = []
dstSizeList = []
srcObjList = []
dstObjList = []
global totalSrcFile #int
global totalDstFile #int
global totalSrcCopy #int
global totalDstRemove #int
global totalDstRename #int
for folderName, subFolders, fileNames in os.walk(srcDir): #{2
if folderName == srcDir: #{3
for fileName in fileNames: #{4
if fileName.lower() != thumbs.lower(): #{5
fsize = os.path.getsize(
func.buildPath(folderName, fileName, delim))
srcSizeList.append(fsize)
srcFileList.append([folderName, fileName, fsize])
#}5
#}4
#}3
#}2
for folderName, subFolders, fileNames in os.walk(dstDir): #{2
if folderName == dstDir: #{3
for fileName in fileNames: #{4
if fileName.lower() != thumbs.lower(): #{5
fsize = os.path.getsize(
func.buildPath(folderName, fileName, delim))
dstSizeList.append(fsize)
dstFileList.append([folderName, fileName, fsize])
#}5
#}4
#}3
#}2
#Assign value to global variables
totalSrcFile = totalSrcFile + len(srcFileList)
totalDstFile = totalDstFile + len(dstFileList)
#At this point path and file size are both ready for comparison
srcFileList.sort(
key=itemgetter(2)) #sort fileList against fSize (file size)
#srcFileList.sort(key=lambda x: x[2])
dstFileList.sort(
key=itemgetter(2)) #sort fileList against fSize (file size)
#dstFileList.sort(key=lambda x: x[2])
#at0 = datetime.datetime.now()
#sec0 = at0.second
#msec0 = at0.microsecond
#print(srcFileList)
func.loadFileObj(srcFileList, srcObjList, False, delim, cmp_Mode,
hash_Mode)
func.loadFileObj(dstFileList, dstObjList, False, delim, cmp_Mode,
hash_Mode)
#at1 = datetime.datetime.now()
#sec1 = at1.second
#msec1 = at1.microsecond
#t0 = sec0 + msec0/1000000
#t1 = sec1 + msec1/1000000
#dt = t1 - t0
#profile1.append(dt)
'''
print(srcDir + ' vs ' + dstDir)
for i in srcObjList:
ts = i.fileNames
print(ts)
print('\n')
for i in dstObjList:
ts = i.fileNames
print(ts)
print('\n')
'''
#srcObjList.sort(key=lambda x: x.fileSize)
#dstObjList.sort(key=lambda x: x.fileSize)
#at0 = datetime.datetime.now()
#sec0 = at0.second
#msec0 = at0.microsecond
#To remove dstDir files that does not exist in srcDir
#To also update dstFileDict and dstFileList
#If true then maybe there are some files to remove
if len(dstObjList) != 0: #{2
#If there is nothing in the srcFolder
#then remove all files in corresponding dstfolder, no need to fileCompare
if len(srcObjList) == 0: #{3
for c in range(len(dstObjList)): #{4
#This bit of code is the same as
#remove command when srcFile is compared against dstFile
if dstObjList[c].num == 1: #{5
tempRM0 = func.buildPath(
dstDir, dstObjList[c].fileNames[0],
delim) #dstDir is always as dstFileList[c][0]
removeList.append(tempRM0)
totalDstRemove = totalDstRemove + 1 #Global variable
#}5
else: #{5
nList = dstObjList[c].fileNames
totalDstRemove = totalDstRemove + dstObjList[c].num
for i in nList: #{6
tempRM = func.buildPath(dstDir, i, delim)
removeList.append(tempRM)
#}6
#}5
#}4
#}3
else: #{3
for c in range(len(dstObjList)): #{4
toRemove = True
dstDirFile = func.buildPath(dstDir,
dstObjList[c].fileNames[0],
delim) #complete path
dstFile = dstObjList[c].fileNames[0] #file name only
dstFsize = dstObjList[c].fileSize
dstFnum = dstObjList[c].num
#Assume if file size different then content different
if dstFsize in srcSizeList: #{5
for d in range(len(srcObjList)): #{6
srcFile = srcObjList[d].fileNames[
0] #file name only
srcFsize = srcObjList[d].fileSize
srcDirFile = func.buildPath(
srcDir, srcObjList[d].fileNames[0],
delim) #complete path
srcFnum = srcObjList[d].num
#return actual size which is same on different file system, not size on disk
#Investigate further about filecmp.cmp
if dstFsize == srcFsize: #{7
#filecmp compare primarily compare file size and modification time
#3rd arg 0-filecmp, 1-hashlib
#last arg passed in from GUI for filecmp shallow=True/False
if func.fCompare(dstDirFile, srcDirFile,
cmp_Mode, hash_Mode,
True): #{8
#And its corresponding srcFile is added to srcToCopy list
#By not adding it to the sameFileSrc list
#before the file is re-copied
#dstFile to be kept, but renamed to be same as corresponding srcFile
if dstFnum == 1 and srcFnum == 1: #{9
if dstFile != srcFile: #{10
#This is to check for files with same size, mod-time but different content
#only thing indicates that they are different files is their file names and content
#If file names mismatch then do a file content comparison
#to verify two files are in fact the same, if not then continue
if not func.fCompare(
dstDirFile, srcDirFile,
cmp_Mode, hash_Mode,
False): #{11
continue
#}11
tempRN1 = func.buildPath(
dstDir, srcFile, delim)
renameList.append(
[dstDirFile, tempRN1])
totalDstRename = totalDstRename + 1 #Global variable
#}10
toRemove = False
sameFileSrc.append(
srcObjList[d]
) #file that is the same
continue
#}9
else: #{9
srcNlist = srcObjList[
d].fileNames.copy()
dstNlist = dstObjList[
c].fileNames.copy()
#print(srcNlist)
#print(dstNlist)
toRM = []
for i in srcNlist: #{10
if i in dstNlist: #{11
dstNlist.remove(i)
toRM.append(
i
) #not to remove elements while looping
#}11
#}10
#When there is a total mismatch in file names, do a file content compare
#to verify two files are in fact the same, if not continue
if len(toRM) == 0: #{10
if not func.fCompare(
dstDirFile, srcDirFile,
cmp_Mode, hash_Mode,
False): #{11
continue
#}11
#}10
for i in toRM: #{10
srcNlist.remove(i)
#}10
srcLen = len(srcNlist)
dstLen = len(dstNlist)
numSum = srcLen + dstLen
if numSum != 0: #{10
numDiff = srcLen - dstLen
if numDiff > 0: #{11
for i in range(
dstLen, srcLen,
1): #{12
checkNcopyList.append([
srcDir, dstDir,
srcNlist[i],
srcNlist[i]
])
totalSrcCopy = totalSrcCopy + 1 #Global variable
#}12
for i in range(0, dstLen,
1): #{12
if srcNlist[i] != dstNlist[
i]: #{13
tempRN1 = func.buildPath(
dstDir,
srcNlist[i], delim)
renameList.append([
dstDirFile, tempRN1
])
totalDstRename = totalDstRename + 1 #Global variable
#}13
#}12
#}11
elif numDiff < 0: #{11
for i in range(
srcLen, dstLen,
1): #{12
tempRM0 = func.buildPath(
dstDir, dstNlist[i],
delim)
removeList.append(tempRM0)
totalDstRemove = totalDstRemove + 1 #Global variable
#}12
for i in range(0, srcLen,
1): #{12
if srcNlist[i] != dstNlist[
i]: #{13
tempRN1 = func.buildPath(
dstDir,
srcNlist[i], delim)
renameList.append([
dstDirFile, tempRN1
])
totalDstRename = totalDstRename + 1 #Global variable
#}13
#}12
#}11
else: #{11
for i in range(0, srcLen,
1): #{12
if srcNlist[i] != dstNlist[
i]: #{13
tempRN1 = func.buildPath(
dstDir,
srcNlist[i], delim)
renameList.append([
dstDirFile, tempRN1
])
totalDstRename = totalDstRename + 1 #Global variable
#}13
#}12
#}11
#}10
toRemove = False
sameFileSrc.append(
srcObjList[d]
) #file that is the same
continue
#}9
#}8
#}7
#if nothing matches then do nothing
#src file will be in srcToCopy list by not being in sameFileSrc list
#dst file will add to delete list by toRemove not being set to False
#this is the only place to break out of looping srcFileList
#coz file size is sorted from small to large
elif srcFsize > dstFsize: #{7
break
#}7
#}6
#}5
#Condition for dst file removal is if the file does not exist in src
#Or dst file exist in src, but it's more than 2 years old
if toRemove == True: #{5
#provision for multi-copy files
if dstFnum == 1: #{6
removeList.append(dstDirFile)
totalDstRemove = totalDstRemove + 1 #Global variable
#}6
else: #{6
nList = dstObjList[c].fileNames
totalDstRemove = totalDstRemove + dstFnum
for i in nList: #{7
tempRM = func.buildPath(dstDir, i, delim)
removeList.append(tempRM)
#}7
#}6
#}5
#}4
#}3
#}2
#To copy srcDir files that does not exist in dstDir to dstDir
if len(srcObjList) == 0: #{2
return 1
#}2
else: #{2
#This if clause is for when dst dir match src dir, but it's empty
#Only empty of files, but there may be sub-folders
if len(dstObjList) == 0: #{3
for d in range(len(srcObjList)): #{4
if srcObjList[d].num == 1: #{5
checkNcopyList.append([
srcDir, dstDir, srcObjList[d].fileNames[0],
srcObjList[d].fileNames[0]
])
totalSrcCopy = totalSrcCopy + 1 #Global variable
#}5
else: #{5
nList = srcObjList[d].fileNames
totalSrcCopy = totalSrcCopy + srcObjList[d].num
for i in nList: #{6
checkNcopyList.append([srcDir, dstDir, i, i])
#}6
#}5
#}4
#}3
else: #{3
#If src file is not in the sameFileSrc list, then it's to be copied to dst
#srcToCopy = list(set(sameFileSrc)^set(srcFileList)) #remove same elements between 2 1D lists
for i in srcObjList: #{4
if not i in sameFileSrc: #{5
srcToCopy.append(i)
#}5
#}4
for d in range(len(srcToCopy)): #{4
if srcToCopy[d].num == 1: #{5
checkNcopyList.append([
srcDir, dstDir, srcToCopy[d].fileNames[0],
srcToCopy[d].fileNames[0]
])
totalSrcCopy = totalSrcCopy + 1 #Global variable
#}5
else: #{5
nList = srcToCopy[d].fileNames
totalSrcCopy = totalSrcCopy + srcToCopy[d].num
for i in nList: #{6
checkNcopyList.append([srcDir, dstDir, i, i])
#}6
#}5
#}4
#}3
#}2
#at1 = datetime.datetime.now()
#sec1 = at1.second
#msec1 = at1.microsecond
#t0 = sec0 + msec0/1000000
#t1 = sec1 + msec1/1000000
#dt = t1 - t0
#profile2.append(dt)
return 1
def testSync(srcDir, dstDir): #{0
testSrcDir = []
testDstDir = []
testSrcFile = []
testDstFile = []
testContent = ["testFile0", "testFile1", "testFile2", "testFile3"]
resFile = []
testSrcDir.append(func.buildPath(srcDir, "testDir", delim))
testDstDir.append(func.buildPath(dstDir, "testDir", delim))
tStr = "testDir" + delim + "subDir1"
testSrcDir.append(func.buildPath(srcDir, tStr, delim))
tStr = "testDir" + delim + "subDir2"
testSrcDir.append(func.buildPath(srcDir, tStr, delim))
testDstDir.append(func.buildPath(dstDir, tStr, delim))
tStr = "testDir" + delim + "subDir3"
testDstDir.append(func.buildPath(dstDir, tStr, delim))
tStr1 = "testDir" + delim + "testFile0.txt"
testSrcFile.append(func.buildPath(srcDir, tStr1, delim))
tStr2 = "testDir" + delim + "subDir1" + delim + "testFile1.txt"
testSrcFile.append(func.buildPath(srcDir, tStr2, delim))
tStr3 = "testDir" + delim + "subDir2" + delim + "testFile2.txt"
testSrcFile.append(func.buildPath(srcDir, tStr3, delim))
testDstFile.append(func.buildPath(dstDir, tStr3, delim))
tStr = "testDir" + delim + "subDir3" + delim + "testFile3.txt"
testDstFile.append(func.buildPath(dstDir, tStr, delim))
resFile.append(func.buildPath(dstDir, tStr1, delim))
resFile.append(func.buildPath(dstDir, tStr2, delim))
resFile.append(func.buildPath(dstDir, tStr3, delim))
for sDir in testSrcDir: #{1
if not os.path.isdir(sDir): #{2
os.mkdir(sDir)
#}2
#}1
for dDir in testDstDir: #{1
if not os.path.isdir(dDir): #{2
os.mkdir(dDir)
#}2
#}1
for i in range(len(testSrcFile)): #{1
if not os.path.isfile(testSrcFile[i]): #{2
srcFile = open(testSrcFile[i], 'a', encoding='utf8')
srcFile.write(testContent[i])
srcFile.close()
#}2
#}1
for j in range(len(testDstFile)): #{1
if not os.path.isfile(testDstFile[j]): #{2
dstFile = open(testDstFile[j], 'a', encoding='utf8')
dstFile.write(testContent[j + 2])
dstFile.close()
#}2
#}1
print("Testing core function..")
backupCopy(testSrcDir[0], testDstDir[0], '', '')
filing(3, 1)
if os.path.isfile(resFile[0]) and os.path.isfile(resFile[1]) \
and os.path.isfile(resFile[2]):#{1
print("Test successful")
shutil.rmtree(testSrcDir[0], onerror=func.remove_readonly)
shutil.rmtree(testDstDir[0], onerror=func.remove_readonly)
return True
#}1
else: #{1
print("Test failed")
#shutil.rmtree(testSrcDir[0], onerror=func.remove_readonly)
#shutil.rmtree(testDstDir[0], onerror=func.remove_readonly)
return False
def sync_size_est(dst_drive): #{0
global dstRM_size
global srcCP_size
global net_sync_size
dstRM_size = 0
srcCP_size = 0
net_sync_size = 0
dst_stat = shutil.disk_usage(dst_drive)
dst_total = dst_stat.total #int in bytes
dst_free = dst_stat.free #int in bytes
dst_total_gb = dst_total * gb_conv #int in GB
dst_free_gb = dst_free * gb_conv #int in GB
print('Destination drive capacity:')
print('Total: ' + str(dst_total_gb) + 'GB' + ' Free: ' + str(dst_free_gb) +
'GB')
if len(removeList) != 0: #{1
#print(removeList)
for f in removeList: #{2
rmFsize = os.path.getsize(f)
dstRM_size = dstRM_size + rmFsize
#}2
#}1
if len(rmtreeList) != 0: #{1
for f in rmtreeList: #{2
for d, sd, fs in os.walk(f): #{3
for fn in fs: #{4
fp = func.buildPath(d, fn, delim)
fp_size = os.path.getsize(fp)
dstRM_size = dstRM_size + fp_size
#}4
#}3
#}2
#}1
if len(checkNcopyList) != 0: #{1
#print(checkNcopyList)
for f in checkNcopyList: #{2
fp = func.buildPath(f[0], f[2], delim)
cpFsize = os.path.getsize(fp)
srcCP_size = srcCP_size + cpFsize
#}2
#}1
if len(copytreeList) != 0: #{1
for f in copytreeList: #{2
fp = f[0]
for d, sd, fs in os.walk(fp): #{3
for fn in fs: #{4
fp = func.buildPath(d, fn, delim)
fp_size = os.path.getsize(fp)
srcCP_size = srcCP_size + fp_size
#}4
#}3
#}2
#}1
net_sync_size = srcCP_size - dstRM_size
srcCP_size_gb = srcCP_size * gb_conv
dstRM_size_gb = dstRM_size * gb_conv
net_sync_size_gb = net_sync_size * gb_conv
if srcCP_size_gb < 1: #{1
srcCP_size_mb = srcCP_size * mb_conv
print('Files to copy: ' + str(srcCP_size_mb) + 'MB')
#}1
else: #{1
print('Files to copy: ' + str(srcCP_size_gb) + 'GB')
#}1
if dstRM_size_gb < 1: #{1
dstRM_size_mb = dstRM_size * mb_conv
print('Files to remove: ' + str(dstRM_size_mb) + 'MB')
#}1
else: #{1
print('Files to remove: ' + str(dstRM_size_gb) + 'GB')
#}1
if net_sync_size_gb < 1: #{1
net_sync_size_mb = net_sync_size * mb_conv
print('Net increase to destination: ' + str(net_sync_size_mb) + 'MB')
#}1
else: #{1
print('Net increase to destination: ' + str(net_sync_size_gb) + 'GB')
def filing(iMode, iCS): #{0
global totalSrcFile
global totalDstFile
global totalSrcCopy
global totalDstRemove
global totalDstRename
#totalSrcFile = 0
#totalDstFile = 0
totalSrcCopy = 0
totalDstRemove = 0
totalDstRename = 0
if iMode == 3: #{1
print('Mirror Syncing...')
#}1
elif iMode == 1: #{1
print('Additive Syncing...')
#}1
elif iMode == 2: #{1
print('File name Syncing...')
#}1
if iMode == 3: #{1
for tf in rmtreeList: #{2
#print('Folder remove ' + tf)
tfTotal = sum([len(cc) for aa, bb, cc in os.walk(tf)])
try: #{3
shutil.rmtree(tf)
totalDstRemove = totalDstRemove + tfTotal
#}3
except: #{3
tfLeft = sum([len(cc) for aa, bb, cc in os.walk(tf)])
tfRM = tfTotal - tfLeft
totalDstRemove = totalDstRemove + tfRM
print('dir removal error on: ' + tf)
continue
#}3
#}2
for rmf in removeList: #{2
try: #{3
os.remove(rmf)
totalDstRemove = totalDstRemove + 1
#}3
except OSError: #{3
print('OSerror during file removal: ' + rmf)
continue
#}3
#print('Remove: ' + rmf)
#}2
#}1
if iMode == 3 or iMode == 2: #{1
for rnf in renameList: #{2
try: #{3
os.rename(rnf[0], rnf[1])
totalDstRename = totalDstRename + 1
#}3
except OSError: #{3
print('OSerror during file rename: ' + rnf)
continue
#}3
#print('Rename: ' + rnf[0] + ' -> ' + rnf[1])
#}2
#}1
cpTreeErr = []
cpFileErr = []
if iMode == 3 or iMode == 1: #{1
#copytree function create dst directory path automatically
for cpt in copytreeList: #{2
try: #{3
shutil.copytree(cpt[0], cpt[1])
temp = sum([len(cc) for aa, bb, cc in os.walk(cpt[1])])
totalSrcCopy = totalSrcCopy + temp
#}3
except Error as err: #{3
print('Error during directory copy')
print('src name: ' + str(err.args[0]))
print('dst name: ' + str(err.args[1]))
print('exception: ' + str(err.args[2]))
cpTreeErr.append(cpt)
continue
#}3
#}2
for cptErr in cpTreeErr: #{2
copytreeList.remove(cptErr)
#}2
for cpf in checkNcopyList: #{2
tempDst = func.buildPath(cpf[1], cpf[3], delim)
#print('Folder copy ' + cpf[0] + ' to ' + cpf[1])
if fOps.checkNcopy(cpf[0], cpf[1], cpf[2], cpf[3],
delim) == True: #{3
totalSrcCopy = totalSrcCopy + 1
#}3
else: #{3
cpFileErr.append(cpf)
#}3
#}2
for cpfErr in cpFileErr: #{2
checkNcopyList.remove(cpfErr)
#}2
#}1
#Checksum for transfered files
if (iMode == 1 or iMode == 3) and iCS == 1: #{1
if len(checkNcopyList) != 0: #{2
v1 = validate(checkNcopyList, '', '', '')
#}2
else: #{2
v1 = 3
#}2
if len(copytreeList) != 0: #{2
for i in copytreeList: #{3
v2 = validate(i[0], i[1], '', '')
if v2 == 0: #{4
break
#}4
#}3
#}2
else: #{2
v2 = 3
#}2
v3 = v1 + v2
if v3 == 2 or v3 == 4: #{2
print('Checksum: Transfered file checksum passed')
#}2
elif v3 == 6: #{2
print('Checksum: Nothing to validate')
#}2
else: #{2
print('Checksum: Transfered file checksum failed')
#}2
#}1
rmtreeList.clear()
copytreeList.clear()
renameList.clear()
removeList.clear()
checkNcopyList.clear()
print('Total source file count: ' + str(totalSrcFile))
print('Total destination file count: ' + str(totalDstFile))
print('Total source file copied: ' + str(totalSrcCopy))
print('Total destination file removed: ' + str(totalDstRemove))
print('Total destination file renamed: ' + str(totalDstRename))
if iMode == 3 and totalSrcFile == totalDstFile - totalDstRemove + totalSrcCopy: #{1
print("No issue encountered with mirroring")
#}1
'''
else:#{1
print("Issue encountered, please investigate")
#}1
'''
print('Sync finished\n')
def localCompare(targetDir, cmpMode, exlDir): #{0
entity0 = entityc.entity(targetDir, targetDir, delim)
targetDir = entity0.targetDir
eList = func.listExl(exlDir, delim)
print('Directories to exclude:')
print(eList)
isExl = False
cwd = os.getcwd()
if not os.path.isdir(targetDir): #{1
print('Error: Target directory not found\n')
return ''
#}1
else: #{1
#Variable init and definition
dupFileList.clear()
dupList = []
opCounter = 0
cpCounter = 0
totalFiles = 0
cMode = ''
if cmpMode == 0: #{2
cMode = 'Filecmp.cmp (shallow=False compare file content)'
#}2
else: #{2
cMode = 'Hashlib (Checksum:SHA256)'
#}2
print('Finding duplicated files within each sub-directory...')
for folderName, subFolders, fileNames in os.walk(targetDir): #{2
fileList = []
objList = []
c = 0
for eDir in eList: #{3
if eDir in folderName: #{4
isExl = True
#}4
#}3
if isExl == True: #{3
isExl = False
#}3
else: #{3
for fileName in fileNames: #{4
if fileName.lower() != thumbs.lower(): #{5
totalFiles = totalFiles + 1
fSize = os.path.getsize(
func.buildPath(folderName, fileName, delim))
#fileList.append([fileName,fsize])
fileList.append([folderName, fileName, fSize])
#}5
#}4
fileList.sort(key=lambda x: x[2])
#loadFileObj populates objList list with list of duplicated files
func.loadFileObj(fileList, objList, True, delim, cmpMode,
hash_Mode)
##for i in objList:#{4
## ts = i.fileNames
## print(ts)
##}4
##print('\n')
if len(objList) != 0: #{4
for i in objList: #{5
timeList = []
Bvalue = 0
Bindex = 0
counter = 0
for j in i.fileNames: #{6
path = func.buildPath(i.fileDir[0], j, delim)
ct = os.path.getctime(path)
at = os.path.getatime(path)
timeList.append(ct)
timeList.append(at)
#}6
while counter < len(timeList): #{6
if timeList[counter] > Bvalue: #{7
Bvalue = timeList[counter]
Bindex = counter
#}7
counter = counter + 1
#}6
cd = Bindex // 2
#cr = Bindex%2
#print(str(cd))
toKeep = 'To keep: ' + i.fileNames[cd] + ', to remove:'
#print(toKeep)
for j in range(len(i.fileNames)): #{6
if j != cd: #{7
toKeep = toKeep + ' ' + i.fileNames[j]
toRM = func.buildPath(i.fileDir[0],
i.fileNames[j], delim)
dupFileList.append(toRM)
#}7
#}6
toKeep = toKeep + ', at ' + i.fileDir[0]
dupList.append(toKeep)
cpCounter = cpCounter + 1
#}5
#}4
#}3
#}2
filecmp.clear_cache()
#encoding arg is for writing nonEng char
with open(cwd + delim + 'findDupFiles_report.txt',
'a',
encoding='utf8') as outputFile: #{2
outputFile.write('\n')
outputFile.write('Target directory: ' + targetDir + '\n')
outputFile.write(
'Find duplicated files within each sub-directory\n')
if len(dupList) > 0: #{3
outputFile.write('Files with duplicated content:\n')
for i in dupList: #{4
outputFile.write(i + '\n')
#}4
#}3
outputFile.write('Method of file comparison: ' + cMode + '\n')
outputFile.write('Total file count: ' + str(totalFiles) + '\n')
outputFile.write('Comparison Ops: ' + str(opCounter) + '\n')
outputFile.write('Duplicated files: ' + str(cpCounter) + '\n')
#outputFile.close() #using "with open" auto close file even with exception
#}2
print('Method of file comparison: ' + cMode)
print('Total file count: ' + str(totalFiles))
print('Duplicated files: ' + str(cpCounter))
dup_size_est()
print('Please find analysis report in ' + cwd + delim +
'findDupFiles_report.txt')
print('Analysis completed\n')
ret = str(totalFiles) + '-0-' + str(cpCounter)
return ret
def globalCompare(targetDir, cmpMode, exlDir): #{0
entity0 = entityc.entity(targetDir, targetDir, delim)
targetDir = entity0.targetDir
eList = func.listExl(exlDir, delim)
print('Directories to exclude:')
print(eList)
isExl = False
cwd = os.getcwd()
if not os.path.isdir(targetDir): #{1
print('Error: Target directory not found\n')
return ''
#}1
else: #{1
fileList = []
objList = []
nameList = []
dupFileList.clear()
opCounter = 0
cpCounter = 0
dnCounter = 0
totalFiles = 0
for folderName, subFolders, fileNames in os.walk(targetDir): #{2
for eDir in eList: #{3
if eDir in folderName: #{4
isExl = True
#}4
#}3
if isExl == True: #{3
isExl = False
#}3
else: #{3
for fileName in fileNames: #{4
if fileName.lower() != thumbs.lower(): #{5
totalFiles = totalFiles + 1
tPath = func.buildPath(folderName, fileName, delim)
fSize = os.path.getsize(tPath)
mTime = os.path.getmtime(tPath)
fileList.append([folderName, fileName, fSize])
nameList.append([folderName, fileName, mTime])
#}5
#}4
#}3
#}2
#Now a list of all files and dir is ready
if len(fileList) == 0: #{2
print('There are no files in target directory\n')
return ''
#}2
else: #{2
cMode = ''
if cmpMode == 0: #{3
cMode = 'Filecmp.cmp (shallow=False compare file content)'
#}3
else: #{3
cMode = 'Hashlib (Checksum:SHA256)'
#}3
c = 0
dupName = []
dupList = []
nameList.sort(key=lambda x: x[1]) #sort in order of file name
while c < (len(nameList) - 1): #{3
nc = c + 1
dir1 = nameList[c][0]
dir2 = nameList[nc][0]
file1 = nameList[c][1]
file2 = nameList[nc][
1] #file to be removed from nameList if same
mtime1 = nameList[c][2]
mtime2 = nameList[nc][2]
if file1 == file2: #{4
#print(file1 + ' => ' + file2)
df1 = func.buildPath(dir1, file1, delim)
df2 = func.buildPath(dir2, file2, delim)
if mtime1 > mtime2: #{5
df3 = 'latter'
#}5
elif mtime2 > mtime1: #{5
df3 = 'former'
#}5
else: #{5
df3 = 'either'
#}5
dupName.append(df1 + ' == ' + df2 + ', Remove: ' + df3)
dnCounter = dnCounter + 1
del nameList[nc]
continue #effect is c is not incremented
#}4
c = c + 1
#}3
fileList.sort(
key=lambda x: x[2]) #sort fileList against fSize (file size)
#loadFileObj populates objList list with list of duplicated files
func.loadFileObj(fileList, objList, True, delim, cmpMode,
hash_Mode)
print('Finding duplicated files across all sub-directories...')
if len(objList) != 0: #{3
for i in objList: #{4
timeList = []
Bvalue = 0
Bindex = 0
counter = 0
for j in range(len(i.fileNames)): #{5
if len(i.fileDir) == 1: #{6
k = 0
#}6
else: #{6
k = j
#}6
path = func.buildPath(i.fileDir[k], i.fileNames[j],
delim)
ct = os.path.getctime(path)
at = os.path.getatime(path)
timeList.append(ct)
timeList.append(at)
#}5
while counter < len(timeList): #{5
if timeList[counter] > Bvalue: #{6
Bvalue = timeList[counter]
Bindex = counter
#}6
counter = counter + 1
#}5
cd = Bindex // 2
#cr = Bindex%2
#print(str(cd))
if len(i.fileDir) == 1: #{5
k = 0
#}5
else: #{5
k = cd
#}5
keepath = func.buildPath(i.fileDir[k], i.fileNames[cd],
delim)
toKeep = 'To keep:\n' + keepath + '\nIn:\n' + keepath + '\n'
#print(toKeep)
for j in range(len(i.fileNames)): #{5
if j != cd: #{6
if len(i.fileDir) == 1: #{7
k = 0
#}7
else: #{7
k = j
#}7
toRM = func.buildPath(i.fileDir[k], i.fileNames[j],
delim)
toKeep = toKeep + toRM + '\n'
dupFileList.append(toRM)
#}6
#}5
dupList.append(toKeep)
cpCounter = cpCounter + 1
#}4
#}3
filecmp.clear_cache()
#encoding arg is for writing nonEng char
with open(cwd + delim + 'findDupFiles_report.txt',
'a',
encoding='utf8') as outputFile: #{3
outputFile.write('\n')
outputFile.write('Target directory: ' + targetDir + '\n')
outputFile.write(
'Find duplicated files across all sub-directories\n')
if len(dupList) > 0: #{4
outputFile.write('Files with duplicated content:\n')
for i in dupList: #{5
outputFile.write(i + '\n')
#}5
#}4
if len(dupName) > 0: #{4
outputFile.write('Files with duplicated names:\n')
for i in dupName: #{5
outputFile.write(i + '\n')
#}5
#}4
outputFile.write('Method of file comparison: ' + cMode + '\n')
outputFile.write('Total file count: ' + str(totalFiles) + '\n')
outputFile.write('Comparison Ops: ' + str(opCounter) + '\n')
outputFile.write('Duplicated files: ' + str(cpCounter) + '\n')
outputFile.write('Duplicated fileNames: ' + str(dnCounter) +
'\n')
outputFile.write(
'Only files with duplicated content are included in deletion\n'
)
outputFile.write(
'Duplicated file naming require user investigation\n')
#outputFile.close() #using "with open" auto close file even with exception
#}3
print('Method of file comparison: ' + cMode)
print('Total file count: ' + str(totalFiles))
print('Duplicated files: ' + str(cpCounter))
print('Duplicated fileNames: ' + str(dnCounter))
dup_size_est()
print(
'Only files with duplicated content are included in deletion')
print('Duplicated file naming require user investigation')
print('Please find analysis report in ' + cwd + delim +
'findDupFiles_report.txt')
print('Analysis completed\n')
ret = str(totalFiles) + '-' + str(dnCounter) + '-' + str(cpCounter)
return ret
def delDirFile(wd, targetName, isFolder, exlDir): #{0
entity0 = entityc.entity(wd, wd, delim)
wd = entity0.targetDir
eList = func.listExl(exlDir, delim)
print('Directories to exclude:')
print(eList)
isExl = False
if not os.path.isdir(wd): #{1
print('Error: target directory not found\n')
return 0
#}1
else: #{1
workFolder = ''
cpCounter = 0
delDFList.clear()
if isFolder == False: #{2
checkTar = targetName.split('.')
ext = checkTar[len(checkTar) - 1]
if ext == 'sys' or ext == 'dll' or len(checkTar) == 1: #{3
print(
'Error: file extension must be specified and cannot be .sys or .dll\n'
)
return 0
#}3
#}2
else: #{2
checkTar = targetName.split('.')
ext = checkTar[0]
if ext == '': #{3
print('Error: folder name cannot start with "."\n')
return 0
#}3
#}2
print('Finding specified file/folder for deletion...')
for folderName, subFolders, fileNames in os.walk(wd): #{2
for eDir in eList: #{3
if eDir in folderName: #{4
isExl = True
#}4
#}3
if isExl == True: #{3
isExl = False
#}3
else: #{3
tWorkFolder = folderName.split(delim)
workFolder = tWorkFolder[len(tWorkFolder) - 1]
#print('Folder inside ' + workFolder)
if isFolder == False: #{4
for fileName in fileNames: #{5
if fileName.lower() == targetName.lower(): #{6
target = func.buildPath(folderName, fileName,
delim)
delDFList.append(target)
cpCounter = cpCounter + 1
#}6
#}5
#}4
else: #{4
if folderName != wd: #{5
if workFolder.lower() == targetName.lower(): #{6
#Add to delete list
delDFList.append(folderName)
cpCounter = cpCounter + 1
#}6
#}5
#}4
#}3
#}2
print('Number of instances found: ' + str(cpCounter))
for i in delDFList: #{2
print(str(i))
#}2
print('Analysis completed\n')
return cpCounter
