def testDel(tarDir, isFolder): #{0 testTarDir = [] testContent = "testFile" testTarDir.append(func.buildPath(tarDir, "testDir", delim)) tStr = "testDir" + delim + "subDir" testTarDir.append(func.buildPath(tarDir, tStr, delim)) tStr = "testDir" + delim + "testFile.txt" testTarFile = func.buildPath(tarDir, tStr, delim) for tDir in testTarDir: #{1 if not os.path.isdir(tDir): #{2 os.mkdir(tDir) #}2 #}1 tFile = open(testTarFile, 'a', encoding='utf8') tFile.write(testContent) tFile.close() if isFolder: #{1 delDirFile(tarDir, "subDir", isFolder, '') delete(isFolder) if os.path.isfile(testTarFile): #{2 print("Test successful") shutil.rmtree(testTarDir[0], onerror=func.remove_readonly) return True #}2 else: #{2 print("Test failed") shutil.rmtree(testTarDir[0], onerror=func.remove_readonly) return False #}2 #}1 else: #{1 delDirFile(tarDir, "testFile.txt", isFolder, '') delete(isFolder) if os.path.isdir(testTarDir[1]): #{2 print("Test successful") shutil.rmtree(testTarDir[0], onerror=func.remove_readonly) return True #}2 else: #{2 print("Test failed") shutil.rmtree(testTarDir[0], onerror=func.remove_readonly) return False
def checkNcopy(srcDir, dstDir, srcName, dstName, delim): #{0 try: #{1 cpCounter = 0 while os.path.isfile(func.buildPath(dstDir, dstName, delim)): #{2 #modift fileName assign to new string var cpCounter = cpCounter + 1 dstName = dstName.split('.') dstName.insert(len(dstName) - 1, str(cpCounter)) dstName[0:len(dstName) - 1] = ['-'.join(dstName[0:len(dstName) - 1])] dstName = '.'.join(dstName) #}2 src = func.buildPath(srcDir, srcName, delim) dst = func.buildPath(dstDir, dstName, delim) shutil.copy2(src, dst) return True #}1 except OSError: #{1 print('OSError during file copying') return False
def testDup(tarDir, mode, cmpMode): #{0 testTarDir = [] testTarFile = [] testContent = "testFile" testTarDir.append(func.buildPath(tarDir, "testDir", delim)) tStr = "testDir" + delim + "subDir1" testTarDir.append(func.buildPath(tarDir, tStr, delim)) tStr = "testDir" + delim + "subDir2" testTarDir.append(func.buildPath(tarDir, tStr, delim)) tStr = "testDir" + delim + "subDir1" + delim + "testFile1.txt" testTarFile.append(func.buildPath(tarDir, tStr, delim)) tStr = "testDir" + delim + "subDir2" + delim + "testFile2.txt" testTarFile.append(func.buildPath(tarDir, tStr, delim)) tStr = "testDir" + delim + "subDir2" + delim + "testFile3.txt" testTarFile.append(func.buildPath(tarDir, tStr, delim)) for tDir in testTarDir: #{1 if not os.path.isdir(tDir): #{2 os.mkdir(tDir) #}2 #}1 for i in range(len(testTarFile)): #{1 if not os.path.isfile(testTarFile[i]): #{2 tFile = open(testTarFile[i], 'a', encoding='utf8') tFile.write(testContent) tFile.close() #}2 #}1 if mode == 1: #{1 globalCompare(testTarDir[0], cmpMode, '') dup() result1 = os.listdir(testTarDir[1]) result2 = os.listdir(testTarDir[2]) if (len(result1) == 0 and len(result2) == 1) or \ (len(result1) == 1 and len(result2) == 0):#{2 print("Test successful") shutil.rmtree(testTarDir[0], onerror=func.remove_readonly) return True #}2 else: #{2 print("Test failed") shutil.rmtree(testTarDir[0], onerror=func.remove_readonly) return False #}2 #}1 elif mode == 2: #{1 localCompare(testTarDir[0], cmpMode, '') dup() result1 = os.listdir(testTarDir[1]) result2 = os.listdir(testTarDir[2]) if len(result1) == 1 and len(result2) == 1: #{2 print("Test successful") shutil.rmtree(testTarDir[0], onerror=func.remove_readonly) return True #}2 else: #{2 print("Test failed") shutil.rmtree(testTarDir[0], onerror=func.remove_readonly) return False
def validate(srcDir, dstDir, exlSrcDir, exlDstDir): #{0 srcList = [] dstList = [] print('Checksum...') if type(srcDir) is list: #{1 for i in srcDir: #{2 p1 = func.buildPath(i[0], i[2], delim) p2 = func.buildPath(i[1], i[3], delim) fileHash1 = func.hasher(p1, False, hash_Mode) fileHash2 = func.hasher(p2, False, hash_Mode) if fileHash1 != fileHash2: #{3 print('Checksum: File checksum failed') return 0 #}3 #}2 print('Checksum: File checksum passed') return 1 #}1 else: #{1 entity0 = entityc.entity(srcDir, dstDir, delim) srcDir = entity0.srcDir dstDir = entity0.dstDir esList = func.listExl(exlSrcDir, delim) edList = func.listExl(exlDstDir, delim) print('Source directories to exclude:') print(esList) print('Destination directories to exclude:') print(edList) isExlSrc = False isExlDst = False for pf, sf, fn in os.walk(srcDir): #{2 for esDir in esList: #{3 if esDir in pf: #{4 isExlSrc = True #}4 #}3 if isExlSrc == True: #{3 isExlSrc = False continue #}3 else: #{3 for f in fn: #{4 tempStr = func.buildPath(pf, f, delim) srcList.append(tempStr) #}4 #}3 #}2 for pf, sf, fn in os.walk(dstDir): #{2 for edDir in edList: #{3 if edDir in pf: #{4 isExlDst = True #}4 #}3 if isExlDst == True: #{3 isExlDst = False continue #}3 else: #{3 for f in fn: #{4 tempStr = func.buildPath(pf, f, delim) dstList.append(tempStr) #}4 #}3 #}2 if len(srcList) != len(dstList): #{2 print('Checksum: directory checksum failed') return 0 #}2 else: #{2 for i in range(len(srcList)): #{3 p1 = srcList[i] p2 = dstList[i] tStr1 = p1.replace(srcDir, '') strHash1 = func.hasher(tStr1, True, hash_Mode) fileHash1 = func.hasher(p1, False, hash_Mode) tStr2 = p2.replace(dstDir, '') strHash2 = func.hasher(tStr2, True, hash_Mode) fileHash2 = func.hasher(p2, False, hash_Mode) if strHash1 != strHash2 or fileHash1 != fileHash2: #{4 print('Checksum: directory checksum failed') return 0 #}4 #}3 #}2 print('Checksum: directory checksum passed') return 1
def compareNcopy(srcDir, dstDir): #{0 entity0 = entityc.entity(srcDir, dstDir, delim) srcDir = entity0.srcDir dstDir = entity0.dstDir if not os.path.isdir(srcDir) or not os.path.isdir( dstDir) or srcDir == dstDir: #{1 return 0 #}1 else: #{1 sameFileSrc = [] srcFileList = [] srcSizeList = [] srcToCopy = [] dstFileList = [] dstSizeList = [] srcObjList = [] dstObjList = [] global totalSrcFile #int global totalDstFile #int global totalSrcCopy #int global totalDstRemove #int global totalDstRename #int for folderName, subFolders, fileNames in os.walk(srcDir): #{2 if folderName == srcDir: #{3 for fileName in fileNames: #{4 if fileName.lower() != thumbs.lower(): #{5 fsize = os.path.getsize( func.buildPath(folderName, fileName, delim)) srcSizeList.append(fsize) srcFileList.append([folderName, fileName, fsize]) #}5 #}4 #}3 #}2 for folderName, subFolders, fileNames in os.walk(dstDir): #{2 if folderName == dstDir: #{3 for fileName in fileNames: #{4 if fileName.lower() != thumbs.lower(): #{5 fsize = os.path.getsize( func.buildPath(folderName, fileName, delim)) dstSizeList.append(fsize) dstFileList.append([folderName, fileName, fsize]) #}5 #}4 #}3 #}2 #Assign value to global variables totalSrcFile = totalSrcFile + len(srcFileList) totalDstFile = totalDstFile + len(dstFileList) #At this point path and file size are both ready for comparison srcFileList.sort( key=itemgetter(2)) #sort fileList against fSize (file size) #srcFileList.sort(key=lambda x: x[2]) dstFileList.sort( key=itemgetter(2)) #sort fileList against fSize (file size) #dstFileList.sort(key=lambda x: x[2]) #at0 = datetime.datetime.now() #sec0 = at0.second #msec0 = at0.microsecond #print(srcFileList) func.loadFileObj(srcFileList, srcObjList, False, delim, cmp_Mode, hash_Mode) func.loadFileObj(dstFileList, dstObjList, False, delim, cmp_Mode, hash_Mode) #at1 = datetime.datetime.now() #sec1 = at1.second #msec1 = at1.microsecond #t0 = sec0 + msec0/1000000 #t1 = sec1 + msec1/1000000 #dt = t1 - t0 #profile1.append(dt) ''' print(srcDir + ' vs ' + dstDir) for i in srcObjList: ts = i.fileNames print(ts) print('\n') for i in dstObjList: ts = i.fileNames print(ts) print('\n') ''' #srcObjList.sort(key=lambda x: x.fileSize) #dstObjList.sort(key=lambda x: x.fileSize) #at0 = datetime.datetime.now() #sec0 = at0.second #msec0 = at0.microsecond #To remove dstDir files that does not exist in srcDir #To also update dstFileDict and dstFileList #If true then maybe there are some files to remove if len(dstObjList) != 0: #{2 #If there is nothing in the srcFolder #then remove all files in corresponding dstfolder, no need to fileCompare if len(srcObjList) == 0: #{3 for c in range(len(dstObjList)): #{4 #This bit of code is the same as #remove command when srcFile is compared against dstFile if dstObjList[c].num == 1: #{5 tempRM0 = func.buildPath( dstDir, dstObjList[c].fileNames[0], delim) #dstDir is always as dstFileList[c][0] removeList.append(tempRM0) totalDstRemove = totalDstRemove + 1 #Global variable #}5 else: #{5 nList = dstObjList[c].fileNames totalDstRemove = totalDstRemove + dstObjList[c].num for i in nList: #{6 tempRM = func.buildPath(dstDir, i, delim) removeList.append(tempRM) #}6 #}5 #}4 #}3 else: #{3 for c in range(len(dstObjList)): #{4 toRemove = True dstDirFile = func.buildPath(dstDir, dstObjList[c].fileNames[0], delim) #complete path dstFile = dstObjList[c].fileNames[0] #file name only dstFsize = dstObjList[c].fileSize dstFnum = dstObjList[c].num #Assume if file size different then content different if dstFsize in srcSizeList: #{5 for d in range(len(srcObjList)): #{6 srcFile = srcObjList[d].fileNames[ 0] #file name only srcFsize = srcObjList[d].fileSize srcDirFile = func.buildPath( srcDir, srcObjList[d].fileNames[0], delim) #complete path srcFnum = srcObjList[d].num #return actual size which is same on different file system, not size on disk #Investigate further about filecmp.cmp if dstFsize == srcFsize: #{7 #filecmp compare primarily compare file size and modification time #3rd arg 0-filecmp, 1-hashlib #last arg passed in from GUI for filecmp shallow=True/False if func.fCompare(dstDirFile, srcDirFile, cmp_Mode, hash_Mode, True): #{8 #And its corresponding srcFile is added to srcToCopy list #By not adding it to the sameFileSrc list #before the file is re-copied #dstFile to be kept, but renamed to be same as corresponding srcFile if dstFnum == 1 and srcFnum == 1: #{9 if dstFile != srcFile: #{10 #This is to check for files with same size, mod-time but different content #only thing indicates that they are different files is their file names and content #If file names mismatch then do a file content comparison #to verify two files are in fact the same, if not then continue if not func.fCompare( dstDirFile, srcDirFile, cmp_Mode, hash_Mode, False): #{11 continue #}11 tempRN1 = func.buildPath( dstDir, srcFile, delim) renameList.append( [dstDirFile, tempRN1]) totalDstRename = totalDstRename + 1 #Global variable #}10 toRemove = False sameFileSrc.append( srcObjList[d] ) #file that is the same continue #}9 else: #{9 srcNlist = srcObjList[ d].fileNames.copy() dstNlist = dstObjList[ c].fileNames.copy() #print(srcNlist) #print(dstNlist) toRM = [] for i in srcNlist: #{10 if i in dstNlist: #{11 dstNlist.remove(i) toRM.append( i ) #not to remove elements while looping #}11 #}10 #When there is a total mismatch in file names, do a file content compare #to verify two files are in fact the same, if not continue if len(toRM) == 0: #{10 if not func.fCompare( dstDirFile, srcDirFile, cmp_Mode, hash_Mode, False): #{11 continue #}11 #}10 for i in toRM: #{10 srcNlist.remove(i) #}10 srcLen = len(srcNlist) dstLen = len(dstNlist) numSum = srcLen + dstLen if numSum != 0: #{10 numDiff = srcLen - dstLen if numDiff > 0: #{11 for i in range( dstLen, srcLen, 1): #{12 checkNcopyList.append([ srcDir, dstDir, srcNlist[i], srcNlist[i] ]) totalSrcCopy = totalSrcCopy + 1 #Global variable #}12 for i in range(0, dstLen, 1): #{12 if srcNlist[i] != dstNlist[ i]: #{13 tempRN1 = func.buildPath( dstDir, srcNlist[i], delim) renameList.append([ dstDirFile, tempRN1 ]) totalDstRename = totalDstRename + 1 #Global variable #}13 #}12 #}11 elif numDiff < 0: #{11 for i in range( srcLen, dstLen, 1): #{12 tempRM0 = func.buildPath( dstDir, dstNlist[i], delim) removeList.append(tempRM0) totalDstRemove = totalDstRemove + 1 #Global variable #}12 for i in range(0, srcLen, 1): #{12 if srcNlist[i] != dstNlist[ i]: #{13 tempRN1 = func.buildPath( dstDir, srcNlist[i], delim) renameList.append([ dstDirFile, tempRN1 ]) totalDstRename = totalDstRename + 1 #Global variable #}13 #}12 #}11 else: #{11 for i in range(0, srcLen, 1): #{12 if srcNlist[i] != dstNlist[ i]: #{13 tempRN1 = func.buildPath( dstDir, srcNlist[i], delim) renameList.append([ dstDirFile, tempRN1 ]) totalDstRename = totalDstRename + 1 #Global variable #}13 #}12 #}11 #}10 toRemove = False sameFileSrc.append( srcObjList[d] ) #file that is the same continue #}9 #}8 #}7 #if nothing matches then do nothing #src file will be in srcToCopy list by not being in sameFileSrc list #dst file will add to delete list by toRemove not being set to False #this is the only place to break out of looping srcFileList #coz file size is sorted from small to large elif srcFsize > dstFsize: #{7 break #}7 #}6 #}5 #Condition for dst file removal is if the file does not exist in src #Or dst file exist in src, but it's more than 2 years old if toRemove == True: #{5 #provision for multi-copy files if dstFnum == 1: #{6 removeList.append(dstDirFile) totalDstRemove = totalDstRemove + 1 #Global variable #}6 else: #{6 nList = dstObjList[c].fileNames totalDstRemove = totalDstRemove + dstFnum for i in nList: #{7 tempRM = func.buildPath(dstDir, i, delim) removeList.append(tempRM) #}7 #}6 #}5 #}4 #}3 #}2 #To copy srcDir files that does not exist in dstDir to dstDir if len(srcObjList) == 0: #{2 return 1 #}2 else: #{2 #This if clause is for when dst dir match src dir, but it's empty #Only empty of files, but there may be sub-folders if len(dstObjList) == 0: #{3 for d in range(len(srcObjList)): #{4 if srcObjList[d].num == 1: #{5 checkNcopyList.append([ srcDir, dstDir, srcObjList[d].fileNames[0], srcObjList[d].fileNames[0] ]) totalSrcCopy = totalSrcCopy + 1 #Global variable #}5 else: #{5 nList = srcObjList[d].fileNames totalSrcCopy = totalSrcCopy + srcObjList[d].num for i in nList: #{6 checkNcopyList.append([srcDir, dstDir, i, i]) #}6 #}5 #}4 #}3 else: #{3 #If src file is not in the sameFileSrc list, then it's to be copied to dst #srcToCopy = list(set(sameFileSrc)^set(srcFileList)) #remove same elements between 2 1D lists for i in srcObjList: #{4 if not i in sameFileSrc: #{5 srcToCopy.append(i) #}5 #}4 for d in range(len(srcToCopy)): #{4 if srcToCopy[d].num == 1: #{5 checkNcopyList.append([ srcDir, dstDir, srcToCopy[d].fileNames[0], srcToCopy[d].fileNames[0] ]) totalSrcCopy = totalSrcCopy + 1 #Global variable #}5 else: #{5 nList = srcToCopy[d].fileNames totalSrcCopy = totalSrcCopy + srcToCopy[d].num for i in nList: #{6 checkNcopyList.append([srcDir, dstDir, i, i]) #}6 #}5 #}4 #}3 #}2 #at1 = datetime.datetime.now() #sec1 = at1.second #msec1 = at1.microsecond #t0 = sec0 + msec0/1000000 #t1 = sec1 + msec1/1000000 #dt = t1 - t0 #profile2.append(dt) return 1
def testSync(srcDir, dstDir): #{0 testSrcDir = [] testDstDir = [] testSrcFile = [] testDstFile = [] testContent = ["testFile0", "testFile1", "testFile2", "testFile3"] resFile = [] testSrcDir.append(func.buildPath(srcDir, "testDir", delim)) testDstDir.append(func.buildPath(dstDir, "testDir", delim)) tStr = "testDir" + delim + "subDir1" testSrcDir.append(func.buildPath(srcDir, tStr, delim)) tStr = "testDir" + delim + "subDir2" testSrcDir.append(func.buildPath(srcDir, tStr, delim)) testDstDir.append(func.buildPath(dstDir, tStr, delim)) tStr = "testDir" + delim + "subDir3" testDstDir.append(func.buildPath(dstDir, tStr, delim)) tStr1 = "testDir" + delim + "testFile0.txt" testSrcFile.append(func.buildPath(srcDir, tStr1, delim)) tStr2 = "testDir" + delim + "subDir1" + delim + "testFile1.txt" testSrcFile.append(func.buildPath(srcDir, tStr2, delim)) tStr3 = "testDir" + delim + "subDir2" + delim + "testFile2.txt" testSrcFile.append(func.buildPath(srcDir, tStr3, delim)) testDstFile.append(func.buildPath(dstDir, tStr3, delim)) tStr = "testDir" + delim + "subDir3" + delim + "testFile3.txt" testDstFile.append(func.buildPath(dstDir, tStr, delim)) resFile.append(func.buildPath(dstDir, tStr1, delim)) resFile.append(func.buildPath(dstDir, tStr2, delim)) resFile.append(func.buildPath(dstDir, tStr3, delim)) for sDir in testSrcDir: #{1 if not os.path.isdir(sDir): #{2 os.mkdir(sDir) #}2 #}1 for dDir in testDstDir: #{1 if not os.path.isdir(dDir): #{2 os.mkdir(dDir) #}2 #}1 for i in range(len(testSrcFile)): #{1 if not os.path.isfile(testSrcFile[i]): #{2 srcFile = open(testSrcFile[i], 'a', encoding='utf8') srcFile.write(testContent[i]) srcFile.close() #}2 #}1 for j in range(len(testDstFile)): #{1 if not os.path.isfile(testDstFile[j]): #{2 dstFile = open(testDstFile[j], 'a', encoding='utf8') dstFile.write(testContent[j + 2]) dstFile.close() #}2 #}1 print("Testing core function..") backupCopy(testSrcDir[0], testDstDir[0], '', '') filing(3, 1) if os.path.isfile(resFile[0]) and os.path.isfile(resFile[1]) \ and os.path.isfile(resFile[2]):#{1 print("Test successful") shutil.rmtree(testSrcDir[0], onerror=func.remove_readonly) shutil.rmtree(testDstDir[0], onerror=func.remove_readonly) return True #}1 else: #{1 print("Test failed") #shutil.rmtree(testSrcDir[0], onerror=func.remove_readonly) #shutil.rmtree(testDstDir[0], onerror=func.remove_readonly) return False
def sync_size_est(dst_drive): #{0 global dstRM_size global srcCP_size global net_sync_size dstRM_size = 0 srcCP_size = 0 net_sync_size = 0 dst_stat = shutil.disk_usage(dst_drive) dst_total = dst_stat.total #int in bytes dst_free = dst_stat.free #int in bytes dst_total_gb = dst_total * gb_conv #int in GB dst_free_gb = dst_free * gb_conv #int in GB print('Destination drive capacity:') print('Total: ' + str(dst_total_gb) + 'GB' + ' Free: ' + str(dst_free_gb) + 'GB') if len(removeList) != 0: #{1 #print(removeList) for f in removeList: #{2 rmFsize = os.path.getsize(f) dstRM_size = dstRM_size + rmFsize #}2 #}1 if len(rmtreeList) != 0: #{1 for f in rmtreeList: #{2 for d, sd, fs in os.walk(f): #{3 for fn in fs: #{4 fp = func.buildPath(d, fn, delim) fp_size = os.path.getsize(fp) dstRM_size = dstRM_size + fp_size #}4 #}3 #}2 #}1 if len(checkNcopyList) != 0: #{1 #print(checkNcopyList) for f in checkNcopyList: #{2 fp = func.buildPath(f[0], f[2], delim) cpFsize = os.path.getsize(fp) srcCP_size = srcCP_size + cpFsize #}2 #}1 if len(copytreeList) != 0: #{1 for f in copytreeList: #{2 fp = f[0] for d, sd, fs in os.walk(fp): #{3 for fn in fs: #{4 fp = func.buildPath(d, fn, delim) fp_size = os.path.getsize(fp) srcCP_size = srcCP_size + fp_size #}4 #}3 #}2 #}1 net_sync_size = srcCP_size - dstRM_size srcCP_size_gb = srcCP_size * gb_conv dstRM_size_gb = dstRM_size * gb_conv net_sync_size_gb = net_sync_size * gb_conv if srcCP_size_gb < 1: #{1 srcCP_size_mb = srcCP_size * mb_conv print('Files to copy: ' + str(srcCP_size_mb) + 'MB') #}1 else: #{1 print('Files to copy: ' + str(srcCP_size_gb) + 'GB') #}1 if dstRM_size_gb < 1: #{1 dstRM_size_mb = dstRM_size * mb_conv print('Files to remove: ' + str(dstRM_size_mb) + 'MB') #}1 else: #{1 print('Files to remove: ' + str(dstRM_size_gb) + 'GB') #}1 if net_sync_size_gb < 1: #{1 net_sync_size_mb = net_sync_size * mb_conv print('Net increase to destination: ' + str(net_sync_size_mb) + 'MB') #}1 else: #{1 print('Net increase to destination: ' + str(net_sync_size_gb) + 'GB')
def filing(iMode, iCS): #{0 global totalSrcFile global totalDstFile global totalSrcCopy global totalDstRemove global totalDstRename #totalSrcFile = 0 #totalDstFile = 0 totalSrcCopy = 0 totalDstRemove = 0 totalDstRename = 0 if iMode == 3: #{1 print('Mirror Syncing...') #}1 elif iMode == 1: #{1 print('Additive Syncing...') #}1 elif iMode == 2: #{1 print('File name Syncing...') #}1 if iMode == 3: #{1 for tf in rmtreeList: #{2 #print('Folder remove ' + tf) tfTotal = sum([len(cc) for aa, bb, cc in os.walk(tf)]) try: #{3 shutil.rmtree(tf) totalDstRemove = totalDstRemove + tfTotal #}3 except: #{3 tfLeft = sum([len(cc) for aa, bb, cc in os.walk(tf)]) tfRM = tfTotal - tfLeft totalDstRemove = totalDstRemove + tfRM print('dir removal error on: ' + tf) continue #}3 #}2 for rmf in removeList: #{2 try: #{3 os.remove(rmf) totalDstRemove = totalDstRemove + 1 #}3 except OSError: #{3 print('OSerror during file removal: ' + rmf) continue #}3 #print('Remove: ' + rmf) #}2 #}1 if iMode == 3 or iMode == 2: #{1 for rnf in renameList: #{2 try: #{3 os.rename(rnf[0], rnf[1]) totalDstRename = totalDstRename + 1 #}3 except OSError: #{3 print('OSerror during file rename: ' + rnf) continue #}3 #print('Rename: ' + rnf[0] + ' -> ' + rnf[1]) #}2 #}1 cpTreeErr = [] cpFileErr = [] if iMode == 3 or iMode == 1: #{1 #copytree function create dst directory path automatically for cpt in copytreeList: #{2 try: #{3 shutil.copytree(cpt[0], cpt[1]) temp = sum([len(cc) for aa, bb, cc in os.walk(cpt[1])]) totalSrcCopy = totalSrcCopy + temp #}3 except Error as err: #{3 print('Error during directory copy') print('src name: ' + str(err.args[0])) print('dst name: ' + str(err.args[1])) print('exception: ' + str(err.args[2])) cpTreeErr.append(cpt) continue #}3 #}2 for cptErr in cpTreeErr: #{2 copytreeList.remove(cptErr) #}2 for cpf in checkNcopyList: #{2 tempDst = func.buildPath(cpf[1], cpf[3], delim) #print('Folder copy ' + cpf[0] + ' to ' + cpf[1]) if fOps.checkNcopy(cpf[0], cpf[1], cpf[2], cpf[3], delim) == True: #{3 totalSrcCopy = totalSrcCopy + 1 #}3 else: #{3 cpFileErr.append(cpf) #}3 #}2 for cpfErr in cpFileErr: #{2 checkNcopyList.remove(cpfErr) #}2 #}1 #Checksum for transfered files if (iMode == 1 or iMode == 3) and iCS == 1: #{1 if len(checkNcopyList) != 0: #{2 v1 = validate(checkNcopyList, '', '', '') #}2 else: #{2 v1 = 3 #}2 if len(copytreeList) != 0: #{2 for i in copytreeList: #{3 v2 = validate(i[0], i[1], '', '') if v2 == 0: #{4 break #}4 #}3 #}2 else: #{2 v2 = 3 #}2 v3 = v1 + v2 if v3 == 2 or v3 == 4: #{2 print('Checksum: Transfered file checksum passed') #}2 elif v3 == 6: #{2 print('Checksum: Nothing to validate') #}2 else: #{2 print('Checksum: Transfered file checksum failed') #}2 #}1 rmtreeList.clear() copytreeList.clear() renameList.clear() removeList.clear() checkNcopyList.clear() print('Total source file count: ' + str(totalSrcFile)) print('Total destination file count: ' + str(totalDstFile)) print('Total source file copied: ' + str(totalSrcCopy)) print('Total destination file removed: ' + str(totalDstRemove)) print('Total destination file renamed: ' + str(totalDstRename)) if iMode == 3 and totalSrcFile == totalDstFile - totalDstRemove + totalSrcCopy: #{1 print("No issue encountered with mirroring") #}1 ''' else:#{1 print("Issue encountered, please investigate") #}1 ''' print('Sync finished\n')
def localCompare(targetDir, cmpMode, exlDir): #{0 entity0 = entityc.entity(targetDir, targetDir, delim) targetDir = entity0.targetDir eList = func.listExl(exlDir, delim) print('Directories to exclude:') print(eList) isExl = False cwd = os.getcwd() if not os.path.isdir(targetDir): #{1 print('Error: Target directory not found\n') return '' #}1 else: #{1 #Variable init and definition dupFileList.clear() dupList = [] opCounter = 0 cpCounter = 0 totalFiles = 0 cMode = '' if cmpMode == 0: #{2 cMode = 'Filecmp.cmp (shallow=False compare file content)' #}2 else: #{2 cMode = 'Hashlib (Checksum:SHA256)' #}2 print('Finding duplicated files within each sub-directory...') for folderName, subFolders, fileNames in os.walk(targetDir): #{2 fileList = [] objList = [] c = 0 for eDir in eList: #{3 if eDir in folderName: #{4 isExl = True #}4 #}3 if isExl == True: #{3 isExl = False #}3 else: #{3 for fileName in fileNames: #{4 if fileName.lower() != thumbs.lower(): #{5 totalFiles = totalFiles + 1 fSize = os.path.getsize( func.buildPath(folderName, fileName, delim)) #fileList.append([fileName,fsize]) fileList.append([folderName, fileName, fSize]) #}5 #}4 fileList.sort(key=lambda x: x[2]) #loadFileObj populates objList list with list of duplicated files func.loadFileObj(fileList, objList, True, delim, cmpMode, hash_Mode) ##for i in objList:#{4 ## ts = i.fileNames ## print(ts) ##}4 ##print('\n') if len(objList) != 0: #{4 for i in objList: #{5 timeList = [] Bvalue = 0 Bindex = 0 counter = 0 for j in i.fileNames: #{6 path = func.buildPath(i.fileDir[0], j, delim) ct = os.path.getctime(path) at = os.path.getatime(path) timeList.append(ct) timeList.append(at) #}6 while counter < len(timeList): #{6 if timeList[counter] > Bvalue: #{7 Bvalue = timeList[counter] Bindex = counter #}7 counter = counter + 1 #}6 cd = Bindex // 2 #cr = Bindex%2 #print(str(cd)) toKeep = 'To keep: ' + i.fileNames[cd] + ', to remove:' #print(toKeep) for j in range(len(i.fileNames)): #{6 if j != cd: #{7 toKeep = toKeep + ' ' + i.fileNames[j] toRM = func.buildPath(i.fileDir[0], i.fileNames[j], delim) dupFileList.append(toRM) #}7 #}6 toKeep = toKeep + ', at ' + i.fileDir[0] dupList.append(toKeep) cpCounter = cpCounter + 1 #}5 #}4 #}3 #}2 filecmp.clear_cache() #encoding arg is for writing nonEng char with open(cwd + delim + 'findDupFiles_report.txt', 'a', encoding='utf8') as outputFile: #{2 outputFile.write('\n') outputFile.write('Target directory: ' + targetDir + '\n') outputFile.write( 'Find duplicated files within each sub-directory\n') if len(dupList) > 0: #{3 outputFile.write('Files with duplicated content:\n') for i in dupList: #{4 outputFile.write(i + '\n') #}4 #}3 outputFile.write('Method of file comparison: ' + cMode + '\n') outputFile.write('Total file count: ' + str(totalFiles) + '\n') outputFile.write('Comparison Ops: ' + str(opCounter) + '\n') outputFile.write('Duplicated files: ' + str(cpCounter) + '\n') #outputFile.close() #using "with open" auto close file even with exception #}2 print('Method of file comparison: ' + cMode) print('Total file count: ' + str(totalFiles)) print('Duplicated files: ' + str(cpCounter)) dup_size_est() print('Please find analysis report in ' + cwd + delim + 'findDupFiles_report.txt') print('Analysis completed\n') ret = str(totalFiles) + '-0-' + str(cpCounter) return ret
def globalCompare(targetDir, cmpMode, exlDir): #{0 entity0 = entityc.entity(targetDir, targetDir, delim) targetDir = entity0.targetDir eList = func.listExl(exlDir, delim) print('Directories to exclude:') print(eList) isExl = False cwd = os.getcwd() if not os.path.isdir(targetDir): #{1 print('Error: Target directory not found\n') return '' #}1 else: #{1 fileList = [] objList = [] nameList = [] dupFileList.clear() opCounter = 0 cpCounter = 0 dnCounter = 0 totalFiles = 0 for folderName, subFolders, fileNames in os.walk(targetDir): #{2 for eDir in eList: #{3 if eDir in folderName: #{4 isExl = True #}4 #}3 if isExl == True: #{3 isExl = False #}3 else: #{3 for fileName in fileNames: #{4 if fileName.lower() != thumbs.lower(): #{5 totalFiles = totalFiles + 1 tPath = func.buildPath(folderName, fileName, delim) fSize = os.path.getsize(tPath) mTime = os.path.getmtime(tPath) fileList.append([folderName, fileName, fSize]) nameList.append([folderName, fileName, mTime]) #}5 #}4 #}3 #}2 #Now a list of all files and dir is ready if len(fileList) == 0: #{2 print('There are no files in target directory\n') return '' #}2 else: #{2 cMode = '' if cmpMode == 0: #{3 cMode = 'Filecmp.cmp (shallow=False compare file content)' #}3 else: #{3 cMode = 'Hashlib (Checksum:SHA256)' #}3 c = 0 dupName = [] dupList = [] nameList.sort(key=lambda x: x[1]) #sort in order of file name while c < (len(nameList) - 1): #{3 nc = c + 1 dir1 = nameList[c][0] dir2 = nameList[nc][0] file1 = nameList[c][1] file2 = nameList[nc][ 1] #file to be removed from nameList if same mtime1 = nameList[c][2] mtime2 = nameList[nc][2] if file1 == file2: #{4 #print(file1 + ' => ' + file2) df1 = func.buildPath(dir1, file1, delim) df2 = func.buildPath(dir2, file2, delim) if mtime1 > mtime2: #{5 df3 = 'latter' #}5 elif mtime2 > mtime1: #{5 df3 = 'former' #}5 else: #{5 df3 = 'either' #}5 dupName.append(df1 + ' == ' + df2 + ', Remove: ' + df3) dnCounter = dnCounter + 1 del nameList[nc] continue #effect is c is not incremented #}4 c = c + 1 #}3 fileList.sort( key=lambda x: x[2]) #sort fileList against fSize (file size) #loadFileObj populates objList list with list of duplicated files func.loadFileObj(fileList, objList, True, delim, cmpMode, hash_Mode) print('Finding duplicated files across all sub-directories...') if len(objList) != 0: #{3 for i in objList: #{4 timeList = [] Bvalue = 0 Bindex = 0 counter = 0 for j in range(len(i.fileNames)): #{5 if len(i.fileDir) == 1: #{6 k = 0 #}6 else: #{6 k = j #}6 path = func.buildPath(i.fileDir[k], i.fileNames[j], delim) ct = os.path.getctime(path) at = os.path.getatime(path) timeList.append(ct) timeList.append(at) #}5 while counter < len(timeList): #{5 if timeList[counter] > Bvalue: #{6 Bvalue = timeList[counter] Bindex = counter #}6 counter = counter + 1 #}5 cd = Bindex // 2 #cr = Bindex%2 #print(str(cd)) if len(i.fileDir) == 1: #{5 k = 0 #}5 else: #{5 k = cd #}5 keepath = func.buildPath(i.fileDir[k], i.fileNames[cd], delim) toKeep = 'To keep:\n' + keepath + '\nIn:\n' + keepath + '\n' #print(toKeep) for j in range(len(i.fileNames)): #{5 if j != cd: #{6 if len(i.fileDir) == 1: #{7 k = 0 #}7 else: #{7 k = j #}7 toRM = func.buildPath(i.fileDir[k], i.fileNames[j], delim) toKeep = toKeep + toRM + '\n' dupFileList.append(toRM) #}6 #}5 dupList.append(toKeep) cpCounter = cpCounter + 1 #}4 #}3 filecmp.clear_cache() #encoding arg is for writing nonEng char with open(cwd + delim + 'findDupFiles_report.txt', 'a', encoding='utf8') as outputFile: #{3 outputFile.write('\n') outputFile.write('Target directory: ' + targetDir + '\n') outputFile.write( 'Find duplicated files across all sub-directories\n') if len(dupList) > 0: #{4 outputFile.write('Files with duplicated content:\n') for i in dupList: #{5 outputFile.write(i + '\n') #}5 #}4 if len(dupName) > 0: #{4 outputFile.write('Files with duplicated names:\n') for i in dupName: #{5 outputFile.write(i + '\n') #}5 #}4 outputFile.write('Method of file comparison: ' + cMode + '\n') outputFile.write('Total file count: ' + str(totalFiles) + '\n') outputFile.write('Comparison Ops: ' + str(opCounter) + '\n') outputFile.write('Duplicated files: ' + str(cpCounter) + '\n') outputFile.write('Duplicated fileNames: ' + str(dnCounter) + '\n') outputFile.write( 'Only files with duplicated content are included in deletion\n' ) outputFile.write( 'Duplicated file naming require user investigation\n') #outputFile.close() #using "with open" auto close file even with exception #}3 print('Method of file comparison: ' + cMode) print('Total file count: ' + str(totalFiles)) print('Duplicated files: ' + str(cpCounter)) print('Duplicated fileNames: ' + str(dnCounter)) dup_size_est() print( 'Only files with duplicated content are included in deletion') print('Duplicated file naming require user investigation') print('Please find analysis report in ' + cwd + delim + 'findDupFiles_report.txt') print('Analysis completed\n') ret = str(totalFiles) + '-' + str(dnCounter) + '-' + str(cpCounter) return ret
def delDirFile(wd, targetName, isFolder, exlDir): #{0 entity0 = entityc.entity(wd, wd, delim) wd = entity0.targetDir eList = func.listExl(exlDir, delim) print('Directories to exclude:') print(eList) isExl = False if not os.path.isdir(wd): #{1 print('Error: target directory not found\n') return 0 #}1 else: #{1 workFolder = '' cpCounter = 0 delDFList.clear() if isFolder == False: #{2 checkTar = targetName.split('.') ext = checkTar[len(checkTar) - 1] if ext == 'sys' or ext == 'dll' or len(checkTar) == 1: #{3 print( 'Error: file extension must be specified and cannot be .sys or .dll\n' ) return 0 #}3 #}2 else: #{2 checkTar = targetName.split('.') ext = checkTar[0] if ext == '': #{3 print('Error: folder name cannot start with "."\n') return 0 #}3 #}2 print('Finding specified file/folder for deletion...') for folderName, subFolders, fileNames in os.walk(wd): #{2 for eDir in eList: #{3 if eDir in folderName: #{4 isExl = True #}4 #}3 if isExl == True: #{3 isExl = False #}3 else: #{3 tWorkFolder = folderName.split(delim) workFolder = tWorkFolder[len(tWorkFolder) - 1] #print('Folder inside ' + workFolder) if isFolder == False: #{4 for fileName in fileNames: #{5 if fileName.lower() == targetName.lower(): #{6 target = func.buildPath(folderName, fileName, delim) delDFList.append(target) cpCounter = cpCounter + 1 #}6 #}5 #}4 else: #{4 if folderName != wd: #{5 if workFolder.lower() == targetName.lower(): #{6 #Add to delete list delDFList.append(folderName) cpCounter = cpCounter + 1 #}6 #}5 #}4 #}3 #}2 print('Number of instances found: ' + str(cpCounter)) for i in delDFList: #{2 print(str(i)) #}2 print('Analysis completed\n') return cpCounter