def WalkFolders(Root):
"""
Fonction permettant de lister des archives et des dossiers
"""
Folders = []
Zipped = []
for root, dirs, files in os.walk(Root, topdown=False):
root = Path(root)
for FolderName in dirs:
Folder = root.joinpath(FolderName)
Inter2 = set(PassThisFiles).intersection(set(Folder.split("/")))
if len(Inter2) == 0:
AreIn = [Type in Folder for Type in AsZippedFolder]
if not True in AreIn:
Folders.append(Folder)
else:
InName = [
element in FolderName for element in AsZippedFolder
]
InPath = [
element in Folder.parent for element in AsZippedFolder
]
if True in InName and True not in InPath:
Zipped.append(Folder)
return Folders, Zipped
def WalkFiles(Root):
Files = []
for root, dirs, files in os.walk(Root, topdown=False):
root = Path(root)
for FileName in files:
File = root.joinpath(FileName)
AreIn = [Type in File for Type in AsZippedFolder]
Inter2 = set(PassThisFiles).intersection(set(File.split("/")))
if len(Inter2) == 0 and True not in AreIn:
Files.append(File)
return Files
def __init__(self,Root) :
Base = Path(Root)
OkRoot = Base.joinpath("MPfolder")
if os.path.isdir(OkRoot)==False :
os.mkdir(OkRoot)
else :
shutil.rmtree(OkRoot)
if " " in OkRoot :
#raise ValueError("The main path should not contain spaces to ensure the execution of the command")
print("There is a space in the root folder, I hope it will not break the rest of the code")
self.Root = OkRoot
self.Read=False
self.Jobs=[]
self.Ready = False
self.Libs=[]
self.RunThisBefore = []
def FileCopy(File, Root1, Root2):
"""
Fonction permettant de copier un fichier dans la destination
en verifiant sa derniere date de modification
"""
Operations = []
RelativePath = str(File)[len(str(Root1)):len(File)]
OutPath = Path(Root2 + "/Data/" + RelativePath)
OldVersionPath = Path(Root2 + "/OldVersion/" + RelativePath)
#si le fichier n'existe pas, il faut le copier
if os.path.isfile(OutPath) == False:
Out = CheckLength(OutPath)
Operations.append([(shutil.copy2, (str(File), Out),
"Saving this new File : " + File)])
#shutil.copy2(str(File),str(OutPath))
else:
#il faut verifier que celui existant ne colle pas en terme de date
Date1 = os.path.getmtime(File)
Date2 = os.path.getmtime(OutPath)
if abs(Date1 - Date2) > 15: #on tolere un ecart max de 15 secondes
#il faut deplacer l'ancien vers sa nouvelles maison
Name = File.name.split(".")
Date = datetime.utcfromtimestamp(int(Date1)).strftime("%d-%m-%Y")
if len(Name) == 1:
FinalName = Name[0] + "_" + Date
else:
FinalName = Name[0] + "_" + Date + "." + ".".join(
Name[1:len(Name)])
#NewFile = GetParent(OldVersionPath)+"/"+FinalName
NewFile = CheckLength(OldVersionPath.parent.joinpath(FinalName))
NewName = FindName(NewFile)
Operations.append([
(shutil.copy2, (OutPath, NewName),
"backing the previous version : " + NewFile),
(os.remove, (OutPath),
"Replacing the olversion (deletion) : " + OutPath),
(shutil.copy2, (File, OutPath),
"Replacing the olversion (copy) : " + OutPath)
])
#shutil.copy2(OutPath,NewFile)
#ensuite il faut supprimer l'ancien
#os.remove(OutPath)
#et finalement y mettre les nouveaux fichiers
#print("Saving the new version : "+OutPath)
#shutil.copy2(File,OutPath)
return Operations
def ArchiveCopy(Folder, Root1, Root2):
"""
Fonction permettant de copier un fichier archive dans la destination
en verifiant sa derniere date de modification
"""
Operations = []
RelativePath = str(Folder)[len(str(Root1)):len(Folder)]
OutPath = Path(Root2 + "\\Data/" + RelativePath)
OldVersionPath = Path(Root2 + "\\OldVersion/" + RelativePath)
#si le fichier n'existe pas, il faut le copier
if OutPath.isdir() == False:
Operations.append([(shutil.copytree, (str(Folder), str(OutPath)),
"Saving this new archive : " + Folder)])
#copy_tree(str(Folder),str(OutPath))
else:
#il faut verifier que celui existant ne colle pas en terme de date
Date1 = os.path.getmtime(Folder)
Date2 = os.path.getmtime(OutPath)
if abs(Date1 - Date2) > 15: #on tolere un ecart max de 15secondes
#il faut deplacer l'ancien vers sa nouvelles maison
Name = Folder.name.split(".")
Date = datetime.utcfromtimestamp(int(Date1)).strftime("%d-%m-%Y")
FinalName = Name[0] + "_" + Date + "." + Name[1]
NewFolder = OldVersionPath.parent.joinpath(FinalName)
NewName = FindName(NewFolder)
Operations.append([
(shutil.copytree, (OutPath, NewName),
"Backing the previous version : " + NewFolder),
(shutil.rmtree, (OutPath),
"Replacing the olversion (deletion) : " + OutPath),
(shutil.copytree, (Folder, OutPath),
"Replacing the olversion (copy) : " + OutPath)
])
#copy_tree(OutPath,NewFolder)
#ensuite il faut supprimer l'ancien
#shutil.rmtree(OutPath)
#et finalement y mettre les nouveaux fichiers
#copy_tree(Folder,OutPath)
return Operations
def FindName(File):
Root = str(File.parent)
Name = File.name
i = 1
while os.path.isfile(File):
Nb = str(i)
if len(Nb) == 1:
Nb = "0" + Nb
if "." in Name:
Elements = Name.split(".")
ActualName = Elements[0] + Nb + "." + ".".join(
Elements[1:len(Elements)])
else:
ActualName = Name + Nb
File = Path(Root + "/" + ActualName)
i += 1
return File
def StartBackup(Root1,
Root2,
UpdateVersions=False,
UpdateDelete=False,
MaxTimeVersion=30,
MaxTimeDelete=30,
NbCores=1):
"""
Fonction principale permettant d'effectuer le backup
"""
## Etape1 : verifier si les dossiers de reception sont la
PrepareOuput(Root2)
DataPath = Root2.joinpath("/Data")
OldPath = Root2.joinpath("/OldVersion")
DeletedPath = Root2.joinpath("/Deleted")
Now = datetime.now()
LogPath = Root2.joinpath("/logs/logs_" + Now.strftime("%d-%m-%Y") + ".txt")
LogFile = FindName(LogPath)
## Etape2 : mettre a jour l'arborescence
print("--------Preparing the folders---------")
CompleteDirTree(Root1, DataPath)
CompleteDirTree(Root1, OldPath)
## Etape4 : parcourir les dossier et copier les archives
Copied = []
Operations1 = []
Folders, Zippeds = WalkFolders(Root1)
for Zipped in Zippeds:
Folder = Path(Zipped)
print(Folder)
Operations1.append(ArchiveCopy(Folder, Root1, Root2))
Copied.append(Folder.name)
# #Execution des copie
print("--------Executing the copy of archives---------")
Messages1 = Execute(Operations1, NbCores)
Archives = set(Copied)
#
## Etape5 : parcourir les fichiers et les copiers
Operations1 = []
Files = WalkFiles(Root1)
print("Number of files found : " + str(len(Files)))
for File in Files:
File = Path(File)
Operations1.append(FileCopy(File, Root1, Root2))
print("--------Executing the copy of the files---------")
Messages2 = Execute(Operations1, NbCores)
## Etape6 : mettre a jour le dossier avec les anciennes versions
if UpdateVersions:
print("--------Executing the update of old files---------")
WaitTimeVersion = MaxTimeVersion * 24 * 60 * 60
Operations1 = CleanOldVersion(OldPath, WaitTimeVersion)
Messages3 = Execute2(Operations1, NbCores)
if UpdateDelete:
print("--------Executing the cleaning of deleted files---------")
WaitTimeDelete = MaxTimeDelete * 24 * 60 * 60
Operations1 = CleanDeleted(DeletedPath, WaitTimeDelete)
Messages4 = Execute2(Operations1, NbCores)
print("--------writing the logs---------")
AllMessages = Messages1 + Messages2 + Messages3 + Messages4
Logs = open(LogFile, "w")
for Messages in AllMessages:
for Message in Messages:
try:
Logs.write(Message + "\n")
except UnicodeEncodeError:
pass
Logs.close()
"CleanOld": True,
"WaitVersion": 30,
"WaitDelete": 30,
"NbCores": 40
},
# {"InputFolder" :"J:/",
# "OutputFolder":"Z:/CLEFS/PyBackup/JG_PROJ",
# "CleanVersion":True,"CleanOld":True,"WaitVersion":30,"WaitDelete":30, "NbCores":40},
# {"InputFolder" :"G:/",
# "OutputFolder":"F:/__PyKeysBackup/JG_PROJ",
# "CleanVersion":True,"CleanOld":True,"WaitVersion":30,"WaitDelete":30, "NbCores":40},
#{"InputFolder" :"","OutputFolder":"","CleanVersion":True,"CleanOld":True,"WaitVersion":30,"WaitDelete":30},
#{"InputFolder" :"","OutputFolder":"","CleanVersion":True,"CleanOld":True,"WaitVersion":30,"WaitDelete":30}
]
i = 1
for Dico in Parameters:
print("-------------------Staring the command number " + str(i))
start_time = time.time()
StartBackup(Path(Dico["InputFolder"]),
Path(Dico["OutputFolder"]),
UpdateVersions=Dico["CleanVersion"],
UpdateDelete=Dico["CleanOld"],
MaxTimeVersion=Dico["WaitVersion"],
MaxTimeDelete=Dico["WaitDelete"],
NbCores=Dico["NbCores"])
elapsed_time = time.time() - start_time
print("Time used to do all operations : " + NiceDuration(elapsed_time))
i += 1
print("")
print("")