def __init__(self, ftp_website, directory, depth, excluded_extensions):
self.root_directory = directory
self.depth = depth
# list of the extensions to exclude during synchronization
self.excluded_extensions = excluded_extensions
# dictionary to remember the instance of File / Directory saved on the FTP
self.synchronize_dict = {}
self.os_separator_count = len(directory.split(os.path.sep))
# list of the path explored for each synchronization
self.paths_explored = []
# list of the File / Directory to removed from the dictionary at the end
# of the synchronization
self.to_remove_from_dict = []
# FTP instance
self.ftp = TalkToFTP(ftp_website)
# create the directory on the FTP if not already existing
self.ftp.connect()
if self.ftp.directory.count(os.path.sep) == 0:
# want to create folder at the root of the server
directory_split = ""
else:
directory_split = self.ftp.directory.rsplit(os.path.sep, 1)[0]
if not self.ftp.if_exist(self.ftp.directory,
self.ftp.get_folder_content(directory_split)):
self.ftp.create_folder(self.ftp.directory)
self.ftp.disconnect()
def __init__(self, ftp_website, directory, depth, excluded_extensions,
thread_enable, nb_thread):
self.root_directory = directory
self.depth = depth
self.arrayThr = queue.Queue()
if thread_enable == 1:
if nb_thread >= 1:
self.nbThr = nb_thread
else:
self.nbThr = 1
else:
self.nbThr = 1
Logger.log_info("number of threads : " + str(self.nbThr))
self.nbThrInUse = 0
self.lock = threading.Lock()
self.excluded_extensions = excluded_extensions
self.synchronize_dict = {}
self.os_separator_count = len(directory.split(os.path.sep))
self.paths_explored = []
self.to_remove_from_dict = []
self.ftp_website = ftp_website
self.ftp = TalkToFTP(ftp_website)
self.ftp.connect()
if self.ftp.directory.count(os.path.sep) == 0:
directory_split = ""
else:
directory_split = self.ftp.directory.rsplit(os.path.sep, 1)[0]
if not self.ftp.if_exist(self.ftp.directory,
self.ftp.get_folder_content(directory_split)):
self.ftp.create_folder(self.ftp.directory)
self.ftp.disconnect()
def __init__(self, ftp_website, directory, depth, nb_multi,
excluded_extensions):
# dictionary to remember the instance of File / Directory saved on the FTP
self.synchronize_dict = {}
# &&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&
self.ftp_website = ftp_website
self.listProcesses = []
self.nb_multi = nb_multi
self.uploadedFilesQueue = multiprocessing.Queue()
if self.nb_multi > 0:
for x in range(self.nb_multi):
p = multiprocessing.Process(
target=multiproc.simultaneousFileUploadV2,
args=(
self.ftp_website,
multiproc.FileUploadTask.QUEUE,
))
p.start()
self.listProcesses.append(p)
# &&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&
self.root_directory = directory
self.depth = depth
# list of the extensions to exclude during synchronization
self.excluded_extensions = excluded_extensions
self.os_separator_count = len(directory.split(os.path.sep))
# list of the path explored for each synchronization
self.paths_explored = []
# list of the File / Directory to removed from the dictionary at the end
# of the synchronization
self.to_remove_from_dict = []
# FTP instance
self.ftp = TalkToFTP(ftp_website)
# create the directory on the FTP if not already existing
self.ftp.connect()
if self.ftp.directory.count(os.path.sep) == 0:
# want to create folder at the root of the server
directory_split = ""
else:
directory_split = self.ftp.directory.rsplit(os.path.sep, 1)[0]
if not self.ftp.if_exist(self.ftp.directory,
self.ftp.get_folder_content(directory_split)):
self.ftp.create_folder(self.ftp.directory)
self.ftp.disconnect()
def threadMain(self):
while True != self.arrayThr.empty():
temp = self.arrayThr.get()
ftptemp = TalkToFTP(self.ftp_website)
ftptemp.connect()
ftptemp.file_transfer(temp[0], temp[1], temp[2])
ftptemp.disconnect()
self.lock.acquire()
try:
self.nbThrInUse -= 1
finally:
self.lock.release()
def simultaneousFileUploadV2(ftp_website, JobQueue):
"""
Wait and execute file upload jobs from jobQueue
"""
from logger import Logger
from talk_to_ftp import TalkToFTP
from File import File
while True:
# wait for job to be sent over from queue
if not JobQueue.empty():
myJob = JobQueue.get()
pathToFile = os.path.join(myJob.path, myJob.file_name)
# if file exists, we send it to ftp server
if os.path.isfile(pathToFile):
with open(pathToFile, 'rb') as file:
myFtp = TalkToFTP(ftp_website)
myFtp.connect()
Logger.log_info(
" FILE Starting : {0} - (Process : {1})".format(
myJob.srv_path, os.getpid()))
myFtp.ftp.storbinary('STOR ' + myJob.srv_path, file)
myFtp.disconnect()
Logger.log_info(
" FILE Done : {0} - (Process : {1})".format(
myJob.srv_path, os.getpid()))
def simultaneousFileUploadV1(ftp_website, path, srv_path, file_name):
"""
Upload file to ftp_website
"""
from logger import Logger
from talk_to_ftp import TalkToFTP
myFtp = TalkToFTP(ftp_website)
with open(os.path.join(path, file_name), 'rb') as file:
Logger.log_info(" FILE Starting : {0} - (Process : {1})".format(
srv_path, os.getpid()))
myFtp.connect()
myFtp.ftp.storbinary('STOR ' + srv_path, file)
myFtp.disconnect()
Logger.log_info(" FILE Done : {0} - (Process : {1})".format(
srv_path, os.getpid()))
class DirectoryManager:
def __init__(self, ftp_website, directory, depth, excluded_extensions):
self.root_directory = directory
self.depth = depth
# list of the extensions to exclude during synchronization
self.excluded_extensions = excluded_extensions
# dictionary to remember the instance of File / Directory saved on the FTP
self.synchronize_dict = {}
self.os_separator_count = len(directory.split(os.path.sep))
# list of the path explored for each synchronization
self.paths_explored = []
# list of the File / Directory to removed from the dictionary at the end
# of the synchronization
self.to_remove_from_dict = []
# FTP instance
self.ftp = TalkToFTP(ftp_website)
# create the directory on the FTP if not already existing
self.ftp.connect()
if self.ftp.directory.count(os.path.sep) == 0:
# want to create folder at the root of the server
directory_split = ""
else:
directory_split = self.ftp.directory.rsplit(os.path.sep, 1)[0]
if not self.ftp.if_exist(self.ftp.directory,
self.ftp.get_folder_content(directory_split)):
self.ftp.create_folder(self.ftp.directory)
self.ftp.disconnect()
def synchronize_directory(self, frequency):
while True:
# init the path explored to an empty list before each synchronization
self.paths_explored = []
# init to an empty list for each synchronization
self.to_remove_from_dict = []
# search for an eventual updates of files in the root directory
self.ftp.connect()
self.search_updates(self.root_directory)
# look for any removals of files / directories
self.any_removals()
self.ftp.disconnect()
# wait before next synchronization
time.sleep(frequency)
def search_updates(self, directory):
# scan recursively all files & directories in the root directory
for path_file, dirs, files in os.walk(directory):
for dir_name in dirs:
folder_path = os.path.join(path_file, dir_name)
# get depth of the current directory by the count of the os separator in a path
# and compare it with the count of the root directory
if self.is_superior_max_depth(folder_path) is False:
self.paths_explored.append(folder_path)
# a folder can't be updated, the only data we get is his creation time
# a folder get created during running time if not present in our list
if folder_path not in self.synchronize_dict.keys():
# directory created
# add it to dictionary
self.synchronize_dict[folder_path] = Directory(
folder_path)
# create it on FTP server
split_path = folder_path.split(self.root_directory)
srv_full_path = '{}{}'.format(self.ftp.directory,
split_path[1])
directory_split = srv_full_path.rsplit(os.path.sep,
1)[0]
if not self.ftp.if_exist(
srv_full_path,
self.ftp.get_folder_content(directory_split)):
# add this directory to the FTP server
self.ftp.create_folder(srv_full_path)
for file_name in files:
file_path = os.path.join(path_file, file_name)
# get depth of the current file by the count of the os separator in a path
# and compare it with the count of the root directory
if self.is_superior_max_depth(file_path) is False and \
(self.contain_excluded_extensions(file_path) is False):
self.paths_explored.append(file_path)
# try if already in the dictionary
if file_path in self.synchronize_dict.keys():
# if yes and he get updated, we update this file on the FTP server
if self.synchronize_dict[file_path].update_instance(
) == 1:
# file get updates
split_path = file_path.split(self.root_directory)
srv_full_path = '{}{}'.format(
self.ftp.directory, split_path[1])
self.ftp.remove_file(srv_full_path)
# update this file on the FTP server
self.ftp.file_transfer(path_file, srv_full_path,
file_name)
else:
# file get created
self.synchronize_dict[file_path] = File(file_path)
split_path = file_path.split(self.root_directory)
srv_full_path = '{}{}'.format(self.ftp.directory,
split_path[1])
# add this file on the FTP server
self.ftp.file_transfer(path_file, srv_full_path,
file_name)
def any_removals(self):
# if the length of the files & folders to synchronize == number of path explored
# no file / folder got removed
if len(self.synchronize_dict.keys()) == len(self.paths_explored):
return
# get the list of the files & folders removed
path_removed_list = [
key for key in self.synchronize_dict.keys()
if key not in self.paths_explored
]
for removed_path in path_removed_list:
# check if the current path is not in the list of path already deleted
# indeed we can't modify path_removed_list now because we're iterating over it
if removed_path not in self.to_remove_from_dict:
# get the instance of the files / folders deleted
# then use the appropriate methods to remove it from the FTP server
if isinstance(self.synchronize_dict[removed_path], File):
split_path = removed_path.split(self.root_directory)
srv_full_path = '{}{}'.format(self.ftp.directory,
split_path[1])
self.ftp.remove_file(srv_full_path)
self.to_remove_from_dict.append(removed_path)
elif isinstance(self.synchronize_dict[removed_path],
Directory):
split_path = removed_path.split(self.root_directory)
srv_full_path = '{}{}'.format(self.ftp.directory,
split_path[1])
self.to_remove_from_dict.append(removed_path)
# if it's a directory, we need to delete all the files and directories he contains
self.remove_all_in_directory(removed_path, srv_full_path,
path_removed_list)
# all the files / folders deleted in the local directory need to be deleted
# from the dictionary use to synchronize
for to_remove in self.to_remove_from_dict:
if to_remove in self.synchronize_dict.keys():
del self.synchronize_dict[to_remove]
def remove_all_in_directory(self, removed_directory, srv_full_path,
path_removed_list):
directory_containers = {}
for path in path_removed_list:
# path string contains removed_directory and this path did not get already deleted
if removed_directory != path and removed_directory in path \
and path not in self.to_remove_from_dict:
# if no path associated to the current depth we init it
if len(path.split(
os.path.sep)) not in directory_containers.keys():
directory_containers[len(path.split(os.path.sep))] = [path]
else:
# if some paths are already associated to the current depth
# we only append the current path
directory_containers[len(path.split(
os.path.sep))].append(path)
# sort the path depending on the file depth
sorted_containers = sorted(directory_containers.values())
# we iterate starting from the innermost file
for i in range(len(sorted_containers) - 1, -1, -1):
for to_delete in sorted_containers[i]:
to_delete_ftp = "{0}{1}{2}".format(
self.ftp.directory, os.path.sep,
to_delete.split(self.root_directory)[1])
if isinstance(self.synchronize_dict[to_delete], File):
self.ftp.remove_file(to_delete_ftp)
self.to_remove_from_dict.append(to_delete)
else:
# if it's again a directory, we delete all his containers also
self.remove_all_in_directory(to_delete, to_delete_ftp,
path_removed_list)
# once all the containers of the directory got removed
# we can delete the directory also
self.ftp.remove_folder(srv_full_path)
self.to_remove_from_dict.append(removed_directory)
# subtract current number of os separator to the number of os separator for the root directory
# if it's superior to the max depth, we do nothing
def is_superior_max_depth(self, path):
if (len(path.split(os.path.sep)) -
self.os_separator_count) <= self.depth:
return False
else:
return True
# check if the file contains a prohibited extensions
def contain_excluded_extensions(self, file):
extension = file.split(".")[1]
if ".{0}".format(extension) in self.excluded_extensions:
return True
else:
return False
def ThingModified(self, listAllThingModified, lock, ftp_website):
# we loop until our list is empty
while listAllThingModified:
# Only one thread at a time takes an element in our list.
with lock:
try:
# we unstack the last element of our list
# elem => [ type of the element ( file or directory ) ,
# what we have to do ( create , update , delete ) ,
# path of the file ,
# optionnal : full path ,
# optionnal : file name
elem = listAllThingModified.pop()
except:
break
ftp = TalkToFTP(ftp_website)
if elem:
# connection to the FTP
ftp.connect()
# we check if our element is a file
if elem[0] == "file":
# if we have to update the file
if elem[1] == "updateAndCreate":
try:
ftp.remove_file(elem[3])
ftp.file_transfer(*elem[2:])
except:
print('\033[33m' + "Cannot update and create " + elem[2])
traceback.print_exc()
# if we have to create the file
elif elem[1] == "create":
try:
ftp.file_transfer(*elem[2:])
except:
print('\033[33m' + "Cannot create file " + elem[2])
traceback.print_exc()
# if we have to delete the file
elif elem[1] == "delete":
try:
ftp.remove_file(elem[2])
except:
print('\033[33m' + "Cannot delete file" + elem[2])
traceback.print_exc()
else:
# if we have to create the directory
if elem[1] == "create":
try:
path = Path(elem[2])
error = False
# Verify that all parent exists
while path != path.parent:
if not os.path.exists(path.parent):
error = True
break
path = path.parent
if not error:
if not ftp.if_exist(elem[2], ftp.get_folder_content(elem[2].rsplit(os.path.sep, 1)[0])):
ftp.create_folder(*elem[2:])
else:
time.sleep(_TIME_SLEEP)
listAllThingModified.insert(0, elem)
except:
print('\033[33m' + "Cannot create dir" + elem[2])
traceback.print_exc()
# if we have to delete the directory
elif elem[1] == "delete":
try:
if not os.listdir(elem[2]):
ftp.remove_folder(elem[2])
else:
time.sleep(_TIME_SLEEP)
listAllThingModified.insert(0, elem)
except:
print('\033[33m' + "Cannot delete dir" + elem[2])
traceback.print_exc()
ftp.disconnect()
def __init__(self, ftp_website):
TalkToFTP.__init__(self, ftp_website)
class DirectoryManager:
def __init__(self, ftp_website, directory, depth, excluded_extensions,
thread_enable, nb_thread):
self.root_directory = directory
self.depth = depth
self.arrayThr = queue.Queue()
if thread_enable == 1:
if nb_thread >= 1:
self.nbThr = nb_thread
else:
self.nbThr = 1
else:
self.nbThr = 1
Logger.log_info("number of threads : " + str(self.nbThr))
self.nbThrInUse = 0
self.lock = threading.Lock()
self.excluded_extensions = excluded_extensions
self.synchronize_dict = {}
self.os_separator_count = len(directory.split(os.path.sep))
self.paths_explored = []
self.to_remove_from_dict = []
self.ftp_website = ftp_website
self.ftp = TalkToFTP(ftp_website)
self.ftp.connect()
if self.ftp.directory.count(os.path.sep) == 0:
directory_split = ""
else:
directory_split = self.ftp.directory.rsplit(os.path.sep, 1)[0]
if not self.ftp.if_exist(self.ftp.directory,
self.ftp.get_folder_content(directory_split)):
self.ftp.create_folder(self.ftp.directory)
self.ftp.disconnect()
def threadMain(self):
while True != self.arrayThr.empty():
temp = self.arrayThr.get()
ftptemp = TalkToFTP(self.ftp_website)
ftptemp.connect()
ftptemp.file_transfer(temp[0], temp[1], temp[2])
ftptemp.disconnect()
self.lock.acquire()
try:
self.nbThrInUse -= 1
finally:
self.lock.release()
def manageThread(self):
array = []
if (self.nbThr - self.nbThrInUse >= 0):
for i in range(self.nbThr - self.nbThrInUse):
self.lock.acquire()
try:
self.nbThrInUse += 1
finally:
self.lock.release()
array.append(threading.Thread(target=self.threadMain, args=[]))
for e in array:
e.start()
def synchronize_directory(self, frequency):
while True:
self.paths_explored = []
self.to_remove_from_dict = []
self.ftp.connect()
self.search_updates(self.root_directory)
self.any_removals()
self.ftp.disconnect()
time.sleep(frequency)
def search_updates(self, directory):
for path_file, dirs, files in os.walk(directory):
for dir_name in dirs:
folder_path = os.path.join(path_file, dir_name)
if self.is_superior_max_depth(folder_path) is False:
self.paths_explored.append(folder_path)
if folder_path not in self.synchronize_dict.keys():
self.synchronize_dict[folder_path] = Directory(
folder_path)
split_path = folder_path.split(self.root_directory)
srv_full_path = '{}{}'.format(self.ftp.directory,
split_path[1])
directory_split = srv_full_path.rsplit(os.path.sep,
1)[0]
if not self.ftp.if_exist(
srv_full_path,
self.ftp.get_folder_content(directory_split)):
self.ftp.create_folder(srv_full_path)
for file_name in files:
file_path = os.path.join(path_file, file_name)
if self.is_superior_max_depth(file_path) is False and \
(self.contain_excluded_extensions(file_path) is False):
self.paths_explored.append(file_path)
if file_path in self.synchronize_dict.keys():
if self.synchronize_dict[file_path].update_instance(
) == 1:
split_path = file_path.split(self.root_directory)
srv_full_path = '{}{}'.format(
self.ftp.directory, split_path[1])
self.ftp.remove_file(srv_full_path)
self.arrayThr.put(
[path_file, srv_full_path, file_name])
else:
self.synchronize_dict[file_path] = File(file_path)
split_path = file_path.split(self.root_directory)
srv_full_path = '{}{}'.format(self.ftp.directory,
split_path[1])
self.arrayThr.put(
[path_file, srv_full_path, file_name])
self.manageThread()
def any_removals(self):
if len(self.synchronize_dict.keys()) == len(self.paths_explored):
return
path_removed_list = [
key for key in self.synchronize_dict.keys()
if key not in self.paths_explored
]
for removed_path in path_removed_list:
if removed_path not in self.to_remove_from_dict:
if isinstance(self.synchronize_dict[removed_path], File):
split_path = removed_path.split(self.root_directory)
srv_full_path = '{}{}'.format(self.ftp.directory,
split_path[1])
self.ftp.remove_file(srv_full_path)
self.to_remove_from_dict.append(removed_path)
elif isinstance(self.synchronize_dict[removed_path],
Directory):
split_path = removed_path.split(self.root_directory)
srv_full_path = '{}{}'.format(self.ftp.directory,
split_path[1])
self.to_remove_from_dict.append(removed_path)
self.remove_all_in_directory(removed_path, srv_full_path,
path_removed_list)
for to_remove in self.to_remove_from_dict:
if to_remove in self.synchronize_dict.keys():
del self.synchronize_dict[to_remove]
def remove_all_in_directory(self, removed_directory, srv_full_path,
path_removed_list):
directory_containers = {}
for path in path_removed_list:
if removed_directory != path and removed_directory in path \
and path not in self.to_remove_from_dict:
if len(path.split(
os.path.sep)) not in directory_containers.keys():
directory_containers[len(path.split(os.path.sep))] = [path]
else:
directory_containers[len(path.split(
os.path.sep))].append(path)
sorted_containers = sorted(directory_containers.values())
for i in range(len(sorted_containers) - 1, -1, -1):
for to_delete in sorted_containers[i]:
to_delete_ftp = "{0}{1}{2}".format(
self.ftp.directory, os.path.sep,
to_delete.split(self.root_directory)[1])
if isinstance(self.synchronize_dict[to_delete], File):
self.ftp.remove_file(to_delete_ftp)
self.to_remove_from_dict.append(to_delete)
else:
self.remove_all_in_directory(to_delete, to_delete_ftp,
path_removed_list)
self.ftp.remove_folder(srv_full_path)
self.to_remove_from_dict.append(removed_directory)
def is_superior_max_depth(self, path):
if (len(path.split(os.path.sep)) -
self.os_separator_count) <= self.depth:
return False
else:
return True
def contain_excluded_extensions(self, file):
extension = file.split(".")[1]
if ".{0}".format(extension) in self.excluded_extensions:
return True
else:
return False