def cleanSync(self):
"""
Removes entries from the database for deleted files
"""
session = Session()
session.query(File).filter(File.inserver == False).filter(File.inlocal == False).delete(synchronize_session=False)
session.commit()
self.statusChanged.emit('Sync completed. Waiting for changes')
def cleanSync(self):
"""
Removes entries from the database for deleted files
"""
session = Session()
session.query(File).filter(File.inserver == False).filter(
File.inlocal == False).delete(synchronize_session=False)
session.commit()
self.statusChanged.emit('Sync completed. Waiting for changes')
def checkout(self):
check_date = dt.utcnow()
fileC = 0
for item in os.walk(self.localdir):
directory = item[0]
subfiles = item[-1]
for file_ in subfiles:
# Added by Simon
# Give feedback on scanning of files.
fileC += 1
if fileC % 100 == 0:
self.textStatus.emit('Scanning local files for changes, ' +
str(fileC) + ' scanned.')
# time.sleep(0.1)
localpath = os.path.join(directory, file_)
localmdate = LocalWatcher.lastModified(localpath)
serverpath = self.serverFromLocal(localpath)
flagg = 0
if localpath.endswith("local.conf"):
flagg = 1
print "Found local.conf"
file_is_in_server = False
with File.fromPath(serverpath) as local_file:
# If the file is not in the local DB,
# then it's new- we're just adding it now
just_added = not local_file.inlocal
print "JUST ADDED: " + str(just_added)
lastmdate = local_file.localmdate
print "LASTMDATE: " + str(lastmdate)
# Update values in the DB for this file
local_file.inlocal = True
local_file.last_checked_local = check_date
local_file.localmdate = localmdate
# Done updating values
delta = 0
file_is_in_server = local_file.inserver
if local_file.inserver:
delta = local_file.timeDiff()
# Emit the signals after the attributes has been set
# and committed.
if just_added is True:
self.fileAdded.emit(LocalWatcher.LOCATION, serverpath)
elif localmdate > lastmdate or delta > Watcher.TOLERANCE \
or not file_is_in_server:
print "FC_EMIT"
self.fileChanged.emit(LocalWatcher.LOCATION, serverpath,
True)
# Deleted files are the ones whose `last_checked_local` attribute
# didn't get updated in the recursive run.
session = Session()
deleted = session.query(File).filter(
File.last_checked_local < check_date).filter(File.inlocal == True)
for file_ in deleted:
self.fileDeleted.emit(LocalWatcher.LOCATION, file_.path)
session.commit()
def checkout(self):
"""
Recursively checks out all files on the server.
Returns a dictionary of files on the server with their last modified date.
:param download: Indicates whether or not the files should be downloaded
"""
# Check `self.deleteQueue`, `self.uploadQueue` and `self.downloadQueue` queues.
# These tasks are done in queues to make sure all FTP commands
# are done sequentially, in the same thread.
self.deleteAll()
self.uploadAll()
self.downloadAll()
# Handy list to keep track of the checkout process.
# This list contain absolute paths only.
checked_dirs = list()
# Sets '/' as initial directory and initializes `downloading_dir`
self.ftp.cwd('/')
downloading_dir = self.currentdir
check_date = dt.utcnow()
sidirlist = list()
root_cached = False
fileC = 0
while True:
# Gets the list of sub directories and files inside the
# current directory `downloading_dir`.
self.textStatus.emit('Remote scan- Downloading folder list of ' +
downloading_dir + '...')
if root_cached and downloading_dir == '/':
dir_subdirs = saved_root_dirs
dirfiles = saved_root_files
else:
dir_subdirs = self.getDirs(downloading_dir)
if downloading_dir == '/':
saved_root_dirs = dir_subdirs
# sidirlist.extend(dir_subdirs)
self.textStatus.emit(
'Remote scan- Downloading files list of ' +
downloading_dir + '...')
dirfiles = self.getFiles(downloading_dir)
if downloading_dir == '/':
saved_root_files = dirfiles
root_cached = True
# Leading '/' in `downloading_dir` breaks the `os.path.join` call
localdir = os.path.join(self.localdir, downloading_dir[1:])
if not os.path.exists(localdir):
# Creates the directory if it doesn't already exists.
os.makedirs(localdir)
for file_ in dirfiles:
# `serverpath` is the absolute path of the file on the server,
# download it only if it hasn't been already downloaded
serverpath = os.path.join(downloading_dir, file_)
serverpath = QDir.fromNativeSeparators(serverpath)
server_file = File.fromPath(serverpath)
self.textStatus.emit('Scanning remote file... ' + serverpath +
'...')
# How do we know if we should check this server file?
# We see if the date last checked is the check start time.
if server_file.last_checked_server != check_date:
# Do this process only once per file
# Added by Simon
# Give feedback on scanning of files.
fileC += 1
if fileC % 1 == 2:
self.textStatus.emit(
'Scanning remote files for changes, ' +
str(fileC) + ' files scanned.')
# STEP: IS THIS THE FIRST TIME WE SAW THE FILE, OR WAS IT ALREADY IN OUR DB?
just_added = not server_file.inserver
# STEP: IF ITS A NEW FILE, ENSURE WE DONT WANT TO SKIP IT
# Example: If it's a temporary file, or a Unix file with a name we don't support.
if just_added:
filename = os.path.basename(serverpath)
if platform.system() == 'Windows':
badName = False
for chr in [
'\\', '/', ':', '?', '"', '<', '>', '|'
]:
if chr in filename:
badName = True
break
if badName:
if filename not in self.warnedNames:
self.warnedNames.append(filename)
self.badFilenameFound.emit(filename)
continue
# STEP: ASSUMING THE FILE DID EXIST IN OUR DB, LETS SAVE THE LAST MODIFICATION DATE
lastmdate = server_file.servermdate
# STEP: SAVE THE MOD DATE TO A VARIABLE
# Now we get the last mod time.
# We expect this to work fine since this file
# was found on the server
servermdate = self.lastModified(serverpath)
# STEP: SET BOOL SHOWING THAT IT WAS ON THE SERVER, SINCE WE KNOW IT IS.
server_file.inserver = True
# STEP: SET THE TIME THE FILE WAS LAST CHECKED TO THE SCAN START TIME
server_file.last_checked_server = check_date
# STEP: SET THE MOD DATE IN THE DATABASE TO THE ONE WE JUST GOT
server_file.servermdate = servermdate
# STEP: SAVE THIS CHANGE TO THE DATABASE
server_file.session.commit()
delta = 0
if server_file.inlocal:
delta = server_file.timeDiff()
# Emit the signals after the attributes has been set and committed
if just_added is True:
self.fileAdded.emit(ServerWatcher.LOCATION, serverpath)
elif server_file.servermdate > lastmdate or delta < -Watcher.TOLERANCE:
self.fileChanged.emit(ServerWatcher.LOCATION,
serverpath, False)
#END FOR
self.textStatus.emit('Remote scan- Finding next folder...')
dir_ready = True
for dir_ in dir_subdirs:
# `dirpath` is the absolute path of the subdirectory on the server,
dirpath = QDir.fromNativeSeparators(
os.path.join(downloading_dir, dir_))
# `downloading_dir` is ready only when all its subdirectory are on the
# `checked_dirs` list.
if dirpath not in checked_dirs:
# Found one subdirectory that is not on `checked_dirs`,
# will process it in the next iteration.
downloading_dir = dirpath
dir_ready = False
break
if dir_ready is True:
# All subdirectories of `downloading_dir` are already in `checked_dirs`
if downloading_dir == '/':
# All directories ready and at '/', means checkout is complete
# So, exit the main While loop!!
break
else:
# Not at '/'. Current directory is ready so is appended to `checked_dirs`
# Back one directory to find directories that are not in `checked_dirs`
checked_dirs.append(downloading_dir)
downloading_dir = os.path.dirname(downloading_dir)
self.textStatus.emit('Remote scan- Found Folder...')
##### END OF WHILE ################
###################################################################
# Deleted files are the ones whose `last_checked_server` attribute
# didn't get updated in the recursive run.
session = Session()
deleted = session.query(File).filter(
File.last_checked_server < check_date).filter(
File.inserver == True)
for file_ in deleted:
self.fileDeleted.emit(ServerWatcher.LOCATION, file_.path)
# Wraps up the checkout process, commits to the database.
session.commit()
def checkout(self):
check_date = dt.utcnow()
fileC = 0
for item in os.walk(self.localdir):
directory = item[0]
subfiles = item[-1]
for file_ in subfiles:
# Added by Simon
# Give feedback on scanning of files.
fileC += 1
if fileC % 100 == 0:
self.textStatus.emit('Scanning local files for changes, '+str(fileC)+' scanned.')
# time.sleep(0.1)
localpath = os.path.join(directory, file_)
localmdate = LocalWatcher.lastModified(localpath)
serverpath = self.serverFromLocal(localpath)
flagg = 0
if localpath.endswith("local.conf"):
flagg = 1
print "Found local.conf"
file_is_in_server = False
with File.fromPath(serverpath) as local_file:
# If the file is not in the local DB,
# then it's new- we're just adding it now
just_added = not local_file.inlocal
print "JUST ADDED: " + str(just_added)
lastmdate = local_file.localmdate
print "LASTMDATE: " + str (lastmdate)
# Update values in the DB for this file
local_file.inlocal = True
local_file.last_checked_local = check_date
local_file.localmdate = localmdate
# Done updating values
delta = 0
file_is_in_server = local_file.inserver
if local_file.inserver:
delta = local_file.timeDiff()
# Emit the signals after the attributes has been set
# and committed.
if just_added is True:
self.fileAdded.emit(LocalWatcher.LOCATION, serverpath)
elif localmdate > lastmdate or delta > Watcher.TOLERANCE \
or not file_is_in_server:
print "FC_EMIT"
self.fileChanged.emit(LocalWatcher.LOCATION, serverpath, True)
# Deleted files are the ones whose `last_checked_local` attribute
# didn't get updated in the recursive run.
session = Session()
deleted = session.query(File).filter(File.last_checked_local < check_date).filter(File.inlocal == True)
for file_ in deleted:
self.fileDeleted.emit(LocalWatcher.LOCATION, file_.path)
session.commit()
def checkout(self):
"""
Recursively checks out all files on the server.
Returns a dictionary of files on the server with their last modified date.
:param download: Indicates whether or not the files should be downloaded
"""
# Check `self.deleteQueue`, `self.uploadQueue` and `self.downloadQueue` queues.
# These tasks are done in queues to make sure all FTP commands
# are done sequentially, in the same thread.
self.deleteAll()
self.uploadAll()
self.downloadAll()
# Handy list to keep track of the checkout process.
# This list contain absolute paths only.
checked_dirs = list()
# Sets '/' as initial directory and initializes `downloading_dir`
self.ftp.cwd('/')
downloading_dir = self.currentdir
check_date = dt.utcnow()
sidirlist = list()
root_cached = False
fileC = 0
while True:
# Gets the list of sub directories and files inside the
# current directory `downloading_dir`.
self.textStatus.emit('Remote scan- Downloading folder list of '+downloading_dir+'...')
if root_cached and downloading_dir == '/':
dir_subdirs = saved_root_dirs
dirfiles = saved_root_files
else:
dir_subdirs = self.getDirs(downloading_dir)
if downloading_dir == '/':
saved_root_dirs = dir_subdirs
# sidirlist.extend(dir_subdirs)
self.textStatus.emit('Remote scan- Downloading files list of '+downloading_dir+'...')
dirfiles = self.getFiles(downloading_dir)
if downloading_dir == '/':
saved_root_files = dirfiles
root_cached = True
# Leading '/' in `downloading_dir` breaks the `os.path.join` call
localdir = os.path.join(self.localdir, downloading_dir[1:])
if not os.path.exists(localdir):
# Creates the directory if it doesn't already exists.
os.makedirs(localdir)
for file_ in dirfiles:
# `serverpath` is the absolute path of the file on the server,
# download it only if it hasn't been already downloaded
serverpath = os.path.join(downloading_dir, file_)
serverpath = QDir.fromNativeSeparators(serverpath)
server_file = File.fromPath(serverpath)
self.textStatus.emit('Scanning remote file... '+serverpath+'...')
# How do we know if we should check this server file?
# We see if the date last checked is the check start time.
if server_file.last_checked_server != check_date:
# Do this process only once per file
# Added by Simon
# Give feedback on scanning of files.
fileC += 1
if fileC % 1 == 2:
self.textStatus.emit('Scanning remote files for changes, '+str(fileC)+' files scanned.')
# STEP: IS THIS THE FIRST TIME WE SAW THE FILE, OR WAS IT ALREADY IN OUR DB?
just_added = not server_file.inserver
# STEP: IF ITS A NEW FILE, ENSURE WE DONT WANT TO SKIP IT
# Example: If it's a temporary file, or a Unix file with a name we don't support.
if just_added:
filename = os.path.basename(serverpath)
if platform.system() == 'Windows':
badName = False
for chr in ['\\', '/', ':', '?', '"', '<', '>', '|']:
if chr in filename:
badName = True
break
if badName:
if filename not in self.warnedNames:
self.warnedNames.append(filename)
self.badFilenameFound.emit(filename)
continue
# STEP: ASSUMING THE FILE DID EXIST IN OUR DB, LETS SAVE THE LAST MODIFICATION DATE
lastmdate = server_file.servermdate
# STEP: SAVE THE MOD DATE TO A VARIABLE
# Now we get the last mod time.
# We expect this to work fine since this file
# was found on the server
servermdate = self.lastModified(serverpath)
# STEP: SET BOOL SHOWING THAT IT WAS ON THE SERVER, SINCE WE KNOW IT IS.
server_file.inserver = True
# STEP: SET THE TIME THE FILE WAS LAST CHECKED TO THE SCAN START TIME
server_file.last_checked_server = check_date
# STEP: SET THE MOD DATE IN THE DATABASE TO THE ONE WE JUST GOT
server_file.servermdate = servermdate
# STEP: SAVE THIS CHANGE TO THE DATABASE
server_file.session.commit()
delta = 0
if server_file.inlocal:
delta = server_file.timeDiff()
# Emit the signals after the attributes has been set and committed
if just_added is True:
self.fileAdded.emit(ServerWatcher.LOCATION, serverpath)
elif server_file.servermdate > lastmdate or delta < -Watcher.TOLERANCE:
self.fileChanged.emit(ServerWatcher.LOCATION, serverpath, False)
#END FOR
self.textStatus.emit('Remote scan- Finding next folder...')
dir_ready = True
for dir_ in dir_subdirs:
# `dirpath` is the absolute path of the subdirectory on the server,
dirpath = QDir.fromNativeSeparators(os.path.join(downloading_dir, dir_))
# `downloading_dir` is ready only when all its subdirectory are on the
# `checked_dirs` list.
if dirpath not in checked_dirs:
# Found one subdirectory that is not on `checked_dirs`,
# will process it in the next iteration.
downloading_dir = dirpath
dir_ready = False
break
if dir_ready is True:
# All subdirectories of `downloading_dir` are already in `checked_dirs`
if downloading_dir == '/':
# All directories ready and at '/', means checkout is complete
# So, exit the main While loop!!
break
else:
# Not at '/'. Current directory is ready so is appended to `checked_dirs`
# Back one directory to find directories that are not in `checked_dirs`
checked_dirs.append(downloading_dir)
downloading_dir = os.path.dirname(downloading_dir)
self.textStatus.emit('Remote scan- Found Folder...')
##### END OF WHILE ################
###################################################################
# Deleted files are the ones whose `last_checked_server` attribute
# didn't get updated in the recursive run.
session = Session()
deleted = session.query(File).filter(File.last_checked_server < check_date).filter(File.inserver == True)
for file_ in deleted:
self.fileDeleted.emit(ServerWatcher.LOCATION, file_.path)
# Wraps up the checkout process, commits to the database.
session.commit()