def onChanged(self, location, serverpath, skipDeltaCheck):
changed_file = File.fromPath(serverpath)
action = None
#if not changed_file.servermdate:
# Probably a local added event that also
# spawned a modified event.
#return
file_name_only = os.path.basename(serverpath)
if engine_tools.isTemporaryFile(file_name_only):
print 'File ' + serverpath + ' ignored since it is a temporary file'
return
print 'File ' + serverpath + ':'
if changed_file.servermdate == None:
mydiff = "** File Not in Server **"
edit_time = "(not in server)"
else:
ttt = (changed_file.localmdate - changed_file.servermdate).total_seconds()
mydiff = str( ttt )
edit_time = str(changed_file.servermdate)
print 'Changed here %s, there %s delta %s' % (
changed_file.localmdate, edit_time, mydiff)
try:
if changed_file.inserver:
diff = changed_file.timeDiff()
MY_TOLERANCE = 10
if skipDeltaCheck == False and abs(diff) < MY_TOLERANCE:
return
if location == FileAction.SERVER:
if changed_file.inlocal:
if changed_file.localmdate < changed_file.servermdate:
action = FileAction(serverpath, FileAction.DOWNLOAD, FileAction.LOCAL)
else:
action = FileAction(serverpath, FileAction.DOWNLOAD, FileAction.LOCAL)
elif location == FileAction.LOCAL:
if changed_file.inserver:
try:
if changed_file.servermdate < changed_file.localmdate:
action = FileAction(serverpath, FileAction.UPLOAD, FileAction.SERVER)
except:
print 'Error:', changed_file, changed_file.servermdate, changed_file.localmdate
else:
action = FileAction(serverpath, FileAction.UPLOAD, FileAction.SERVER)
if action is not None:
self.actionQueue.add(action)
except:
info = traceback.format_exception(*sys.exc_info())
for i in info: sys.stderr.write(i)
def actionFromPath(serverpath):
f = File()
fileExistsOnServer = True
try:
f.servermdate = self.lastModified(serverpath)
except error_perm:
fileExistsOnServer = False
f.servermdate = 0
f.localmdate = LocalWatcher.lastModified(self.localFromServer(serverpath))
diff = f.timeDiff()
action = None
if abs(diff) > Watcher.TOLERANCE:
if not fileExistsOnServer or diff > 0:
action = FileAction(serverpath, FileAction.UPLOAD, ServerWatcher.LOCATION)
else:
action = FileAction(serverpath, FileAction.DOWNLOAD, LocalWatcher.LOCATION)
return action
def on_modified(self, event):
serverpath = self.serverFromLocal(event.src_path)
with File.fromPath(serverpath) as changed_file:
# Updating the database.
# Ensure file exists. Excel for example makes temp files.
try:
md = LocalWatcher.lastModified(event.src_path)
except:
# File doesn't exist anymore
return
changed_file.localmdate = md
self.fileChanged.emit(LocalWatcher.LOCATION, serverpath, True)
def on_modified(self, event):
serverpath = self.serverFromLocal(event.src_path)
with File.fromPath(serverpath) as changed_file:
# Updating the database.
# Ensure file exists. Excel for example makes temp files.
try:
md=LocalWatcher.lastModified(event.src_path)
except:
# File doesn't exist anymore
return
changed_file.localmdate = md
self.fileChanged.emit(LocalWatcher.LOCATION, serverpath, True)
def uploadFile(self, filename):
"""
Uploads the file `filename` to the server, creating
the needed directories.
:param filename: Absolute or relative path to the file
"""
def handle(buf):
"""This function is meant to be used as callback for the `storbinary` method."""
self.upload_progress += 1024
self.uploadProgress.emit(self.upload_size, self.upload_progress)
# Creates the directory where the file will be uploaded to
self.mkpath(os.path.dirname(filename))
localpath = self.localFromServer(filename)
print 'Uploading %s to %s' % (localpath, filename)
try:
# Uploads file and updates its modified date in the server
# to match the date in the local filesystem.
self.upload_progress = 0
self.upload_size = os.path.getsize(localpath)
self.fileEvent.emit(localpath)
self.ftp.storbinary('STOR %s' % filename,
open(localpath, 'rb'),
1024,
handle)
print 'Upload finished'
with File.fromPath(filename) as uploaded:
modified = uploaded.localmdate
uploaded.servermdate = modified
self.setLastModified(filename, modified)
uploaded = True
except (IOError, OSError):
uploaded = False
self.ioError.emit(localpath)
except (error_reply, error_perm, OSError) as err:
print 'Error uploading %s, %s' % (filename, err)
uploaded = False
# TODO: Sometimes the file doesn't complete properly.
# in that case we maybe shouldn't call this?
self.fileEventCompleted.emit()
return uploaded
def on_created(self, event):
serverpath = self.serverFromLocal(event.src_path)
with File.fromPath(serverpath) as added_file:
# Updating the database.
# First, ensure the file still exists.
try:
md = LocalWatcher.lastModified(event.src_path)
except:
return
added_file.localmdate = md
added_file.inlocal = True
self.fileAdded.emit(LocalWatcher.LOCATION, serverpath)
def on_created(self, event):
serverpath = self.serverFromLocal(event.src_path)
with File.fromPath(serverpath) as added_file:
# Updating the database.
# First, ensure the file still exists.
try:
md = LocalWatcher.lastModified(event.src_path)
except:
return
added_file.localmdate = md
added_file.inlocal = True
self.fileAdded.emit(LocalWatcher.LOCATION, serverpath)
def uploadFile(self, filename):
"""
Uploads the file `filename` to the server, creating
the needed directories.
:param filename: Absolute or relative path to the file
"""
def handle(buf):
"""This function is meant to be used as callback for the `storbinary` method."""
self.upload_progress += 1024
self.uploadProgress.emit(self.upload_size, self.upload_progress)
# Creates the directory where the file will be uploaded to
self.mkpath(os.path.dirname(filename))
localpath = self.localFromServer(filename)
print 'Uploading %s to %s' % (localpath, filename)
try:
# Uploads file and updates its modified date in the server
# to match the date in the local filesystem.
self.upload_progress = 0
self.upload_size = os.path.getsize(localpath)
self.fileEvent.emit(localpath)
self.ftp.storbinary('STOR %s' % filename, open(localpath, 'rb'),
1024, handle)
print 'Upload finished'
with File.fromPath(filename) as uploaded:
modified = uploaded.localmdate
uploaded.servermdate = modified
self.setLastModified(filename, modified)
uploaded = True
except (IOError, OSError):
uploaded = False
self.ioError.emit(localpath)
except (error_reply, error_perm, OSError) as err:
print 'Error uploading %s, %s' % (filename, err)
uploaded = False
# TODO: Sometimes the file doesn't complete properly.
# in that case we maybe shouldn't call this?
self.fileEventCompleted.emit()
return uploaded
def onAdded(self, location, serverpath):
file_name_only = os.path.basename(serverpath)
if engine_tools.isTemporaryFile(file_name_only):
print 'File ' + serverpath + ' was created but ignored since it is a temporary file'
return
added_file = File.fromPath(serverpath)
action = None
if location == FileAction.SERVER and not added_file.inlocal:
action = FileAction(serverpath, FileAction.DOWNLOAD, FileAction.LOCAL)
elif location == FileAction.LOCAL and not added_file.inserver:
action = FileAction(serverpath, FileAction.UPLOAD, FileAction.SERVER)
if action is not None:
self.actionQueue.add(action)
def onDeleted(self, location, serverpath):
# NOTE: For temporary files, the current action is to delete it.
# Reason 1: We need to remove it from the database.
# Reason 2: If somehow there is a temporary file
# there on the other side, then it makes sense to delete it.
deleted_file = File.fromPath(serverpath)
action = None
if location == FileAction.SERVER:
if deleted_file.inlocal:
action = FileAction(serverpath, FileAction.DELETE, FileAction.LOCAL)
elif location == FileAction.LOCAL:
if deleted_file.inserver:
action = FileAction(serverpath, FileAction.DELETE, FileAction.SERVER)
if action is not None:
self.actionQueue.add(action)
def onAdded(self, location, serverpath):
file_name_only = os.path.basename(serverpath)
if engine_tools.isTemporaryFile(file_name_only):
print 'File ' + serverpath + ' was created but ignored since it is a temporary file'
return
added_file = File.fromPath(serverpath)
action = None
if location == FileAction.SERVER and not added_file.inlocal:
action = FileAction(serverpath, FileAction.DOWNLOAD,
FileAction.LOCAL)
elif location == FileAction.LOCAL and not added_file.inserver:
action = FileAction(serverpath, FileAction.UPLOAD,
FileAction.SERVER)
if action is not None:
self.actionQueue.add(action)
def onDeleted(self, location, serverpath):
# NOTE: For temporary files, the current action is to delete it.
# Reason 1: We need to remove it from the database.
# Reason 2: If somehow there is a temporary file
# there on the other side, then it makes sense to delete it.
deleted_file = File.fromPath(serverpath)
action = None
if location == FileAction.SERVER:
if deleted_file.inlocal:
action = FileAction(serverpath, FileAction.DELETE,
FileAction.LOCAL)
elif location == FileAction.LOCAL:
if deleted_file.inserver:
action = FileAction(serverpath, FileAction.DELETE,
FileAction.SERVER)
if action is not None:
self.actionQueue.add(action)
def actionFromPath(serverpath):
f = File()
fileExistsOnServer = True
try:
f.servermdate = self.lastModified(serverpath)
except error_perm:
fileExistsOnServer = False
f.servermdate = 0
f.localmdate = LocalWatcher.lastModified(
self.localFromServer(serverpath))
diff = f.timeDiff()
action = None
if abs(diff) > Watcher.TOLERANCE:
if not fileExistsOnServer or diff > 0:
action = FileAction(serverpath, FileAction.UPLOAD,
ServerWatcher.LOCATION)
else:
action = FileAction(serverpath, FileAction.DOWNLOAD,
LocalWatcher.LOCATION)
return action
def checkout(self):
check_date = dt.utcnow()
fileC = 0
for item in os.walk(self.localdir):
directory = item[0]
subfiles = item[-1]
for file_ in subfiles:
# Added by Simon
# Give feedback on scanning of files.
fileC += 1
if fileC % 100 == 0:
self.textStatus.emit('Scanning local files for changes, '+str(fileC)+' scanned.')
# time.sleep(0.1)
localpath = os.path.join(directory, file_)
localmdate = LocalWatcher.lastModified(localpath)
serverpath = self.serverFromLocal(localpath)
flagg = 0
if localpath.endswith("local.conf"):
flagg = 1
print "Found local.conf"
file_is_in_server = False
with File.fromPath(serverpath) as local_file:
# If the file is not in the local DB,
# then it's new- we're just adding it now
just_added = not local_file.inlocal
print "JUST ADDED: " + str(just_added)
lastmdate = local_file.localmdate
print "LASTMDATE: " + str (lastmdate)
# Update values in the DB for this file
local_file.inlocal = True
local_file.last_checked_local = check_date
local_file.localmdate = localmdate
# Done updating values
delta = 0
file_is_in_server = local_file.inserver
if local_file.inserver:
delta = local_file.timeDiff()
# Emit the signals after the attributes has been set
# and committed.
if just_added is True:
self.fileAdded.emit(LocalWatcher.LOCATION, serverpath)
elif localmdate > lastmdate or delta > Watcher.TOLERANCE \
or not file_is_in_server:
print "FC_EMIT"
self.fileChanged.emit(LocalWatcher.LOCATION, serverpath, True)
# Deleted files are the ones whose `last_checked_local` attribute
# didn't get updated in the recursive run.
session = Session()
deleted = session.query(File).filter(File.last_checked_local < check_date).filter(File.inlocal == True)
for file_ in deleted:
self.fileDeleted.emit(LocalWatcher.LOCATION, file_.path)
session.commit()
def checkout(self):
check_date = dt.utcnow()
fileC = 0
for item in os.walk(self.localdir):
directory = item[0]
subfiles = item[-1]
for file_ in subfiles:
# Added by Simon
# Give feedback on scanning of files.
fileC += 1
if fileC % 100 == 0:
self.textStatus.emit('Scanning local files for changes, ' +
str(fileC) + ' scanned.')
# time.sleep(0.1)
localpath = os.path.join(directory, file_)
localmdate = LocalWatcher.lastModified(localpath)
serverpath = self.serverFromLocal(localpath)
flagg = 0
if localpath.endswith("local.conf"):
flagg = 1
print "Found local.conf"
file_is_in_server = False
with File.fromPath(serverpath) as local_file:
# If the file is not in the local DB,
# then it's new- we're just adding it now
just_added = not local_file.inlocal
print "JUST ADDED: " + str(just_added)
lastmdate = local_file.localmdate
print "LASTMDATE: " + str(lastmdate)
# Update values in the DB for this file
local_file.inlocal = True
local_file.last_checked_local = check_date
local_file.localmdate = localmdate
# Done updating values
delta = 0
file_is_in_server = local_file.inserver
if local_file.inserver:
delta = local_file.timeDiff()
# Emit the signals after the attributes has been set
# and committed.
if just_added is True:
self.fileAdded.emit(LocalWatcher.LOCATION, serverpath)
elif localmdate > lastmdate or delta > Watcher.TOLERANCE \
or not file_is_in_server:
print "FC_EMIT"
self.fileChanged.emit(LocalWatcher.LOCATION, serverpath,
True)
# Deleted files are the ones whose `last_checked_local` attribute
# didn't get updated in the recursive run.
session = Session()
deleted = session.query(File).filter(
File.last_checked_local < check_date).filter(File.inlocal == True)
for file_ in deleted:
self.fileDeleted.emit(LocalWatcher.LOCATION, file_.path)
session.commit()
def deleted(self, location, serverpath):
super(LocalWatcher, self).deleted(location, serverpath)
with File.fromPath(serverpath) as deleted:
deleted.inlocal = False
def takeAction(self):
self.actionTimer.stop()
if self.doPreemptive:
# Preemptive check is a bit of a workaround to deal with
# initial unexpected conditions: database file is gone
self.doPreemptive = False
self.server.preemptiveCheck = True
self.local.fileAdded.connect(self.server.added)
self.local.checkout()
self.server.checkout()
self.local.fileAdded.disconnect(self.server.added)
self.server.preemptiveCheck = False
for action in self.server.preemptiveActions:
self.actionQueue.add(action)
# After preemptive check, it is safe to do the connections
# for normal operations
self.connections()
serverActionCount = 0
localActionCount = 0
for action in self.actionQueue:
if action is not None:
print 'Next action: %s' % action
path = action.path
do = action.action
location = action.location
if location == FileAction.LOCAL and (do == FileAction.UPLOAD \
or do == FileAction.DELETE):
if not engine_tools.file_exists_local(path):
# File no longer exists at the time of processing.
# Maybe it was temporary or a quick rename.
# So we ignore it
print "Ignored action on " + path + ": File doesn't exist on local."
continue
if do == FileAction.UPLOAD:
self.uploadFile.emit(path)
localActionCount += 1
elif do == FileAction.DOWNLOAD:
self.downloadFile.emit(path)
serverActionCount += 1
elif do == FileAction.DELETE:
with File.fromPath(path) as deleted_file:
# `action.location` attribute only makes sense when deciding
# whether to delete a file on the server or local.
if location == FileAction.LOCAL:
localpath = self.local.localFromServer(path)
self.deleteLocalFile.emit(localpath)
deleted_file.inlocal = False
localActionCount += 1
elif location == FileAction.SERVER:
self.deleteServerFile.emit(path)
deleted_file.inserver = False
serverActionCount += 1
self.actionQueue.clear()
# Scan server for file changes
self.statusChanged.emit('Scanning remote files for changes')
self.server.checkout()
if self.firstScan:
# First do a full scan to check for offline changes.
# From there we will rely on real time notifications watchdog.
self.firstScan = False
self.statusChanged.emit('Scanning local files for changes')
self.local.checkout()
self.local.startObserver()
# Si Added
# Since its the first scan, we should also
# set the timer interval
self.actionTimer.setInterval(5000)
self.cleanSync()
# Si Added
# Set check interval intelligently.
# If there's no activity there, wait longer.
# Since if there's just no usage, then
# no reason to take up CPU cycles.
tempInterval = 0
if serverActionCount + localActionCount > 0:
tempInterval = 5000
else:
tempInterval = 1000 * 10
self.actionTimer.start()
def checkout(self):
"""
Recursively checks out all files on the server.
Returns a dictionary of files on the server with their last modified date.
:param download: Indicates whether or not the files should be downloaded
"""
# Check `self.deleteQueue`, `self.uploadQueue` and `self.downloadQueue` queues.
# These tasks are done in queues to make sure all FTP commands
# are done sequentially, in the same thread.
self.deleteAll()
self.uploadAll()
self.downloadAll()
# Handy list to keep track of the checkout process.
# This list contain absolute paths only.
checked_dirs = list()
# Sets '/' as initial directory and initializes `downloading_dir`
self.ftp.cwd('/')
downloading_dir = self.currentdir
check_date = dt.utcnow()
sidirlist = list()
root_cached = False
fileC = 0
while True:
# Gets the list of sub directories and files inside the
# current directory `downloading_dir`.
self.textStatus.emit('Remote scan- Downloading folder list of ' +
downloading_dir + '...')
if root_cached and downloading_dir == '/':
dir_subdirs = saved_root_dirs
dirfiles = saved_root_files
else:
dir_subdirs = self.getDirs(downloading_dir)
if downloading_dir == '/':
saved_root_dirs = dir_subdirs
# sidirlist.extend(dir_subdirs)
self.textStatus.emit(
'Remote scan- Downloading files list of ' +
downloading_dir + '...')
dirfiles = self.getFiles(downloading_dir)
if downloading_dir == '/':
saved_root_files = dirfiles
root_cached = True
# Leading '/' in `downloading_dir` breaks the `os.path.join` call
localdir = os.path.join(self.localdir, downloading_dir[1:])
if not os.path.exists(localdir):
# Creates the directory if it doesn't already exists.
os.makedirs(localdir)
for file_ in dirfiles:
# `serverpath` is the absolute path of the file on the server,
# download it only if it hasn't been already downloaded
serverpath = os.path.join(downloading_dir, file_)
serverpath = QDir.fromNativeSeparators(serverpath)
server_file = File.fromPath(serverpath)
self.textStatus.emit('Scanning remote file... ' + serverpath +
'...')
# How do we know if we should check this server file?
# We see if the date last checked is the check start time.
if server_file.last_checked_server != check_date:
# Do this process only once per file
# Added by Simon
# Give feedback on scanning of files.
fileC += 1
if fileC % 1 == 2:
self.textStatus.emit(
'Scanning remote files for changes, ' +
str(fileC) + ' files scanned.')
# STEP: IS THIS THE FIRST TIME WE SAW THE FILE, OR WAS IT ALREADY IN OUR DB?
just_added = not server_file.inserver
# STEP: IF ITS A NEW FILE, ENSURE WE DONT WANT TO SKIP IT
# Example: If it's a temporary file, or a Unix file with a name we don't support.
if just_added:
filename = os.path.basename(serverpath)
if platform.system() == 'Windows':
badName = False
for chr in [
'\\', '/', ':', '?', '"', '<', '>', '|'
]:
if chr in filename:
badName = True
break
if badName:
if filename not in self.warnedNames:
self.warnedNames.append(filename)
self.badFilenameFound.emit(filename)
continue
# STEP: ASSUMING THE FILE DID EXIST IN OUR DB, LETS SAVE THE LAST MODIFICATION DATE
lastmdate = server_file.servermdate
# STEP: SAVE THE MOD DATE TO A VARIABLE
# Now we get the last mod time.
# We expect this to work fine since this file
# was found on the server
servermdate = self.lastModified(serverpath)
# STEP: SET BOOL SHOWING THAT IT WAS ON THE SERVER, SINCE WE KNOW IT IS.
server_file.inserver = True
# STEP: SET THE TIME THE FILE WAS LAST CHECKED TO THE SCAN START TIME
server_file.last_checked_server = check_date
# STEP: SET THE MOD DATE IN THE DATABASE TO THE ONE WE JUST GOT
server_file.servermdate = servermdate
# STEP: SAVE THIS CHANGE TO THE DATABASE
server_file.session.commit()
delta = 0
if server_file.inlocal:
delta = server_file.timeDiff()
# Emit the signals after the attributes has been set and committed
if just_added is True:
self.fileAdded.emit(ServerWatcher.LOCATION, serverpath)
elif server_file.servermdate > lastmdate or delta < -Watcher.TOLERANCE:
self.fileChanged.emit(ServerWatcher.LOCATION,
serverpath, False)
#END FOR
self.textStatus.emit('Remote scan- Finding next folder...')
dir_ready = True
for dir_ in dir_subdirs:
# `dirpath` is the absolute path of the subdirectory on the server,
dirpath = QDir.fromNativeSeparators(
os.path.join(downloading_dir, dir_))
# `downloading_dir` is ready only when all its subdirectory are on the
# `checked_dirs` list.
if dirpath not in checked_dirs:
# Found one subdirectory that is not on `checked_dirs`,
# will process it in the next iteration.
downloading_dir = dirpath
dir_ready = False
break
if dir_ready is True:
# All subdirectories of `downloading_dir` are already in `checked_dirs`
if downloading_dir == '/':
# All directories ready and at '/', means checkout is complete
# So, exit the main While loop!!
break
else:
# Not at '/'. Current directory is ready so is appended to `checked_dirs`
# Back one directory to find directories that are not in `checked_dirs`
checked_dirs.append(downloading_dir)
downloading_dir = os.path.dirname(downloading_dir)
self.textStatus.emit('Remote scan- Found Folder...')
##### END OF WHILE ################
###################################################################
# Deleted files are the ones whose `last_checked_server` attribute
# didn't get updated in the recursive run.
session = Session()
deleted = session.query(File).filter(
File.last_checked_server < check_date).filter(
File.inserver == True)
for file_ in deleted:
self.fileDeleted.emit(ServerWatcher.LOCATION, file_.path)
# Wraps up the checkout process, commits to the database.
session.commit()
def onChanged(self, location, serverpath, skipDeltaCheck):
changed_file = File.fromPath(serverpath)
action = None
#if not changed_file.servermdate:
# Probably a local added event that also
# spawned a modified event.
#return
file_name_only = os.path.basename(serverpath)
if engine_tools.isTemporaryFile(file_name_only):
print 'File ' + serverpath + ' ignored since it is a temporary file'
return
print 'File ' + serverpath + ':'
if changed_file.servermdate == None:
mydiff = "** File Not in Server **"
edit_time = "(not in server)"
else:
ttt = (changed_file.localmdate -
changed_file.servermdate).total_seconds()
mydiff = str(ttt)
edit_time = str(changed_file.servermdate)
print 'Changed here %s, there %s delta %s' % (changed_file.localmdate,
edit_time, mydiff)
try:
if changed_file.inserver:
diff = changed_file.timeDiff()
MY_TOLERANCE = 10
if skipDeltaCheck == False and abs(diff) < MY_TOLERANCE:
return
if location == FileAction.SERVER:
if changed_file.inlocal:
if changed_file.localmdate < changed_file.servermdate:
action = FileAction(serverpath, FileAction.DOWNLOAD,
FileAction.LOCAL)
else:
action = FileAction(serverpath, FileAction.DOWNLOAD,
FileAction.LOCAL)
elif location == FileAction.LOCAL:
if changed_file.inserver:
try:
if changed_file.servermdate < changed_file.localmdate:
action = FileAction(serverpath, FileAction.UPLOAD,
FileAction.SERVER)
except:
print 'Error:', changed_file, changed_file.servermdate, changed_file.localmdate
else:
action = FileAction(serverpath, FileAction.UPLOAD,
FileAction.SERVER)
if action is not None:
self.actionQueue.add(action)
except:
info = traceback.format_exception(*sys.exc_info())
for i in info:
sys.stderr.write(i)
def downloadFile(self, filename, localpath=None):
"""
Performs a binary download to the file `filename` located on the server.
`filename` parameter can be either absolute or relative, though it can
fail for relative paths if the current directory is not appropiate.
:param filename: Relative or absolute path to the file
:param localpath: Absolute local path where the file will be saved
"""
def handleChunk(chunk):
"""
Receives chuncks of data downloaded from the server.
This function is meant to be used as callback for the `retrbinary` method.
:params chunk: Chunk of downloaded bytes to be written into the file
"""
# Simply writes the received data into the file `self.downloading`
self.downloading.write(chunk)
self.download_progress += len(chunk)
self.downloadProgress.emit(self.download_size,
self.download_progress)
if localpath is None:
localpath = self.localFromServer(filename)
localdir = os.path.dirname(localpath)
if not os.path.exists(localdir):
# Creates the directory if it doesn't already exists.
os.makedirs(localdir)
print 'Downloading: %s to %s' % (filename, localpath)
try:
with open(localpath, 'wb') as f:
# Opens the file at `localname` which will hold the downloaded file.
# Object attributes regarding download status are updated accordingly.
self.fileEvent.emit(filename)
self.downloading = f
self.download_progress = 0
self.download_size = int(
self.ftp.sendcmd('SIZE %s' % filename).split(' ')[-1])
self.ftp.retrbinary('RETR %s' % filename, handleChunk)
print 'Download finished'
# Let's set the same modified time to that on the server.
with File.fromPath(filename) as downloadedfile:
mdate = LocalWatcher.lastModified(localpath)
downloadedfile.localmdate = mdate
downloadedfile.servermdate = mdate
self.setLastModified(filename, mdate)
downloaded = True
except (IOError, OSError):
downloaded = False
self.ioError.emit(localpath)
except (error_reply, error_perm) as ftperr:
print 'Error downloading %s, %s' % (filename, ftperr)
downloaded = False
# TODO: Sometimes the file doesn't complete properly.
# in that case we maybe shouldn't call this?
self.fileEventCompleted.emit()
return downloaded
def downloadFile(self, filename, localpath=None):
"""
Performs a binary download to the file `filename` located on the server.
`filename` parameter can be either absolute or relative, though it can
fail for relative paths if the current directory is not appropiate.
:param filename: Relative or absolute path to the file
:param localpath: Absolute local path where the file will be saved
"""
def handleChunk(chunk):
"""
Receives chuncks of data downloaded from the server.
This function is meant to be used as callback for the `retrbinary` method.
:params chunk: Chunk of downloaded bytes to be written into the file
"""
# Simply writes the received data into the file `self.downloading`
self.downloading.write(chunk)
self.download_progress += len(chunk)
self.downloadProgress.emit(self.download_size, self.download_progress)
if localpath is None:
localpath = self.localFromServer(filename)
localdir = os.path.dirname(localpath)
if not os.path.exists(localdir):
# Creates the directory if it doesn't already exists.
os.makedirs(localdir)
print 'Downloading: %s to %s' % (filename, localpath)
try:
with open(localpath, 'wb') as f:
# Opens the file at `localname` which will hold the downloaded file.
# Object attributes regarding download status are updated accordingly.
self.fileEvent.emit(filename)
self.downloading = f
self.download_progress = 0
self.download_size = int(self.ftp.sendcmd('SIZE %s' % filename).split(' ')[-1])
self.ftp.retrbinary('RETR %s' % filename, handleChunk)
print 'Download finished'
# Let's set the same modified time to that on the server.
with File.fromPath(filename) as downloadedfile:
mdate = LocalWatcher.lastModified(localpath)
downloadedfile.localmdate = mdate
downloadedfile.servermdate = mdate
self.setLastModified(filename, mdate)
downloaded = True
except (IOError, OSError):
downloaded = False
self.ioError.emit(localpath)
except (error_reply, error_perm) as ftperr:
print 'Error downloading %s, %s' % (filename, ftperr)
downloaded = False
# TODO: Sometimes the file doesn't complete properly.
# in that case we maybe shouldn't call this?
self.fileEventCompleted.emit()
return downloaded
def deleted(self, location, serverpath):
super(LocalWatcher, self).deleted(location, serverpath)
with File.fromPath(serverpath) as deleted:
deleted.inlocal = False
def takeAction(self):
self.actionTimer.stop()
if self.doPreemptive:
# Preemptive check is a bit of a workaround to deal with
# initial unexpected conditions: database file is gone
self.doPreemptive = False
self.server.preemptiveCheck = True
self.local.fileAdded.connect(self.server.added)
self.local.checkout()
self.server.checkout()
self.local.fileAdded.disconnect(self.server.added)
self.server.preemptiveCheck = False
for action in self.server.preemptiveActions:
self.actionQueue.add(action)
# After preemptive check, it is safe to do the connections
# for normal operations
self.connections()
serverActionCount = 0
localActionCount = 0
for action in self.actionQueue:
if action is not None:
print 'Next action: %s' % action
path = action.path
do = action.action
location = action.location
if location == FileAction.LOCAL and (do == FileAction.UPLOAD \
or do == FileAction.DELETE):
if not engine_tools.file_exists_local(path):
# File no longer exists at the time of processing.
# Maybe it was temporary or a quick rename.
# So we ignore it
print "Ignored action on " + path + ": File doesn't exist on local."
continue
if do == FileAction.UPLOAD:
self.uploadFile.emit(path)
localActionCount += 1
elif do == FileAction.DOWNLOAD:
self.downloadFile.emit(path)
serverActionCount += 1
elif do == FileAction.DELETE:
with File.fromPath(path) as deleted_file:
# `action.location` attribute only makes sense when deciding
# whether to delete a file on the server or local.
if location == FileAction.LOCAL:
localpath = self.local.localFromServer(path)
self.deleteLocalFile.emit(localpath)
deleted_file.inlocal = False
localActionCount += 1
elif location == FileAction.SERVER:
self.deleteServerFile.emit(path)
deleted_file.inserver = False
serverActionCount += 1
self.actionQueue.clear()
# Scan server for file changes
self.statusChanged.emit('Scanning remote files for changes')
self.server.checkout()
if self.firstScan:
# First do a full scan to check for offline changes.
# From there we will rely on real time notifications watchdog.
self.firstScan = False
self.statusChanged.emit('Scanning local files for changes')
self.local.checkout()
self.local.startObserver()
# Si Added
# Since its the first scan, we should also
# set the timer interval
self.actionTimer.setInterval(5000)
self.cleanSync()
# Si Added
# Set check interval intelligently.
# If there's no activity there, wait longer.
# Since if there's just no usage, then
# no reason to take up CPU cycles.
tempInterval = 0
if serverActionCount+localActionCount > 0:
tempInterval = 5000
else:
tempInterval = 1000 * 10
self.actionTimer.start()
def checkout(self):
"""
Recursively checks out all files on the server.
Returns a dictionary of files on the server with their last modified date.
:param download: Indicates whether or not the files should be downloaded
"""
# Check `self.deleteQueue`, `self.uploadQueue` and `self.downloadQueue` queues.
# These tasks are done in queues to make sure all FTP commands
# are done sequentially, in the same thread.
self.deleteAll()
self.uploadAll()
self.downloadAll()
# Handy list to keep track of the checkout process.
# This list contain absolute paths only.
checked_dirs = list()
# Sets '/' as initial directory and initializes `downloading_dir`
self.ftp.cwd('/')
downloading_dir = self.currentdir
check_date = dt.utcnow()
sidirlist = list()
root_cached = False
fileC = 0
while True:
# Gets the list of sub directories and files inside the
# current directory `downloading_dir`.
self.textStatus.emit('Remote scan- Downloading folder list of '+downloading_dir+'...')
if root_cached and downloading_dir == '/':
dir_subdirs = saved_root_dirs
dirfiles = saved_root_files
else:
dir_subdirs = self.getDirs(downloading_dir)
if downloading_dir == '/':
saved_root_dirs = dir_subdirs
# sidirlist.extend(dir_subdirs)
self.textStatus.emit('Remote scan- Downloading files list of '+downloading_dir+'...')
dirfiles = self.getFiles(downloading_dir)
if downloading_dir == '/':
saved_root_files = dirfiles
root_cached = True
# Leading '/' in `downloading_dir` breaks the `os.path.join` call
localdir = os.path.join(self.localdir, downloading_dir[1:])
if not os.path.exists(localdir):
# Creates the directory if it doesn't already exists.
os.makedirs(localdir)
for file_ in dirfiles:
# `serverpath` is the absolute path of the file on the server,
# download it only if it hasn't been already downloaded
serverpath = os.path.join(downloading_dir, file_)
serverpath = QDir.fromNativeSeparators(serverpath)
server_file = File.fromPath(serverpath)
self.textStatus.emit('Scanning remote file... '+serverpath+'...')
# How do we know if we should check this server file?
# We see if the date last checked is the check start time.
if server_file.last_checked_server != check_date:
# Do this process only once per file
# Added by Simon
# Give feedback on scanning of files.
fileC += 1
if fileC % 1 == 2:
self.textStatus.emit('Scanning remote files for changes, '+str(fileC)+' files scanned.')
# STEP: IS THIS THE FIRST TIME WE SAW THE FILE, OR WAS IT ALREADY IN OUR DB?
just_added = not server_file.inserver
# STEP: IF ITS A NEW FILE, ENSURE WE DONT WANT TO SKIP IT
# Example: If it's a temporary file, or a Unix file with a name we don't support.
if just_added:
filename = os.path.basename(serverpath)
if platform.system() == 'Windows':
badName = False
for chr in ['\\', '/', ':', '?', '"', '<', '>', '|']:
if chr in filename:
badName = True
break
if badName:
if filename not in self.warnedNames:
self.warnedNames.append(filename)
self.badFilenameFound.emit(filename)
continue
# STEP: ASSUMING THE FILE DID EXIST IN OUR DB, LETS SAVE THE LAST MODIFICATION DATE
lastmdate = server_file.servermdate
# STEP: SAVE THE MOD DATE TO A VARIABLE
# Now we get the last mod time.
# We expect this to work fine since this file
# was found on the server
servermdate = self.lastModified(serverpath)
# STEP: SET BOOL SHOWING THAT IT WAS ON THE SERVER, SINCE WE KNOW IT IS.
server_file.inserver = True
# STEP: SET THE TIME THE FILE WAS LAST CHECKED TO THE SCAN START TIME
server_file.last_checked_server = check_date
# STEP: SET THE MOD DATE IN THE DATABASE TO THE ONE WE JUST GOT
server_file.servermdate = servermdate
# STEP: SAVE THIS CHANGE TO THE DATABASE
server_file.session.commit()
delta = 0
if server_file.inlocal:
delta = server_file.timeDiff()
# Emit the signals after the attributes has been set and committed
if just_added is True:
self.fileAdded.emit(ServerWatcher.LOCATION, serverpath)
elif server_file.servermdate > lastmdate or delta < -Watcher.TOLERANCE:
self.fileChanged.emit(ServerWatcher.LOCATION, serverpath, False)
#END FOR
self.textStatus.emit('Remote scan- Finding next folder...')
dir_ready = True
for dir_ in dir_subdirs:
# `dirpath` is the absolute path of the subdirectory on the server,
dirpath = QDir.fromNativeSeparators(os.path.join(downloading_dir, dir_))
# `downloading_dir` is ready only when all its subdirectory are on the
# `checked_dirs` list.
if dirpath not in checked_dirs:
# Found one subdirectory that is not on `checked_dirs`,
# will process it in the next iteration.
downloading_dir = dirpath
dir_ready = False
break
if dir_ready is True:
# All subdirectories of `downloading_dir` are already in `checked_dirs`
if downloading_dir == '/':
# All directories ready and at '/', means checkout is complete
# So, exit the main While loop!!
break
else:
# Not at '/'. Current directory is ready so is appended to `checked_dirs`
# Back one directory to find directories that are not in `checked_dirs`
checked_dirs.append(downloading_dir)
downloading_dir = os.path.dirname(downloading_dir)
self.textStatus.emit('Remote scan- Found Folder...')
##### END OF WHILE ################
###################################################################
# Deleted files are the ones whose `last_checked_server` attribute
# didn't get updated in the recursive run.
session = Session()
deleted = session.query(File).filter(File.last_checked_server < check_date).filter(File.inserver == True)
for file_ in deleted:
self.fileDeleted.emit(ServerWatcher.LOCATION, file_.path)
# Wraps up the checkout process, commits to the database.
session.commit()
