Exemple #1
0
    def onChanged(self, location, serverpath, skipDeltaCheck):
        changed_file = File.fromPath(serverpath)
        action = None
        
        #if not changed_file.servermdate:
            # Probably a local added event that also
            # spawned a modified event.
            #return

        file_name_only = os.path.basename(serverpath)
        if engine_tools.isTemporaryFile(file_name_only):
            print 'File ' + serverpath + ' ignored since it is a temporary file'
            return
           
        print 'File ' + serverpath + ':'
        
        if changed_file.servermdate == None:
            mydiff = "** File Not in Server **"
            edit_time = "(not in server)"
        else:
            ttt = (changed_file.localmdate - changed_file.servermdate).total_seconds() 
            mydiff = str( ttt )
            edit_time = str(changed_file.servermdate)
        
        print 'Changed here %s, there %s delta %s' % (
                    changed_file.localmdate, edit_time, mydiff)
                    
        
        try:
            if changed_file.inserver:
                diff = changed_file.timeDiff()

                MY_TOLERANCE = 10
            
                if skipDeltaCheck == False and abs(diff) < MY_TOLERANCE:
                    return
            
            if location == FileAction.SERVER:
                if changed_file.inlocal:
                    if changed_file.localmdate < changed_file.servermdate:
                        action = FileAction(serverpath, FileAction.DOWNLOAD, FileAction.LOCAL)
                else:
                    action = FileAction(serverpath, FileAction.DOWNLOAD, FileAction.LOCAL)
           
            elif location == FileAction.LOCAL:
                if changed_file.inserver:
                    try:
                        if changed_file.servermdate < changed_file.localmdate:
                            action = FileAction(serverpath, FileAction.UPLOAD, FileAction.SERVER)
                    except:
                        print 'Error:', changed_file, changed_file.servermdate, changed_file.localmdate
                        
                else:
                    action = FileAction(serverpath, FileAction.UPLOAD, FileAction.SERVER)
                
            if action is not None:
                self.actionQueue.add(action)
        except:
            info = traceback.format_exception(*sys.exc_info())
            for i in info: sys.stderr.write(i)
Exemple #2
0
 def actionFromPath(serverpath):
     f = File()
     fileExistsOnServer = True
     try:    
         f.servermdate = self.lastModified(serverpath)
     except error_perm:
         fileExistsOnServer = False
         f.servermdate = 0
     
     f.localmdate = LocalWatcher.lastModified(self.localFromServer(serverpath))
     diff = f.timeDiff()
     action = None
     if abs(diff) > Watcher.TOLERANCE:
         if not fileExistsOnServer or diff > 0:
             action = FileAction(serverpath, FileAction.UPLOAD, ServerWatcher.LOCATION)
         else:
             action = FileAction(serverpath, FileAction.DOWNLOAD, LocalWatcher.LOCATION)
     
     return action
Exemple #3
0
 def on_modified(self, event):
     serverpath = self.serverFromLocal(event.src_path)
     with File.fromPath(serverpath) as changed_file:
         # Updating the database.
         # Ensure file exists. Excel for example makes temp files.
         try:
             md = LocalWatcher.lastModified(event.src_path)
         except:
             # File doesn't exist anymore
             return
         changed_file.localmdate = md
     self.fileChanged.emit(LocalWatcher.LOCATION, serverpath, True)
Exemple #4
0
 def on_modified(self, event):
     serverpath = self.serverFromLocal(event.src_path)
     with File.fromPath(serverpath) as changed_file:
         # Updating the database.
         # Ensure file exists. Excel for example makes temp files.
         try:
             md=LocalWatcher.lastModified(event.src_path)
         except:
             # File doesn't exist anymore
             return
         changed_file.localmdate = md
     self.fileChanged.emit(LocalWatcher.LOCATION, serverpath, True)
Exemple #5
0
    def uploadFile(self, filename):
        """
        Uploads the file `filename` to the server, creating
        the needed directories.
        
        :param filename: Absolute or relative path to the file
        """
        
        def handle(buf):
            """This function is meant to be used as callback for the `storbinary` method."""
        
            self.upload_progress += 1024
            self.uploadProgress.emit(self.upload_size, self.upload_progress)
        
        
        # Creates the directory where the file will be uploaded to
        self.mkpath(os.path.dirname(filename))
        
        localpath = self.localFromServer(filename)
        print 'Uploading %s to %s' % (localpath, filename)
        
        try:
            # Uploads file and updates its modified date in the server
            # to match the date in the local filesystem.
            self.upload_progress = 0
            self.upload_size = os.path.getsize(localpath)
            self.fileEvent.emit(localpath)
            self.ftp.storbinary('STOR %s' % filename,
                                open(localpath, 'rb'), 
                                1024,
                                handle)
            print 'Upload finished'
            with File.fromPath(filename) as uploaded:
                modified = uploaded.localmdate
                uploaded.servermdate = modified
                
                self.setLastModified(filename, modified)
            
            uploaded = True

        except (IOError, OSError):
            uploaded = False
            self.ioError.emit(localpath)
        except (error_reply, error_perm, OSError) as err:
            print 'Error uploading %s, %s' % (filename, err)
            uploaded = False
            
        # TODO: Sometimes the file doesn't complete properly.
        # in that case we maybe shouldn't call this?            
        self.fileEventCompleted.emit()
        
        return uploaded
Exemple #6
0
 def on_created(self, event):
     
     serverpath = self.serverFromLocal(event.src_path)
     with File.fromPath(serverpath) as added_file:
         # Updating the database.
         # First, ensure the file still exists.
         try:
             md = LocalWatcher.lastModified(event.src_path)
         except:
             return
         added_file.localmdate = md            
         added_file.inlocal = True
         
     self.fileAdded.emit(LocalWatcher.LOCATION, serverpath)
Exemple #7
0
    def on_created(self, event):

        serverpath = self.serverFromLocal(event.src_path)
        with File.fromPath(serverpath) as added_file:
            # Updating the database.
            # First, ensure the file still exists.
            try:
                md = LocalWatcher.lastModified(event.src_path)
            except:
                return
            added_file.localmdate = md
            added_file.inlocal = True

        self.fileAdded.emit(LocalWatcher.LOCATION, serverpath)
Exemple #8
0
    def uploadFile(self, filename):
        """
        Uploads the file `filename` to the server, creating
        the needed directories.
        
        :param filename: Absolute or relative path to the file
        """
        def handle(buf):
            """This function is meant to be used as callback for the `storbinary` method."""

            self.upload_progress += 1024
            self.uploadProgress.emit(self.upload_size, self.upload_progress)

        # Creates the directory where the file will be uploaded to
        self.mkpath(os.path.dirname(filename))

        localpath = self.localFromServer(filename)
        print 'Uploading %s to %s' % (localpath, filename)

        try:
            # Uploads file and updates its modified date in the server
            # to match the date in the local filesystem.
            self.upload_progress = 0
            self.upload_size = os.path.getsize(localpath)
            self.fileEvent.emit(localpath)
            self.ftp.storbinary('STOR %s' % filename, open(localpath, 'rb'),
                                1024, handle)
            print 'Upload finished'
            with File.fromPath(filename) as uploaded:
                modified = uploaded.localmdate
                uploaded.servermdate = modified

                self.setLastModified(filename, modified)

            uploaded = True

        except (IOError, OSError):
            uploaded = False
            self.ioError.emit(localpath)
        except (error_reply, error_perm, OSError) as err:
            print 'Error uploading %s, %s' % (filename, err)
            uploaded = False

        # TODO: Sometimes the file doesn't complete properly.
        # in that case we maybe shouldn't call this?
        self.fileEventCompleted.emit()

        return uploaded
Exemple #9
0
    def onAdded(self, location, serverpath):

        file_name_only = os.path.basename(serverpath)
        if engine_tools.isTemporaryFile(file_name_only):
            print 'File ' + serverpath + ' was created but ignored since it is a temporary file'
            return
        
        added_file = File.fromPath(serverpath)
        action = None
        
        if location == FileAction.SERVER and not added_file.inlocal:
            action = FileAction(serverpath, FileAction.DOWNLOAD, FileAction.LOCAL)
        elif location == FileAction.LOCAL and not added_file.inserver:
            action = FileAction(serverpath, FileAction.UPLOAD, FileAction.SERVER)
            
        if action is not None:
            self.actionQueue.add(action)
Exemple #10
0
    def onDeleted(self, location, serverpath):

        # NOTE: For temporary files, the current action is to delete it.
        # Reason 1: We need to remove it from the database.
        # Reason 2: If somehow there is a temporary file 
        # there on the other side, then it makes sense to delete it.
        
        deleted_file = File.fromPath(serverpath)
        action = None
        
        if location == FileAction.SERVER:
            if deleted_file.inlocal:
                action = FileAction(serverpath, FileAction.DELETE, FileAction.LOCAL)
        elif location == FileAction.LOCAL:
            if deleted_file.inserver:
                action = FileAction(serverpath, FileAction.DELETE, FileAction.SERVER)
        
        if action is not None:
            self.actionQueue.add(action)
Exemple #11
0
    def onAdded(self, location, serverpath):

        file_name_only = os.path.basename(serverpath)
        if engine_tools.isTemporaryFile(file_name_only):
            print 'File ' + serverpath + ' was created but ignored since it is a temporary file'
            return

        added_file = File.fromPath(serverpath)
        action = None

        if location == FileAction.SERVER and not added_file.inlocal:
            action = FileAction(serverpath, FileAction.DOWNLOAD,
                                FileAction.LOCAL)
        elif location == FileAction.LOCAL and not added_file.inserver:
            action = FileAction(serverpath, FileAction.UPLOAD,
                                FileAction.SERVER)

        if action is not None:
            self.actionQueue.add(action)
Exemple #12
0
    def onDeleted(self, location, serverpath):

        # NOTE: For temporary files, the current action is to delete it.
        # Reason 1: We need to remove it from the database.
        # Reason 2: If somehow there is a temporary file
        # there on the other side, then it makes sense to delete it.

        deleted_file = File.fromPath(serverpath)
        action = None

        if location == FileAction.SERVER:
            if deleted_file.inlocal:
                action = FileAction(serverpath, FileAction.DELETE,
                                    FileAction.LOCAL)
        elif location == FileAction.LOCAL:
            if deleted_file.inserver:
                action = FileAction(serverpath, FileAction.DELETE,
                                    FileAction.SERVER)

        if action is not None:
            self.actionQueue.add(action)
Exemple #13
0
        def actionFromPath(serverpath):
            f = File()
            fileExistsOnServer = True
            try:
                f.servermdate = self.lastModified(serverpath)
            except error_perm:
                fileExistsOnServer = False
                f.servermdate = 0

            f.localmdate = LocalWatcher.lastModified(
                self.localFromServer(serverpath))
            diff = f.timeDiff()
            action = None
            if abs(diff) > Watcher.TOLERANCE:
                if not fileExistsOnServer or diff > 0:
                    action = FileAction(serverpath, FileAction.UPLOAD,
                                        ServerWatcher.LOCATION)
                else:
                    action = FileAction(serverpath, FileAction.DOWNLOAD,
                                        LocalWatcher.LOCATION)

            return action
Exemple #14
0
    def checkout(self):
        check_date = dt.utcnow()
        fileC = 0
        for item in os.walk(self.localdir):
            directory = item[0]
            subfiles = item[-1]

            for file_ in subfiles:

                # Added by Simon
                # Give feedback on scanning of files.
                fileC += 1
                if fileC % 100 == 0:
                    self.textStatus.emit('Scanning local files for changes, '+str(fileC)+' scanned.')
                    # time.sleep(0.1)
                
                localpath = os.path.join(directory, file_)
                localmdate = LocalWatcher.lastModified(localpath)
                serverpath = self.serverFromLocal(localpath)

                flagg = 0
                if localpath.endswith("local.conf"):
                    flagg = 1
                    print "Found local.conf"

                file_is_in_server = False
                
                with File.fromPath(serverpath) as local_file:
                    # If the file is not in the local DB,
                    # then it's new- we're just adding it now
                    just_added = not local_file.inlocal                        
                    print "JUST ADDED: " + str(just_added)
                    
                    lastmdate = local_file.localmdate
                    print "LASTMDATE: " + str (lastmdate)
                    
                    # Update values in the DB for this file
                    local_file.inlocal = True
                    local_file.last_checked_local = check_date
                    local_file.localmdate = localmdate
                    # Done updating values
                    
                    delta = 0
                    file_is_in_server = local_file.inserver                    
                    if local_file.inserver:
                        delta = local_file.timeDiff()
                    
                # Emit the signals after the attributes has been set
                # and committed.
                if just_added is True:
                    self.fileAdded.emit(LocalWatcher.LOCATION, serverpath)
                elif localmdate > lastmdate or delta > Watcher.TOLERANCE \
                     or not file_is_in_server:
                    print "FC_EMIT"
                    self.fileChanged.emit(LocalWatcher.LOCATION, serverpath, True)

        # Deleted files are the ones whose `last_checked_local` attribute 
        # didn't get updated in the recursive run.
        session = Session()
        deleted = session.query(File).filter(File.last_checked_local < check_date).filter(File.inlocal == True)
        for file_ in deleted:
            self.fileDeleted.emit(LocalWatcher.LOCATION, file_.path)
            
        session.commit()
Exemple #15
0
    def checkout(self):
        check_date = dt.utcnow()
        fileC = 0
        for item in os.walk(self.localdir):
            directory = item[0]
            subfiles = item[-1]

            for file_ in subfiles:

                # Added by Simon
                # Give feedback on scanning of files.
                fileC += 1
                if fileC % 100 == 0:
                    self.textStatus.emit('Scanning local files for changes, ' +
                                         str(fileC) + ' scanned.')
                    # time.sleep(0.1)

                localpath = os.path.join(directory, file_)
                localmdate = LocalWatcher.lastModified(localpath)
                serverpath = self.serverFromLocal(localpath)

                flagg = 0
                if localpath.endswith("local.conf"):
                    flagg = 1
                    print "Found local.conf"

                file_is_in_server = False

                with File.fromPath(serverpath) as local_file:
                    # If the file is not in the local DB,
                    # then it's new- we're just adding it now
                    just_added = not local_file.inlocal
                    print "JUST ADDED: " + str(just_added)

                    lastmdate = local_file.localmdate
                    print "LASTMDATE: " + str(lastmdate)

                    # Update values in the DB for this file
                    local_file.inlocal = True
                    local_file.last_checked_local = check_date
                    local_file.localmdate = localmdate
                    # Done updating values

                    delta = 0
                    file_is_in_server = local_file.inserver
                    if local_file.inserver:
                        delta = local_file.timeDiff()

                # Emit the signals after the attributes has been set
                # and committed.
                if just_added is True:
                    self.fileAdded.emit(LocalWatcher.LOCATION, serverpath)
                elif localmdate > lastmdate or delta > Watcher.TOLERANCE \
                     or not file_is_in_server:
                    print "FC_EMIT"
                    self.fileChanged.emit(LocalWatcher.LOCATION, serverpath,
                                          True)

        # Deleted files are the ones whose `last_checked_local` attribute
        # didn't get updated in the recursive run.
        session = Session()
        deleted = session.query(File).filter(
            File.last_checked_local < check_date).filter(File.inlocal == True)
        for file_ in deleted:
            self.fileDeleted.emit(LocalWatcher.LOCATION, file_.path)

        session.commit()
Exemple #16
0
 def deleted(self, location, serverpath):
     super(LocalWatcher, self).deleted(location, serverpath)
     with File.fromPath(serverpath) as deleted:
         deleted.inlocal = False
Exemple #17
0
    def takeAction(self):
        self.actionTimer.stop()

        if self.doPreemptive:
            # Preemptive check is a bit of a workaround to deal with
            # initial unexpected conditions: database file is gone
            self.doPreemptive = False
            self.server.preemptiveCheck = True
            self.local.fileAdded.connect(self.server.added)
            self.local.checkout()
            self.server.checkout()
            self.local.fileAdded.disconnect(self.server.added)
            self.server.preemptiveCheck = False
            for action in self.server.preemptiveActions:
                self.actionQueue.add(action)

        # After preemptive check, it is safe to do the connections
        # for normal operations
        self.connections()

        serverActionCount = 0
        localActionCount = 0
        for action in self.actionQueue:
            if action is not None:
                print 'Next action: %s' % action
                path = action.path
                do = action.action
                location = action.location

                if location == FileAction.LOCAL and (do == FileAction.UPLOAD \
                   or do == FileAction.DELETE):
                    if not engine_tools.file_exists_local(path):
                        # File no longer exists at the time of processing.
                        # Maybe it was temporary or a quick rename.
                        # So we ignore it
                        print "Ignored action on " + path + ": File doesn't exist on local."
                        continue

                if do == FileAction.UPLOAD:
                    self.uploadFile.emit(path)
                    localActionCount += 1
                elif do == FileAction.DOWNLOAD:
                    self.downloadFile.emit(path)
                    serverActionCount += 1
                elif do == FileAction.DELETE:
                    with File.fromPath(path) as deleted_file:
                        # `action.location` attribute only makes sense when deciding
                        # whether to delete a file on the server or local.
                        if location == FileAction.LOCAL:
                            localpath = self.local.localFromServer(path)
                            self.deleteLocalFile.emit(localpath)
                            deleted_file.inlocal = False
                            localActionCount += 1

                        elif location == FileAction.SERVER:
                            self.deleteServerFile.emit(path)
                            deleted_file.inserver = False
                            serverActionCount += 1

        self.actionQueue.clear()

        # Scan server for file changes
        self.statusChanged.emit('Scanning remote files for changes')
        self.server.checkout()

        if self.firstScan:
            # First do a full scan to check for offline changes.
            # From there we will rely on real time notifications watchdog.
            self.firstScan = False
            self.statusChanged.emit('Scanning local files for changes')
            self.local.checkout()
            self.local.startObserver()
            # Si Added
            # Since its the first scan, we should also
            # set the timer interval
            self.actionTimer.setInterval(5000)
        self.cleanSync()

        # Si Added
        # Set check interval intelligently.
        # If there's no activity there, wait longer.
        # Since if there's just no usage, then
        # no reason to take up CPU cycles.
        tempInterval = 0
        if serverActionCount + localActionCount > 0:
            tempInterval = 5000
        else:
            tempInterval = 1000 * 10

        self.actionTimer.start()
Exemple #18
0
    def checkout(self):
        """
        Recursively checks out all files on the server.
        Returns a dictionary of files on the server with their last modified date.
        
        :param download: Indicates whether or not the files should be downloaded
        """

        # Check  `self.deleteQueue`, `self.uploadQueue` and `self.downloadQueue` queues.
        # These tasks are done in queues to make sure all FTP commands
        # are done sequentially, in the same thread.
        self.deleteAll()
        self.uploadAll()
        self.downloadAll()

        # Handy list to keep track of the checkout process.
        # This list contain absolute paths only.
        checked_dirs = list()

        # Sets '/' as initial directory and initializes `downloading_dir`
        self.ftp.cwd('/')
        downloading_dir = self.currentdir
        check_date = dt.utcnow()

        sidirlist = list()

        root_cached = False

        fileC = 0
        while True:
            # Gets the list of sub directories and files inside the
            # current directory `downloading_dir`.
            self.textStatus.emit('Remote scan- Downloading folder list of ' +
                                 downloading_dir + '...')

            if root_cached and downloading_dir == '/':
                dir_subdirs = saved_root_dirs
                dirfiles = saved_root_files
            else:

                dir_subdirs = self.getDirs(downloading_dir)

                if downloading_dir == '/':
                    saved_root_dirs = dir_subdirs

                # sidirlist.extend(dir_subdirs)
                self.textStatus.emit(
                    'Remote scan- Downloading files list of ' +
                    downloading_dir + '...')
                dirfiles = self.getFiles(downloading_dir)

                if downloading_dir == '/':
                    saved_root_files = dirfiles
                    root_cached = True

            # Leading '/' in `downloading_dir` breaks the `os.path.join` call
            localdir = os.path.join(self.localdir, downloading_dir[1:])
            if not os.path.exists(localdir):
                # Creates the directory if it doesn't already exists.
                os.makedirs(localdir)

            for file_ in dirfiles:

                # `serverpath` is the absolute path of the file on the server,
                # download it only if it hasn't been already downloaded
                serverpath = os.path.join(downloading_dir, file_)
                serverpath = QDir.fromNativeSeparators(serverpath)
                server_file = File.fromPath(serverpath)

                self.textStatus.emit('Scanning remote file... ' + serverpath +
                                     '...')

                # How do we know if we should check this server file?
                # We see if the date last checked is the check start time.
                if server_file.last_checked_server != check_date:

                    # Do this process only once per file

                    # Added by Simon
                    # Give feedback on scanning of files.
                    fileC += 1
                    if fileC % 1 == 2:
                        self.textStatus.emit(
                            'Scanning remote files for changes, ' +
                            str(fileC) + ' files scanned.')

                    # STEP: IS THIS THE FIRST TIME WE SAW THE FILE, OR WAS IT ALREADY IN OUR DB?
                    just_added = not server_file.inserver

                    # STEP: IF ITS A NEW FILE, ENSURE WE DONT WANT TO SKIP IT
                    # Example: If it's a temporary file, or a Unix file with a name we don't support.

                    if just_added:
                        filename = os.path.basename(serverpath)

                        if platform.system() == 'Windows':

                            badName = False
                            for chr in [
                                    '\\', '/', ':', '?', '"', '<', '>', '|'
                            ]:
                                if chr in filename:
                                    badName = True
                                    break
                            if badName:
                                if filename not in self.warnedNames:
                                    self.warnedNames.append(filename)
                                    self.badFilenameFound.emit(filename)
                                continue

                    # STEP: ASSUMING THE FILE DID EXIST IN OUR DB, LETS SAVE THE LAST MODIFICATION DATE
                    lastmdate = server_file.servermdate

                    # STEP: SAVE THE MOD DATE TO A VARIABLE
                    # Now we get the last mod time.
                    # We expect this to work fine since this file
                    # was found on the server
                    servermdate = self.lastModified(serverpath)

                    # STEP: SET BOOL SHOWING THAT IT WAS ON THE SERVER, SINCE WE KNOW IT IS.
                    server_file.inserver = True

                    # STEP: SET THE TIME THE FILE WAS LAST CHECKED TO THE SCAN START TIME
                    server_file.last_checked_server = check_date

                    # STEP: SET THE MOD DATE IN THE DATABASE TO THE ONE WE JUST GOT
                    server_file.servermdate = servermdate

                    # STEP: SAVE THIS CHANGE TO THE DATABASE
                    server_file.session.commit()

                    delta = 0
                    if server_file.inlocal:
                        delta = server_file.timeDiff()

                    # Emit the signals after the attributes has been set and committed
                    if just_added is True:
                        self.fileAdded.emit(ServerWatcher.LOCATION, serverpath)
                    elif server_file.servermdate > lastmdate or delta < -Watcher.TOLERANCE:
                        self.fileChanged.emit(ServerWatcher.LOCATION,
                                              serverpath, False)
            #END FOR

            self.textStatus.emit('Remote scan- Finding next folder...')
            dir_ready = True
            for dir_ in dir_subdirs:
                # `dirpath` is the absolute path of the subdirectory on the server,
                dirpath = QDir.fromNativeSeparators(
                    os.path.join(downloading_dir, dir_))
                # `downloading_dir` is ready only when all its subdirectory are on the
                # `checked_dirs` list.
                if dirpath not in checked_dirs:
                    # Found one subdirectory that is not on `checked_dirs`,
                    # will process it in the next iteration.
                    downloading_dir = dirpath
                    dir_ready = False
                    break

            if dir_ready is True:
                # All subdirectories of `downloading_dir` are already in `checked_dirs`
                if downloading_dir == '/':
                    # All directories ready and at '/', means checkout is complete
                    # So, exit the main While loop!!
                    break

                else:
                    # Not at '/'. Current directory is ready so is appended to `checked_dirs`
                    # Back one directory to find directories that are not in `checked_dirs`
                    checked_dirs.append(downloading_dir)
                    downloading_dir = os.path.dirname(downloading_dir)

            self.textStatus.emit('Remote scan- Found Folder...')


##### END OF WHILE ################
###################################################################

# Deleted files are the ones whose `last_checked_server` attribute
# didn't get updated in the recursive run.
        session = Session()
        deleted = session.query(File).filter(
            File.last_checked_server < check_date).filter(
                File.inserver == True)
        for file_ in deleted:
            self.fileDeleted.emit(ServerWatcher.LOCATION, file_.path)

        # Wraps up the checkout process, commits to the database.
        session.commit()
Exemple #19
0
    def onChanged(self, location, serverpath, skipDeltaCheck):
        changed_file = File.fromPath(serverpath)
        action = None

        #if not changed_file.servermdate:
        # Probably a local added event that also
        # spawned a modified event.
        #return

        file_name_only = os.path.basename(serverpath)
        if engine_tools.isTemporaryFile(file_name_only):
            print 'File ' + serverpath + ' ignored since it is a temporary file'
            return

        print 'File ' + serverpath + ':'

        if changed_file.servermdate == None:
            mydiff = "** File Not in Server **"
            edit_time = "(not in server)"
        else:
            ttt = (changed_file.localmdate -
                   changed_file.servermdate).total_seconds()
            mydiff = str(ttt)
            edit_time = str(changed_file.servermdate)

        print 'Changed here %s, there %s delta %s' % (changed_file.localmdate,
                                                      edit_time, mydiff)

        try:
            if changed_file.inserver:
                diff = changed_file.timeDiff()

                MY_TOLERANCE = 10

                if skipDeltaCheck == False and abs(diff) < MY_TOLERANCE:
                    return

            if location == FileAction.SERVER:
                if changed_file.inlocal:
                    if changed_file.localmdate < changed_file.servermdate:
                        action = FileAction(serverpath, FileAction.DOWNLOAD,
                                            FileAction.LOCAL)
                else:
                    action = FileAction(serverpath, FileAction.DOWNLOAD,
                                        FileAction.LOCAL)

            elif location == FileAction.LOCAL:
                if changed_file.inserver:
                    try:
                        if changed_file.servermdate < changed_file.localmdate:
                            action = FileAction(serverpath, FileAction.UPLOAD,
                                                FileAction.SERVER)
                    except:
                        print 'Error:', changed_file, changed_file.servermdate, changed_file.localmdate

                else:
                    action = FileAction(serverpath, FileAction.UPLOAD,
                                        FileAction.SERVER)

            if action is not None:
                self.actionQueue.add(action)
        except:
            info = traceback.format_exception(*sys.exc_info())
            for i in info:
                sys.stderr.write(i)
Exemple #20
0
    def downloadFile(self, filename, localpath=None):
        """
        Performs a binary download to the file `filename` located on the server.
        `filename` parameter can be either absolute or relative, though it can
        fail for relative paths if the current directory is not appropiate.
        
        :param filename: Relative or absolute path to the file
        :param localpath: Absolute local path where the file will be saved
        """
        def handleChunk(chunk):
            """
            Receives chuncks of data downloaded from the server.
            This function is meant to be used as callback for the `retrbinary` method.
            
            :params chunk: Chunk of downloaded bytes to be written into the file
            """

            # Simply writes the received data into the file `self.downloading`
            self.downloading.write(chunk)
            self.download_progress += len(chunk)
            self.downloadProgress.emit(self.download_size,
                                       self.download_progress)

        if localpath is None:
            localpath = self.localFromServer(filename)

        localdir = os.path.dirname(localpath)
        if not os.path.exists(localdir):
            # Creates the directory if it doesn't already exists.
            os.makedirs(localdir)

        print 'Downloading: %s to %s' % (filename, localpath)
        try:
            with open(localpath, 'wb') as f:
                # Opens the file at `localname` which will hold the downloaded file.
                # Object attributes regarding download status are updated accordingly.
                self.fileEvent.emit(filename)
                self.downloading = f
                self.download_progress = 0

                self.download_size = int(
                    self.ftp.sendcmd('SIZE %s' % filename).split(' ')[-1])
                self.ftp.retrbinary('RETR %s' % filename, handleChunk)

                print 'Download finished'

                # Let's set the same modified time to that on the server.
                with File.fromPath(filename) as downloadedfile:
                    mdate = LocalWatcher.lastModified(localpath)
                    downloadedfile.localmdate = mdate
                    downloadedfile.servermdate = mdate

                self.setLastModified(filename, mdate)

                downloaded = True
        except (IOError, OSError):
            downloaded = False
            self.ioError.emit(localpath)
        except (error_reply, error_perm) as ftperr:
            print 'Error downloading %s, %s' % (filename, ftperr)
            downloaded = False

        # TODO: Sometimes the file doesn't complete properly.
        # in that case we maybe shouldn't call this?
        self.fileEventCompleted.emit()

        return downloaded
Exemple #21
0
    def downloadFile(self, filename, localpath=None):
        """
        Performs a binary download to the file `filename` located on the server.
        `filename` parameter can be either absolute or relative, though it can
        fail for relative paths if the current directory is not appropiate.
        
        :param filename: Relative or absolute path to the file
        :param localpath: Absolute local path where the file will be saved
        """
        
        def handleChunk(chunk):
            """
            Receives chuncks of data downloaded from the server.
            This function is meant to be used as callback for the `retrbinary` method.
            
            :params chunk: Chunk of downloaded bytes to be written into the file
            """
        
            # Simply writes the received data into the file `self.downloading`
            self.downloading.write(chunk)
            self.download_progress += len(chunk)
            self.downloadProgress.emit(self.download_size, self.download_progress)
        
        if localpath is None:
            localpath = self.localFromServer(filename)
        
        localdir = os.path.dirname(localpath)
        if not os.path.exists(localdir):
            # Creates the directory if it doesn't already exists.
            os.makedirs(localdir)
        
        print 'Downloading: %s to %s' % (filename, localpath) 
        try:
            with open(localpath, 'wb') as f:
                # Opens the file at `localname` which will hold the downloaded file.
                # Object attributes regarding download status are updated accordingly.
                self.fileEvent.emit(filename)
                self.downloading = f
                self.download_progress = 0

                self.download_size = int(self.ftp.sendcmd('SIZE %s' % filename).split(' ')[-1])
                self.ftp.retrbinary('RETR %s' % filename, handleChunk)
                
                print 'Download finished'
                
                # Let's set the same modified time to that on the server.
                with File.fromPath(filename) as downloadedfile:
                    mdate = LocalWatcher.lastModified(localpath)
                    downloadedfile.localmdate = mdate
                    downloadedfile.servermdate = mdate
                    
                self.setLastModified(filename, mdate)

                downloaded = True
        except (IOError, OSError):
            downloaded = False
            self.ioError.emit(localpath)
        except (error_reply, error_perm) as ftperr:
            print 'Error downloading %s, %s' % (filename, ftperr)
            downloaded = False
        
        # TODO: Sometimes the file doesn't complete properly.
        # in that case we maybe shouldn't call this?
        self.fileEventCompleted.emit()
        
        return downloaded
Exemple #22
0
 def deleted(self, location, serverpath):
     super(LocalWatcher, self).deleted(location, serverpath)
     with File.fromPath(serverpath) as deleted:
         deleted.inlocal = False
Exemple #23
0
    def takeAction(self):
        self.actionTimer.stop()

        if self.doPreemptive:
            # Preemptive check is a bit of a workaround to deal with
            # initial unexpected conditions: database file is gone
            self.doPreemptive = False
            self.server.preemptiveCheck = True
            self.local.fileAdded.connect(self.server.added)
            self.local.checkout()
            self.server.checkout()
            self.local.fileAdded.disconnect(self.server.added)
            self.server.preemptiveCheck = False
            for action in self.server.preemptiveActions:
                self.actionQueue.add(action)
        
        # After preemptive check, it is safe to do the connections
        # for normal operations
        self.connections()

        serverActionCount = 0
        localActionCount = 0
        for action in self.actionQueue:
            if action is not None:
                print 'Next action: %s' % action 
                path = action.path
                do = action.action
                location = action.location
                
                if location == FileAction.LOCAL and (do == FileAction.UPLOAD \
                   or do == FileAction.DELETE):
                    if not engine_tools.file_exists_local(path):
                        # File no longer exists at the time of processing.
                        # Maybe it was temporary or a quick rename.
                        # So we ignore it
                        print "Ignored action on " + path + ": File doesn't exist on local."
                        continue
                    
                
                if do == FileAction.UPLOAD:
                    self.uploadFile.emit(path)
                    localActionCount += 1
                elif do == FileAction.DOWNLOAD:
                    self.downloadFile.emit(path)
                    serverActionCount += 1
                elif do == FileAction.DELETE:
                    with File.fromPath(path) as deleted_file:
                        # `action.location` attribute only makes sense when deciding
                        # whether to delete a file on the server or local.
                        if location == FileAction.LOCAL:
                            localpath = self.local.localFromServer(path)
                            self.deleteLocalFile.emit(localpath)
                            deleted_file.inlocal = False
                            localActionCount += 1

                        elif location == FileAction.SERVER:
                            self.deleteServerFile.emit(path)
                            deleted_file.inserver = False
                            serverActionCount += 1
        
        self.actionQueue.clear()
        
        # Scan server for file changes
        self.statusChanged.emit('Scanning remote files for changes')
        self.server.checkout()
        
        if self.firstScan:
            # First do a full scan to check for offline changes.
            # From there we will rely on real time notifications watchdog.
            self.firstScan = False
            self.statusChanged.emit('Scanning local files for changes')
            self.local.checkout()
            self.local.startObserver()
            # Si Added
            # Since its the first scan, we should also
            # set the timer interval
            self.actionTimer.setInterval(5000)        
        self.cleanSync()

        # Si Added
        # Set check interval intelligently.
        # If there's no activity there, wait longer.
        # Since if there's just no usage, then
        # no reason to take up CPU cycles.
        tempInterval = 0
        if serverActionCount+localActionCount > 0:
            tempInterval = 5000
        else:
            tempInterval = 1000 * 10
        
        self.actionTimer.start()
Exemple #24
0
    def checkout(self):
        """
        Recursively checks out all files on the server.
        Returns a dictionary of files on the server with their last modified date.
        
        :param download: Indicates whether or not the files should be downloaded
        """
        
        # Check  `self.deleteQueue`, `self.uploadQueue` and `self.downloadQueue` queues.
        # These tasks are done in queues to make sure all FTP commands
        # are done sequentially, in the same thread.
        self.deleteAll()
        self.uploadAll()
        self.downloadAll()
        
        # Handy list to keep track of the checkout process.
        # This list contain absolute paths only.
        checked_dirs = list()

        # Sets '/' as initial directory and initializes `downloading_dir`
        self.ftp.cwd('/')
        downloading_dir = self.currentdir
        check_date = dt.utcnow()

        sidirlist = list()

        root_cached = False
        
        fileC = 0
        while True:
            # Gets the list of sub directories and files inside the 
            # current directory `downloading_dir`.
            self.textStatus.emit('Remote scan- Downloading folder list of '+downloading_dir+'...')
            
            if root_cached and downloading_dir == '/':
                dir_subdirs = saved_root_dirs
                dirfiles = saved_root_files
            else:
                
                dir_subdirs = self.getDirs(downloading_dir)
            
                if downloading_dir == '/':
                    saved_root_dirs = dir_subdirs
    
                # sidirlist.extend(dir_subdirs)
                self.textStatus.emit('Remote scan- Downloading files list of '+downloading_dir+'...')            
                dirfiles = self.getFiles(downloading_dir)
     
                if downloading_dir == '/':
                    saved_root_files = dirfiles
                    root_cached = True
                
           
            # Leading '/' in `downloading_dir` breaks the `os.path.join` call
            localdir = os.path.join(self.localdir, downloading_dir[1:])
            if not os.path.exists(localdir):
                # Creates the directory if it doesn't already exists.
                os.makedirs(localdir)
            
            for file_ in dirfiles:
                                
                # `serverpath` is the absolute path of the file on the server,
                # download it only if it hasn't been already downloaded
                serverpath = os.path.join(downloading_dir, file_)
                serverpath = QDir.fromNativeSeparators(serverpath)
                server_file = File.fromPath(serverpath)

                self.textStatus.emit('Scanning remote file... '+serverpath+'...')

                # How do we know if we should check this server file?
                # We see if the date last checked is the check start time.
                if server_file.last_checked_server != check_date:
                    
                    # Do this process only once per file
    
                    # Added by Simon
                    # Give feedback on scanning of files.
                    fileC += 1
                    if fileC % 1 == 2:
                        self.textStatus.emit('Scanning remote files for changes, '+str(fileC)+' files scanned.')
                        
                    
                    # STEP: IS THIS THE FIRST TIME WE SAW THE FILE, OR WAS IT ALREADY IN OUR DB?
                    just_added = not server_file.inserver

                    # STEP: IF ITS A NEW FILE, ENSURE WE DONT WANT TO SKIP IT
                    # Example: If it's a temporary file, or a Unix file with a name we don't support.
                    
                    if just_added:    
                        filename = os.path.basename(serverpath)
     
                        if platform.system() == 'Windows':
                            
                            badName = False
                            for chr in ['\\', '/', ':', '?', '"', '<', '>', '|']:
                                if chr in filename:
                                    badName = True
                                    break
                            if badName:
                                if filename not in self.warnedNames:
                                    self.warnedNames.append(filename)
                                    self.badFilenameFound.emit(filename)
                                continue
                        
                    
                    # STEP: ASSUMING THE FILE DID EXIST IN OUR DB, LETS SAVE THE LAST MODIFICATION DATE
                    lastmdate = server_file.servermdate
                    
                    # STEP: SAVE THE MOD DATE TO A VARIABLE
                    # Now we get the last mod time.
                    # We expect this to work fine since this file
                    # was found on the server
                    servermdate = self.lastModified(serverpath)
                    
                    # STEP: SET BOOL SHOWING THAT IT WAS ON THE SERVER, SINCE WE KNOW IT IS.
                    server_file.inserver = True
                    
                    # STEP: SET THE TIME THE FILE WAS LAST CHECKED TO THE SCAN START TIME
                    server_file.last_checked_server = check_date
                    
                    # STEP: SET THE MOD DATE IN THE DATABASE TO THE ONE WE JUST GOT
                    server_file.servermdate = servermdate
                    
                    # STEP: SAVE THIS CHANGE TO THE DATABASE
                    server_file.session.commit()
                    
                    delta = 0
                    if server_file.inlocal:
                        delta = server_file.timeDiff()

                    # Emit the signals after the attributes has been set and committed
                    if just_added is True:
                        self.fileAdded.emit(ServerWatcher.LOCATION, serverpath)
                    elif server_file.servermdate > lastmdate or delta < -Watcher.TOLERANCE:
                        self.fileChanged.emit(ServerWatcher.LOCATION, serverpath, False) 
            #END FOR            
            
            self.textStatus.emit('Remote scan- Finding next folder...')
            dir_ready = True
            for dir_ in dir_subdirs:
                # `dirpath` is the absolute path of the subdirectory on the server,
                dirpath = QDir.fromNativeSeparators(os.path.join(downloading_dir, dir_))
                # `downloading_dir` is ready only when all its subdirectory are on the 
                # `checked_dirs` list.
                if dirpath not in checked_dirs:
                    # Found one subdirectory that is not on `checked_dirs`,
                    # will process it in the next iteration.
                    downloading_dir = dirpath
                    dir_ready = False
                    break
                    
            if dir_ready is True:
                # All subdirectories of `downloading_dir` are already in `checked_dirs`
                if downloading_dir == '/':
                    # All directories ready and at '/', means checkout is complete
                    # So, exit the main While loop!!
                    break
                    
                else:
                    # Not at '/'. Current directory is ready so is appended to `checked_dirs`
                    # Back one directory to find directories that are not in `checked_dirs`
                    checked_dirs.append(downloading_dir)
                    downloading_dir = os.path.dirname(downloading_dir)
            
            self.textStatus.emit('Remote scan- Found Folder...')
                    
##### END OF WHILE ################
###################################################################
        
        # Deleted files are the ones whose `last_checked_server` attribute 
        # didn't get updated in the recursive run.
        session = Session()
        deleted = session.query(File).filter(File.last_checked_server < check_date).filter(File.inserver == True)
        for file_ in deleted:
            self.fileDeleted.emit(ServerWatcher.LOCATION, file_.path)
        
        # Wraps up the checkout process, commits to the database.
        session.commit()