Beispiel #1
0
 def cleanSync(self):
     """
     Removes entries from the database for deleted files
     """
     
     session = Session()
     session.query(File).filter(File.inserver == False).filter(File.inlocal == False).delete(synchronize_session=False)
     session.commit()
     self.statusChanged.emit('Sync completed. Waiting for changes')
Beispiel #2
0
    def cleanSync(self):
        """
        Removes entries from the database for deleted files
        """

        session = Session()
        session.query(File).filter(File.inserver == False).filter(
            File.inlocal == False).delete(synchronize_session=False)
        session.commit()
        self.statusChanged.emit('Sync completed. Waiting for changes')
Beispiel #3
0
    def checkout(self):
        check_date = dt.utcnow()
        fileC = 0
        for item in os.walk(self.localdir):
            directory = item[0]
            subfiles = item[-1]

            for file_ in subfiles:

                # Added by Simon
                # Give feedback on scanning of files.
                fileC += 1
                if fileC % 100 == 0:
                    self.textStatus.emit('Scanning local files for changes, ' +
                                         str(fileC) + ' scanned.')
                    # time.sleep(0.1)

                localpath = os.path.join(directory, file_)
                localmdate = LocalWatcher.lastModified(localpath)
                serverpath = self.serverFromLocal(localpath)

                flagg = 0
                if localpath.endswith("local.conf"):
                    flagg = 1
                    print "Found local.conf"

                file_is_in_server = False

                with File.fromPath(serverpath) as local_file:
                    # If the file is not in the local DB,
                    # then it's new- we're just adding it now
                    just_added = not local_file.inlocal
                    print "JUST ADDED: " + str(just_added)

                    lastmdate = local_file.localmdate
                    print "LASTMDATE: " + str(lastmdate)

                    # Update values in the DB for this file
                    local_file.inlocal = True
                    local_file.last_checked_local = check_date
                    local_file.localmdate = localmdate
                    # Done updating values

                    delta = 0
                    file_is_in_server = local_file.inserver
                    if local_file.inserver:
                        delta = local_file.timeDiff()

                # Emit the signals after the attributes has been set
                # and committed.
                if just_added is True:
                    self.fileAdded.emit(LocalWatcher.LOCATION, serverpath)
                elif localmdate > lastmdate or delta > Watcher.TOLERANCE \
                     or not file_is_in_server:
                    print "FC_EMIT"
                    self.fileChanged.emit(LocalWatcher.LOCATION, serverpath,
                                          True)

        # Deleted files are the ones whose `last_checked_local` attribute
        # didn't get updated in the recursive run.
        session = Session()
        deleted = session.query(File).filter(
            File.last_checked_local < check_date).filter(File.inlocal == True)
        for file_ in deleted:
            self.fileDeleted.emit(LocalWatcher.LOCATION, file_.path)

        session.commit()
Beispiel #4
0
    def checkout(self):
        """
        Recursively checks out all files on the server.
        Returns a dictionary of files on the server with their last modified date.
        
        :param download: Indicates whether or not the files should be downloaded
        """

        # Check  `self.deleteQueue`, `self.uploadQueue` and `self.downloadQueue` queues.
        # These tasks are done in queues to make sure all FTP commands
        # are done sequentially, in the same thread.
        self.deleteAll()
        self.uploadAll()
        self.downloadAll()

        # Handy list to keep track of the checkout process.
        # This list contain absolute paths only.
        checked_dirs = list()

        # Sets '/' as initial directory and initializes `downloading_dir`
        self.ftp.cwd('/')
        downloading_dir = self.currentdir
        check_date = dt.utcnow()

        sidirlist = list()

        root_cached = False

        fileC = 0
        while True:
            # Gets the list of sub directories and files inside the
            # current directory `downloading_dir`.
            self.textStatus.emit('Remote scan- Downloading folder list of ' +
                                 downloading_dir + '...')

            if root_cached and downloading_dir == '/':
                dir_subdirs = saved_root_dirs
                dirfiles = saved_root_files
            else:

                dir_subdirs = self.getDirs(downloading_dir)

                if downloading_dir == '/':
                    saved_root_dirs = dir_subdirs

                # sidirlist.extend(dir_subdirs)
                self.textStatus.emit(
                    'Remote scan- Downloading files list of ' +
                    downloading_dir + '...')
                dirfiles = self.getFiles(downloading_dir)

                if downloading_dir == '/':
                    saved_root_files = dirfiles
                    root_cached = True

            # Leading '/' in `downloading_dir` breaks the `os.path.join` call
            localdir = os.path.join(self.localdir, downloading_dir[1:])
            if not os.path.exists(localdir):
                # Creates the directory if it doesn't already exists.
                os.makedirs(localdir)

            for file_ in dirfiles:

                # `serverpath` is the absolute path of the file on the server,
                # download it only if it hasn't been already downloaded
                serverpath = os.path.join(downloading_dir, file_)
                serverpath = QDir.fromNativeSeparators(serverpath)
                server_file = File.fromPath(serverpath)

                self.textStatus.emit('Scanning remote file... ' + serverpath +
                                     '...')

                # How do we know if we should check this server file?
                # We see if the date last checked is the check start time.
                if server_file.last_checked_server != check_date:

                    # Do this process only once per file

                    # Added by Simon
                    # Give feedback on scanning of files.
                    fileC += 1
                    if fileC % 1 == 2:
                        self.textStatus.emit(
                            'Scanning remote files for changes, ' +
                            str(fileC) + ' files scanned.')

                    # STEP: IS THIS THE FIRST TIME WE SAW THE FILE, OR WAS IT ALREADY IN OUR DB?
                    just_added = not server_file.inserver

                    # STEP: IF ITS A NEW FILE, ENSURE WE DONT WANT TO SKIP IT
                    # Example: If it's a temporary file, or a Unix file with a name we don't support.

                    if just_added:
                        filename = os.path.basename(serverpath)

                        if platform.system() == 'Windows':

                            badName = False
                            for chr in [
                                    '\\', '/', ':', '?', '"', '<', '>', '|'
                            ]:
                                if chr in filename:
                                    badName = True
                                    break
                            if badName:
                                if filename not in self.warnedNames:
                                    self.warnedNames.append(filename)
                                    self.badFilenameFound.emit(filename)
                                continue

                    # STEP: ASSUMING THE FILE DID EXIST IN OUR DB, LETS SAVE THE LAST MODIFICATION DATE
                    lastmdate = server_file.servermdate

                    # STEP: SAVE THE MOD DATE TO A VARIABLE
                    # Now we get the last mod time.
                    # We expect this to work fine since this file
                    # was found on the server
                    servermdate = self.lastModified(serverpath)

                    # STEP: SET BOOL SHOWING THAT IT WAS ON THE SERVER, SINCE WE KNOW IT IS.
                    server_file.inserver = True

                    # STEP: SET THE TIME THE FILE WAS LAST CHECKED TO THE SCAN START TIME
                    server_file.last_checked_server = check_date

                    # STEP: SET THE MOD DATE IN THE DATABASE TO THE ONE WE JUST GOT
                    server_file.servermdate = servermdate

                    # STEP: SAVE THIS CHANGE TO THE DATABASE
                    server_file.session.commit()

                    delta = 0
                    if server_file.inlocal:
                        delta = server_file.timeDiff()

                    # Emit the signals after the attributes has been set and committed
                    if just_added is True:
                        self.fileAdded.emit(ServerWatcher.LOCATION, serverpath)
                    elif server_file.servermdate > lastmdate or delta < -Watcher.TOLERANCE:
                        self.fileChanged.emit(ServerWatcher.LOCATION,
                                              serverpath, False)
            #END FOR

            self.textStatus.emit('Remote scan- Finding next folder...')
            dir_ready = True
            for dir_ in dir_subdirs:
                # `dirpath` is the absolute path of the subdirectory on the server,
                dirpath = QDir.fromNativeSeparators(
                    os.path.join(downloading_dir, dir_))
                # `downloading_dir` is ready only when all its subdirectory are on the
                # `checked_dirs` list.
                if dirpath not in checked_dirs:
                    # Found one subdirectory that is not on `checked_dirs`,
                    # will process it in the next iteration.
                    downloading_dir = dirpath
                    dir_ready = False
                    break

            if dir_ready is True:
                # All subdirectories of `downloading_dir` are already in `checked_dirs`
                if downloading_dir == '/':
                    # All directories ready and at '/', means checkout is complete
                    # So, exit the main While loop!!
                    break

                else:
                    # Not at '/'. Current directory is ready so is appended to `checked_dirs`
                    # Back one directory to find directories that are not in `checked_dirs`
                    checked_dirs.append(downloading_dir)
                    downloading_dir = os.path.dirname(downloading_dir)

            self.textStatus.emit('Remote scan- Found Folder...')


##### END OF WHILE ################
###################################################################

# Deleted files are the ones whose `last_checked_server` attribute
# didn't get updated in the recursive run.
        session = Session()
        deleted = session.query(File).filter(
            File.last_checked_server < check_date).filter(
                File.inserver == True)
        for file_ in deleted:
            self.fileDeleted.emit(ServerWatcher.LOCATION, file_.path)

        # Wraps up the checkout process, commits to the database.
        session.commit()
Beispiel #5
0
    def checkout(self):
        check_date = dt.utcnow()
        fileC = 0
        for item in os.walk(self.localdir):
            directory = item[0]
            subfiles = item[-1]

            for file_ in subfiles:

                # Added by Simon
                # Give feedback on scanning of files.
                fileC += 1
                if fileC % 100 == 0:
                    self.textStatus.emit('Scanning local files for changes, '+str(fileC)+' scanned.')
                    # time.sleep(0.1)
                
                localpath = os.path.join(directory, file_)
                localmdate = LocalWatcher.lastModified(localpath)
                serverpath = self.serverFromLocal(localpath)

                flagg = 0
                if localpath.endswith("local.conf"):
                    flagg = 1
                    print "Found local.conf"

                file_is_in_server = False
                
                with File.fromPath(serverpath) as local_file:
                    # If the file is not in the local DB,
                    # then it's new- we're just adding it now
                    just_added = not local_file.inlocal                        
                    print "JUST ADDED: " + str(just_added)
                    
                    lastmdate = local_file.localmdate
                    print "LASTMDATE: " + str (lastmdate)
                    
                    # Update values in the DB for this file
                    local_file.inlocal = True
                    local_file.last_checked_local = check_date
                    local_file.localmdate = localmdate
                    # Done updating values
                    
                    delta = 0
                    file_is_in_server = local_file.inserver                    
                    if local_file.inserver:
                        delta = local_file.timeDiff()
                    
                # Emit the signals after the attributes has been set
                # and committed.
                if just_added is True:
                    self.fileAdded.emit(LocalWatcher.LOCATION, serverpath)
                elif localmdate > lastmdate or delta > Watcher.TOLERANCE \
                     or not file_is_in_server:
                    print "FC_EMIT"
                    self.fileChanged.emit(LocalWatcher.LOCATION, serverpath, True)

        # Deleted files are the ones whose `last_checked_local` attribute 
        # didn't get updated in the recursive run.
        session = Session()
        deleted = session.query(File).filter(File.last_checked_local < check_date).filter(File.inlocal == True)
        for file_ in deleted:
            self.fileDeleted.emit(LocalWatcher.LOCATION, file_.path)
            
        session.commit()
Beispiel #6
0
    def checkout(self):
        """
        Recursively checks out all files on the server.
        Returns a dictionary of files on the server with their last modified date.
        
        :param download: Indicates whether or not the files should be downloaded
        """
        
        # Check  `self.deleteQueue`, `self.uploadQueue` and `self.downloadQueue` queues.
        # These tasks are done in queues to make sure all FTP commands
        # are done sequentially, in the same thread.
        self.deleteAll()
        self.uploadAll()
        self.downloadAll()
        
        # Handy list to keep track of the checkout process.
        # This list contain absolute paths only.
        checked_dirs = list()

        # Sets '/' as initial directory and initializes `downloading_dir`
        self.ftp.cwd('/')
        downloading_dir = self.currentdir
        check_date = dt.utcnow()

        sidirlist = list()

        root_cached = False
        
        fileC = 0
        while True:
            # Gets the list of sub directories and files inside the 
            # current directory `downloading_dir`.
            self.textStatus.emit('Remote scan- Downloading folder list of '+downloading_dir+'...')
            
            if root_cached and downloading_dir == '/':
                dir_subdirs = saved_root_dirs
                dirfiles = saved_root_files
            else:
                
                dir_subdirs = self.getDirs(downloading_dir)
            
                if downloading_dir == '/':
                    saved_root_dirs = dir_subdirs
    
                # sidirlist.extend(dir_subdirs)
                self.textStatus.emit('Remote scan- Downloading files list of '+downloading_dir+'...')            
                dirfiles = self.getFiles(downloading_dir)
     
                if downloading_dir == '/':
                    saved_root_files = dirfiles
                    root_cached = True
                
           
            # Leading '/' in `downloading_dir` breaks the `os.path.join` call
            localdir = os.path.join(self.localdir, downloading_dir[1:])
            if not os.path.exists(localdir):
                # Creates the directory if it doesn't already exists.
                os.makedirs(localdir)
            
            for file_ in dirfiles:
                                
                # `serverpath` is the absolute path of the file on the server,
                # download it only if it hasn't been already downloaded
                serverpath = os.path.join(downloading_dir, file_)
                serverpath = QDir.fromNativeSeparators(serverpath)
                server_file = File.fromPath(serverpath)

                self.textStatus.emit('Scanning remote file... '+serverpath+'...')

                # How do we know if we should check this server file?
                # We see if the date last checked is the check start time.
                if server_file.last_checked_server != check_date:
                    
                    # Do this process only once per file
    
                    # Added by Simon
                    # Give feedback on scanning of files.
                    fileC += 1
                    if fileC % 1 == 2:
                        self.textStatus.emit('Scanning remote files for changes, '+str(fileC)+' files scanned.')
                        
                    
                    # STEP: IS THIS THE FIRST TIME WE SAW THE FILE, OR WAS IT ALREADY IN OUR DB?
                    just_added = not server_file.inserver

                    # STEP: IF ITS A NEW FILE, ENSURE WE DONT WANT TO SKIP IT
                    # Example: If it's a temporary file, or a Unix file with a name we don't support.
                    
                    if just_added:    
                        filename = os.path.basename(serverpath)
     
                        if platform.system() == 'Windows':
                            
                            badName = False
                            for chr in ['\\', '/', ':', '?', '"', '<', '>', '|']:
                                if chr in filename:
                                    badName = True
                                    break
                            if badName:
                                if filename not in self.warnedNames:
                                    self.warnedNames.append(filename)
                                    self.badFilenameFound.emit(filename)
                                continue
                        
                    
                    # STEP: ASSUMING THE FILE DID EXIST IN OUR DB, LETS SAVE THE LAST MODIFICATION DATE
                    lastmdate = server_file.servermdate
                    
                    # STEP: SAVE THE MOD DATE TO A VARIABLE
                    # Now we get the last mod time.
                    # We expect this to work fine since this file
                    # was found on the server
                    servermdate = self.lastModified(serverpath)
                    
                    # STEP: SET BOOL SHOWING THAT IT WAS ON THE SERVER, SINCE WE KNOW IT IS.
                    server_file.inserver = True
                    
                    # STEP: SET THE TIME THE FILE WAS LAST CHECKED TO THE SCAN START TIME
                    server_file.last_checked_server = check_date
                    
                    # STEP: SET THE MOD DATE IN THE DATABASE TO THE ONE WE JUST GOT
                    server_file.servermdate = servermdate
                    
                    # STEP: SAVE THIS CHANGE TO THE DATABASE
                    server_file.session.commit()
                    
                    delta = 0
                    if server_file.inlocal:
                        delta = server_file.timeDiff()

                    # Emit the signals after the attributes has been set and committed
                    if just_added is True:
                        self.fileAdded.emit(ServerWatcher.LOCATION, serverpath)
                    elif server_file.servermdate > lastmdate or delta < -Watcher.TOLERANCE:
                        self.fileChanged.emit(ServerWatcher.LOCATION, serverpath, False) 
            #END FOR            
            
            self.textStatus.emit('Remote scan- Finding next folder...')
            dir_ready = True
            for dir_ in dir_subdirs:
                # `dirpath` is the absolute path of the subdirectory on the server,
                dirpath = QDir.fromNativeSeparators(os.path.join(downloading_dir, dir_))
                # `downloading_dir` is ready only when all its subdirectory are on the 
                # `checked_dirs` list.
                if dirpath not in checked_dirs:
                    # Found one subdirectory that is not on `checked_dirs`,
                    # will process it in the next iteration.
                    downloading_dir = dirpath
                    dir_ready = False
                    break
                    
            if dir_ready is True:
                # All subdirectories of `downloading_dir` are already in `checked_dirs`
                if downloading_dir == '/':
                    # All directories ready and at '/', means checkout is complete
                    # So, exit the main While loop!!
                    break
                    
                else:
                    # Not at '/'. Current directory is ready so is appended to `checked_dirs`
                    # Back one directory to find directories that are not in `checked_dirs`
                    checked_dirs.append(downloading_dir)
                    downloading_dir = os.path.dirname(downloading_dir)
            
            self.textStatus.emit('Remote scan- Found Folder...')
                    
##### END OF WHILE ################
###################################################################
        
        # Deleted files are the ones whose `last_checked_server` attribute 
        # didn't get updated in the recursive run.
        session = Session()
        deleted = session.query(File).filter(File.last_checked_server < check_date).filter(File.inserver == True)
        for file_ in deleted:
            self.fileDeleted.emit(ServerWatcher.LOCATION, file_.path)
        
        # Wraps up the checkout process, commits to the database.
        session.commit()