Ejemplo n.º 1
0
    def _update_file_index(self, file_path, changes):
        relative_path = pathext.normalize(
            os.path.relpath(file_path, self.dir_path)
        )
        file_data = self._index.setdefault(relative_path, dict())

        if not file_data:
            # New file
            file_hash = bintools.hash_file(file_path)
            file_data['last_update'] = int(os.path.getmtime(file_path))
            file_data['hash'] = file_hash
            file_data['last_update_location'] = self.uuid

            sync_log = file_data.setdefault('sync_log', dict())
            sync_log[self.uuid] = file_data['last_update']

            changes.add(relative_path)

        elif int(os.path.getmtime(file_path)) > file_data['last_update']:
            # Check if file is actually changed or the system time is off
            file_hash = bintools.hash_file(file_path)

            if file_data['hash'] != file_hash:
                # File modified locally (since last sync)
                file_data['last_update'] = int(os.path.getmtime(file_path))
                file_data['hash'] = file_hash
                file_data['last_update_location'] = self.uuid

                sync_log = file_data.setdefault('sync_log', dict())
                sync_log[self.uuid] = file_data['last_update']

                changes.add(relative_path)

        if 'deleted' in file_data:
            file_data['last_update'] = datetime.now().timestamp()
            file_data['hash'] = bintools.hash_file(file_path)
            file_data['last_update_location'] = self.uuid

            sync_log = file_data.setdefault('sync_log', dict())
            sync_log[self.uuid] = file_data['last_update']

            changes.add(relative_path)

            del file_data['deleted']

        if 'not_found' in file_data:
            del file_data['not_found']
Ejemplo n.º 2
0
    def _update_file_index(self, file_path, changes):
        relative_path = pathext.normalize(
            os.path.relpath(file_path, self.dir_path))
        file_data = self._index.setdefault(relative_path, dict())

        if not file_data:
            # New file
            file_hash = bintools.hash_file(file_path)
            file_data['last_update'] = int(os.path.getmtime(file_path))
            file_data['hash'] = file_hash
            file_data['last_update_location'] = self.uuid

            sync_log = file_data.setdefault('sync_log', dict())
            sync_log[self.uuid] = file_data['last_update']

            changes.add(relative_path)

        elif int(os.path.getmtime(file_path)) > file_data['last_update']:
            # Check if file is actually changed or the system time is off
            file_hash = bintools.hash_file(file_path)

            if file_data['hash'] != file_hash:
                # File modified locally (since last sync)
                file_data['last_update'] = int(os.path.getmtime(file_path))
                file_data['hash'] = file_hash
                file_data['last_update_location'] = self.uuid

                sync_log = file_data.setdefault('sync_log', dict())
                sync_log[self.uuid] = file_data['last_update']

                changes.add(relative_path)

        if 'deleted' in file_data:
            file_data['last_update'] = datetime.now().timestamp()
            file_data['hash'] = bintools.hash_file(file_path)
            file_data['last_update_location'] = self.uuid

            sync_log = file_data.setdefault('sync_log', dict())
            sync_log[self.uuid] = file_data['last_update']

            changes.add(relative_path)

            del file_data['deleted']

        if 'not_found' in file_data:
            del file_data['not_found']
Ejemplo n.º 3
0
    def update_index(self, save_index=True, force=False):
        """
        Update self._index (use the get_index() method to get it).

        Return True if index changed, False otherwise.

        The index structure is:
            <index> ::= {
                <file_name>: <file_info>,
                ...
            }
            <file_name> ::= file path relative to directory top
            <file_info> ::= {
                'sync_log': {
                    <remote_uuid (as string)>: <timestamp>,
                    ...
                },
                'last_update_location': <remote_uuid (or the local UUID) (str)>
                'last_update': <timestamp>,
                'hash': <md5 byte-string>,
                [optional 'deleted': (True|False)]
            }
            <timestamp> ::= Datetime in unix timestamp (seconds).
                            Depends on the os time on the system on which
                            the change happened.
        """
        changes = set()

        with self.fs_access_lock:
            for file_data in self._index.values():
                file_data['not_found'] = True

            for dirpath, dirnames, filenames in os.walk(self.dir_path):
                for name in filenames:
                    file_path = pathext.normalize(os.path.join(dirpath, name))

                    if not re.search(self.IGNORE_PATTERNS, file_path):
                        self._update_file_index(file_path, changes)

            # Mark each deleted file with the current timestamp
            # and UUID to avoid conflicts and to propagate properly
            timestamp = datetime.now().timestamp()
            for file_name, file_data in self._index.items():
                if 'not_found' in file_data:
                    del file_data['not_found']

                    if 'deleted' in file_data and file_data['deleted']:
                        # File has been deleted some time ago...
                        continue

                    # File has been deleted now
                    file_data['deleted'] = True
                    file_data['last_update'] = timestamp
                    file_data['last_update_location'] = self.uuid
                    file_data['hash'] = b''

                    sync_log = file_data.setdefault('sync_log', dict())
                    sync_log[self.uuid] = timestamp

                    changes.add(file_name)

            if changes:
                self.last_update = datetime.now().timestamp()

        if save_index and changes:
            self.save_index()

        if force:
            self.index_updated.notify(None)
        elif changes:
            self.index_updated.notify(changes)
Ejemplo n.º 4
0
    def update_index(self, save_index=True, force=False):
        """
        Update self._index (use the get_index() method to get it).

        Return True if index changed, False otherwise.

        The index structure is:
            <index> ::= {
                <file_name>: <file_info>,
                ...
            }
            <file_name> ::= file path relative to directory top
            <file_info> ::= {
                'sync_log': {
                    <remote_uuid (as string)>: <timestamp>,
                    ...
                },
                'last_update_location': <remote_uuid (or the local UUID) (str)>
                'last_update': <timestamp>,
                'hash': <md5 byte-string>,
                [optional 'deleted': (True|False)]
            }
            <timestamp> ::= Datetime in unix timestamp (seconds).
                            Depends on the os time on the system on which
                            the change happened.
        """
        changes = set()

        with self.fs_access_lock:
            for file_data in self._index.values():
                file_data['not_found'] = True

            for dirpath, dirnames, filenames in os.walk(self.dir_path):
                for name in filenames:
                    file_path = pathext.normalize(os.path.join(dirpath, name))

                    if not re.search(self.IGNORE_PATTERNS, file_path):
                        self._update_file_index(file_path, changes)

            # Mark each deleted file with the current timestamp
            # and UUID to avoid conflicts and to propagate properly
            timestamp = datetime.now().timestamp()
            for file_name, file_data in self._index.items():
                if 'not_found' in file_data:
                    del file_data['not_found']

                    if 'deleted' in file_data and file_data['deleted']:
                        # File has been deleted some time ago...
                        continue

                    # File has been deleted now
                    file_data['deleted'] = True
                    file_data['last_update'] = timestamp
                    file_data['last_update_location'] = self.uuid
                    file_data['hash'] = b''

                    sync_log = file_data.setdefault('sync_log', dict())
                    sync_log[self.uuid] = timestamp

                    changes.add(file_name)

            if changes:
                self.last_update = datetime.now().timestamp()

        if save_index and changes:
            self.save_index()

        if force:
            self.index_updated.notify(None)
        elif changes:
            self.index_updated.notify(changes)