Ejemplo n.º 1
0
    def __call__(self, segment):
        # TODO :: Move arbitray path construction to StorageLayout Object
        url = '{0}/wal_{1}/{2}.lzo'.format(self.layout.prefix.rstrip('/'),
                                           storage.CURRENT_VERSION,
                                           segment.name)

        logger.info(msg='begin archiving a file',
                    detail=('Uploading "{wal_path}" to "{url}".'
                            .format(wal_path=segment.path, url=url)),
                    structured={'action': 'push-wal',
                                'key': url,
                                'seg': segment.name,
                                'prefix': self.layout.path_prefix,
                                'state': 'begin'})

        # Upload and record the rate at which it happened.
        kib_per_second = do_lzop_put(self.creds, url, segment.path,
                                     self.gpg_key_id)

        logger.info(msg='completed archiving to a file ',
                    detail=('Archiving to "{url}" complete at '
                            '{kib_per_second}KiB/s. '
                            .format(url=url, kib_per_second=kib_per_second)),
                    structured={'action': 'push-wal',
                                'key': url,
                                'rate': kib_per_second,
                                'seg': segment.name,
                                'prefix': self.layout.path_prefix,
                                'state': 'complete'})

        return segment
Ejemplo n.º 2
0
    def __call__(self, segment):
        # TODO :: Move arbitray path construction to StorageLayout Object
        url = '{0}/wal_{1}/{2}.lzo'.format(self.layout.prefix.rstrip('/'),
                                           storage.CURRENT_VERSION,
                                           segment.name)

        logger.info(msg='begin archiving a file',
                    detail=('Uploading "{wal_path}" to "{url}".'
                            .format(wal_path=segment.path, url=url)),
                    structured={'action': 'push-wal',
                                'key': url,
                                'seg': segment.name,
                                'prefix': self.layout.path_prefix,
                                'state': 'begin'})

        # Upload and record the rate at which it happened.
        kib_per_second = do_lzop_put(self.creds, url, segment.path,
                                     self.gpg_key_id)

        logger.info(msg='completed archiving to a file ',
                    detail=('Archiving to "{url}" complete at '
                            '{kib_per_second}KiB/s. '
                            .format(url=url, kib_per_second=kib_per_second)),
                    structured={'action': 'push-wal',
                                'key': url,
                                'rate': kib_per_second,
                                'seg': segment.name,
                                'prefix': self.layout.path_prefix,
                                'state': 'complete'})

        return segment
Ejemplo n.º 3
0
    def __call__(self, segment):
        # TODO :: Move arbitray path construction to StorageLayout Object
        url = '{0}/wal_{1}/{2}.lzo'.format(self.layout.prefix.rstrip('/'),
                                           storage.CURRENT_VERSION,
                                           segment.name)

        logger.info(msg='begin archiving a file',
                    detail=('Uploading "{wal_path}" to "{url}".'.format(
                        wal_path=segment.path, url=url)),
                    structured={
                        'action': 'push-wal',
                        'key': url,
                        'seg': segment.name,
                        'prefix': self.layout.path_prefix,
                        'state': 'begin'
                    })

        structured_template = {
            'action': 'push-wal',
            'key': url,
            'seg': segment.name,
            'prefix': self.layout.path_prefix
        }

        try:
            # Upload and record the rate at which it happened.
            kib_per_second = do_lzop_put(self.creds, url, segment.path,
                                         self.gpg_key_id)
        except EnvironmentError as e:
            if not segment.explicit and e.errno == errno.ENOENT:
                structured = dict(state='skip', **structured_template)
                logger.info(msg='skip parallel archiving of a file',
                            detail=('The segment {0} did not exist.'.format(
                                segment.path)),
                            structured=structured)
            else:
                raise
        else:
            structured = dict(rate=str(kib_per_second),
                              state='complete',
                              **structured_template)
            logger.info(msg='completed archiving to a file',
                        detail=('Archiving to "{url}" complete at '
                                '{kib_per_second}KiB/s.'.format(
                                    url=url, kib_per_second=kib_per_second)),
                        structured=structured)

        return segment
Ejemplo n.º 4
0
    def __call__(self, segment):
        # TODO :: Move arbitray path construction to StorageLayout Object
        url = '{0}/wal_{1}/{2}.lzo'.format(self.layout.prefix.rstrip('/'),
                                           storage.CURRENT_VERSION,
                                           segment.name)

        logger.info(msg='begin archiving a file',
                    detail=('Uploading "{wal_path}" to "{url}".'
                            .format(wal_path=segment.path, url=url)),
                    structured={'action': 'push-wal',
                                'key': url,
                                'seg': segment.name,
                                'prefix': self.layout.path_prefix,
                                'state': 'begin'})

        structured_template = {'action': 'push-wal',
                               'key': url,
                               'seg': segment.name,
                               'prefix': self.layout.path_prefix}

        try:
            # Upload and record the rate at which it happened.
            kib_per_second = do_lzop_put(self.creds, url, segment.path,
                                         self.gpg_key_id)
        except EnvironmentError as e:
            if not segment.explicit and e.errno == errno.ENOENT:
                structured = dict(state='skip', **structured_template)
                logger.info(msg='skip parallel archiving of a file',
                            detail=('The segment {0} did not exist.'
                                    .format(segment.path)),
                            structured=structured)
            else:
                raise
        else:
            structured = dict(rate=str(kib_per_second), state='complete',
                              **structured_template)
            logger.info(msg='completed archiving to a file',
                        detail=('Archiving to "{url}" complete at '
                                '{kib_per_second}KiB/s.'
                                .format(url=url,
                                        kib_per_second=kib_per_second)),
                        structured=structured)

        return segment
Ejemplo n.º 5
0
    def __call__(self, segment):
        # TODO :: Move arbitray path construction to StorageLayout Object
        url = "{0}/wal_{1}/{2}.lzo".format(self.layout.prefix.rstrip("/"), storage.CURRENT_VERSION, segment.name)

        logger.info(
            msg="begin archiving a file",
            detail=('Uploading "{wal_path}" to "{url}".'.format(wal_path=segment.path, url=url)),
            structured={
                "action": "push-wal",
                "key": url,
                "seg": segment.name,
                "prefix": self.layout.path_prefix,
                "state": "begin",
            },
        )

        # Upload and record the rate at which it happened.
        kib_per_second = do_lzop_put(self.creds, url, segment.path, self.gpg_key_id)

        logger.info(
            msg="completed archiving to a file ",
            detail=(
                'Archiving to "{url}" complete at '
                "{kib_per_second}KiB/s. ".format(url=url, kib_per_second=kib_per_second)
            ),
            structured={
                "action": "push-wal",
                "key": url,
                "rate": kib_per_second,
                "seg": segment.name,
                "prefix": self.layout.path_prefix,
                "state": "complete",
            },
        )

        return segment