Ejemplo n.º 1
0
def do_lzop_put(creds, url, local_path, gpg_key):
    """
    Compress and upload a given local path.

    :type url: string
    :param url: A (s3|wabs)://bucket/key style URL that is the destination

    :type local_path: string
    :param local_path: a path to a file to be compressed

    """
    assert url.endswith('.lzo')
    blobstore = get_blobstore(storage.StorageLayout(url))

    with tempfile.NamedTemporaryFile(mode='r+b') as tf:
        pipeline = get_upload_pipeline(
            open(local_path, 'r'), tf, gpg_key=gpg_key)
        pipeline.finish()

        tf.flush()

        clock_start = time.time()
        tf.seek(0)
        k = blobstore.uri_put_file(creds, url, tf)
        clock_finish = time.time()

        kib_per_second = format_kib_per_second(
            clock_start, clock_finish, k.size)

        return kib_per_second
Ejemplo n.º 2
0
def do_lzop_s3_put(s3_url, local_path, gpg_key):
    """
    Compress and upload a given local path.

    :type s3_url: string
    :param s3_url: A s3://bucket/key style URL that is the destination

    :type local_path: string
    :param local_path: a path to a file to be compressed

    """

    assert not s3_url.endswith('.lzo')
    s3_url += '.lzo'

    with tempfile.NamedTemporaryFile(mode='rwb') as tf:
        pipeline = get_upload_pipeline(
            open(local_path, 'r'), tf, gpg_key=gpg_key)
        pipeline.finish()

        tf.flush()

        clock_start = time.clock()
        tf.seek(0)
        k = uri_put_file(s3_url, tf)
        clock_finish = time.clock()

        kib_per_second = format_kib_per_second(clock_start, clock_finish,
                                               k.size)

        return kib_per_second
Ejemplo n.º 3
0
def do_lzop_s3_put(s3_url, local_path, gpg_key):
    """
    Compress and upload a given local path.

    :type s3_url: string
    :param s3_url: A s3://bucket/key style URL that is the destination

    :type local_path: string
    :param local_path: a path to a file to be compressed

    """

    assert not s3_url.endswith('.lzo')
    s3_url += '.lzo'

    with tempfile.NamedTemporaryFile(mode='rwb') as tf:
        pipeline = get_upload_pipeline(open(local_path, 'r'),
                                       tf,
                                       gpg_key=gpg_key)
        pipeline.finish()

        tf.flush()

        clock_start = time.clock()
        tf.seek(0)
        k = uri_put_file(s3_url, tf)
        clock_finish = time.clock()

        kib_per_second = format_kib_per_second(clock_start, clock_finish,
                                               k.size)

        return kib_per_second
Ejemplo n.º 4
0
def test_upload_download_pipeline(tmpdir, rate_limit):
    payload, payload_file = create_bogus_payload(tmpdir)

    # Upload section
    test_upload = tmpdir.join('upload')
    with open(unicode(test_upload), 'w') as upload:
        with open(unicode(payload_file)) as inp:
            pl = pipeline.get_upload_pipeline(inp,
                                              upload,
                                              rate_limit=rate_limit)
            pl.finish()

    with open(unicode(test_upload)) as completed:
        round_trip = completed.read()

    # Download section
    test_download = tmpdir.join('download')
    with open(unicode(test_upload)) as upload:
        with open(unicode(test_download), 'w') as download:
            pl = pipeline.get_download_pipeline(upload, download)
            pl.finish()

    with open(unicode(test_download)) as completed:
        round_trip = completed.read()

    assert round_trip == payload
Ejemplo n.º 5
0
def do_lzop_s3_put(s3_url, local_path, gpg_key):
    """
    Compress and upload a given local path.

    :type s3_url: string
    :param s3_url: A s3://bucket/key style URL that is the destination

    :type local_path: string
    :param local_path: a path to a file to be compressed

    """

    assert not s3_url.endswith(".lzo")
    s3_url += ".lzo"

    with tempfile.NamedTemporaryFile(mode="rwb") as tf:
        pipeline = get_upload_pipeline(file(local_path, "r"), tf, gpg_key=gpg_key)
        pipeline.finish()

        tf.flush()

        logger.info(msg="begin archiving a file", detail=('Uploading "{local_path}" to "{s3_url}".'.format(**locals())))

        clock_start = time.clock()
        tf.seek(0)
        k = uri_put_file(s3_url, tf)
        clock_finish = time.clock()

        kib_per_second = format_kib_per_second(clock_start, clock_finish, k.size)
        logger.info(
            msg="completed archiving to a file ",
            detail=('Archiving to "{s3_url}" complete at ' "{kib_per_second}KiB/s. ").format(
                s3_url=s3_url, kib_per_second=kib_per_second
            ),
        )
Ejemplo n.º 6
0
def test_upload_download_pipeline(tmpdir, rate_limit):
    payload, payload_file = create_bogus_payload(tmpdir)

    # Upload section
    test_upload = tmpdir.join('upload')
    with open(unicode(test_upload), 'w') as upload:
        with open(unicode(payload_file)) as inp:
            pl = pipeline.get_upload_pipeline(
                inp, upload, rate_limit=rate_limit)
            pl.finish()

    with open(unicode(test_upload)) as completed:
        round_trip = completed.read()

    # Download section
    test_download = tmpdir.join('download')
    with open(unicode(test_upload)) as upload:
        with open(unicode(test_download), 'w') as download:
            pl = pipeline.get_download_pipeline(upload, download)
            pl.finish()

    with open(unicode(test_download)) as completed:
        round_trip = completed.read()

    assert round_trip == payload
Ejemplo n.º 7
0
def test_upload_download_pipeline(tmpdir, rate_limit):
    payload, payload_file = create_bogus_payload(tmpdir)

    # Upload section
    test_upload = tmpdir.join('upload')
    with open(str(test_upload), 'wb') as upload:
        with open(str(payload_file), 'rb') as inp:
            with pipeline.get_upload_pipeline(inp,
                                              upload,
                                              rate_limit=rate_limit):
                pass

    with open(str(test_upload), 'rb') as completed:
        round_trip = completed.read()

    # Download section
    test_download = tmpdir.join('download')
    with open(str(test_upload), 'rb') as upload:
        with open(str(test_download), 'wb') as download:
            with pipeline.get_download_pipeline(upload, download):
                pass

    with open(str(test_download), 'rb') as completed:
        round_trip = completed.read()

    assert round_trip == payload
Ejemplo n.º 8
0
def test_upload_download_pipeline(tmpdir, rate_limit):
    payload, payload_file = create_bogus_payload(tmpdir)

    # Upload section
    test_upload = tmpdir.join('upload')
    with open(str(test_upload), 'wb') as upload:
        with open(str(payload_file), 'rb') as inp:
            with pipeline.get_upload_pipeline(
                    inp, upload, rate_limit=rate_limit):
                pass

    with open(str(test_upload), 'rb') as completed:
        round_trip = completed.read()

    # Download section
    test_download = tmpdir.join('download')
    with open(str(test_upload), 'rb') as upload:
        with open(str(test_download), 'wb') as download:
            with pipeline.get_download_pipeline(upload, download):
                pass

    with open(str(test_download), 'rb') as completed:
        round_trip = completed.read()

    assert round_trip == payload
Ejemplo n.º 9
0
def do_lzop_s3_put(s3_url, local_path, gpg_key):
    """
    Compress and upload a given local path.

    :type s3_url: string
    :param s3_url: A s3://bucket/key style URL that is the destination

    :type local_path: string
    :param local_path: a path to a file to be compressed

    """

    assert not s3_url.endswith('.lzo')
    s3_url += '.lzo'

    with tempfile.NamedTemporaryFile(mode='rwb') as tf:
        pipeline = get_upload_pipeline(file(local_path, 'r'),
                                       tf,
                                       gpg_key=gpg_key)
        pipeline.finish()

        tf.flush()

        logger.info(msg='begin archiving a file',
                    detail=('Uploading "{local_path}" to "{s3_url}".'.format(
                        **locals())))

        clock_start = time.clock()
        tf.seek(0)
        k = uri_put_file(s3_url, tf)
        clock_finish = time.clock()

        kib_per_second = format_kib_per_second(clock_start, clock_finish,
                                               k.size)
        logger.info(msg='completed archiving to a file ',
                    detail=('Archiving to "{s3_url}" complete at '
                            '{kib_per_second}KiB/s. ').format(
                                s3_url=s3_url, kib_per_second=kib_per_second))
Ejemplo n.º 10
0
    def __call__(self, tpart):
        """
        Synchronous version of the upload wrapper

        """
        logger.info(msg='beginning volume compression',
                    detail='Building volume {name}.'.format(name=tpart.name))

        with tempfile.NamedTemporaryFile(mode='r+b',
                                         bufsize=pipebuf.PIPE_BUF_BYTES) as tf:
            with pipeline.get_upload_pipeline(PIPE,
                                              tf,
                                              rate_limit=self.rate_limit,
                                              gpg_key=self.gpg_key) as pl:
                tpart.tarfile_write(pl.stdin)

            tf.flush()

            # TODO :: Move arbitray path construction to StorageLayout Object
            url = '{0}/tar_partitions/part_{number:08d}.tar.lzo'.format(
                self.backup_prefix.rstrip('/'), number=tpart.name)

            manifest_file = StringIO(tpart.format_manifest())
            manifest_url = '{0}/manifests/part_{number:08d}.json'.format(
                self.backup_prefix.rstrip('/'), number=tpart.name)

            logger.info(msg='begin uploading a base backup volume',
                        detail='Uploading to "{url}".'.format(url=url))

            def log_volume_failures_on_error(exc_tup, exc_processor_cxt):
                def standard_detail_message(prefix=''):
                    return (prefix +
                            '  There have been {n} attempts to send the '
                            'volume {name} so far.'.format(n=exc_processor_cxt,
                                                           name=tpart.name))

                typ, value, tb = exc_tup
                del exc_tup

                # Screen for certain kinds of known-errors to retry from
                if issubclass(typ, socket.error):
                    socketmsg = value[1] if isinstance(value, tuple) else value

                    logger.info(
                        msg='Retrying send because of a socket error',
                        detail=standard_detail_message(
                            "The socket error's message is '{0}'.".format(
                                socketmsg)))
                elif (issubclass(typ, boto.exception.S3ResponseError)
                      and value.error_code == 'RequestTimeTooSkewed'):
                    logger.info(
                        msg='Retrying send because of a Request Skew time',
                        detail=standard_detail_message())

                else:
                    # This type of error is unrecognized as a retry-able
                    # condition, so propagate it, original stacktrace and
                    # all.
                    raise typ, value, tb

            @retry(retry_with_count(log_volume_failures_on_error))
            def put_file_helper(_creds, _url, _file):
                _file.seek(0)
                return self.blobstore.uri_put_file(_creds, _url, _file)

            # Actually do work, retrying if necessary, and timing how long
            # it takes.
            clock_start = time.time()
            k = put_file_helper(self.creds, url, tf)
            k2 = put_file_helper(self.creds, manifest_url, manifest_file)
            clock_finish = time.time()

            kib_per_second = format_kib_per_second(clock_start, clock_finish,
                                                   k.size + k2.size)
            logger.info(msg='finish uploading a base backup volume',
                        detail=('Uploading to "{url}" complete at '
                                '{kib_per_second}KiB/s. '.format(
                                    url=url, kib_per_second=kib_per_second)))

        return tpart
Ejemplo n.º 11
0
    def __call__(self, tpart):
        """
        Synchronous version of the s3-upload wrapper

        """
        logger.info(msg='beginning volume compression',
                    detail='Building volume {name}.'.format(name=tpart.name))

        with tempfile.NamedTemporaryFile(mode='rwb') as tf:
            pipeline = get_upload_pipeline(PIPE,
                                           tf,
                                           rate_limit=self.rate_limit,
                                           gpg_key=self.gpg_key)

            tpart.tarfile_write(pipeline.stdin)
            pipeline.stdin.flush()
            pipeline.stdin.close()
            pipeline.finish()

            tf.flush()

            s3_url = '/'.join([self.backup_s3_prefix, 'tar_partitions',
                               'part_{number}.tar.lzo'
                               .format(number=tpart.name)])

            logger.info(
                msg='begin uploading a base backup volume',
                detail=('Uploading to "{s3_url}".')
                .format(s3_url=s3_url))

            def log_volume_failures_on_error(exc_tup, exc_processor_cxt):
                def standard_detail_message(prefix=''):
                    return (prefix +
                            '  There have been {n} attempts to send the '
                            'volume {name} so far.'.format(n=exc_processor_cxt,
                                                           name=tpart.name))

                typ, value, tb = exc_tup
                del exc_tup

                # Screen for certain kinds of known-errors to retry from
                if issubclass(typ, socket.error):
                    socketmsg = value[1] if isinstance(value, tuple) else value

                    logger.info(
                        msg='Retrying send because of a socket error',
                        detail=standard_detail_message(
                            "The socket error's message is '{0}'."
                            .format(socketmsg)))
                elif (issubclass(typ, boto.exception.S3ResponseError) and
                      value.error_code == 'RequestTimeTooSkewed'):
                    logger.info(
                        msg='Retrying send because of a Request Skew time',
                        detail=standard_detail_message())

                else:
                    # This type of error is unrecognized as a retry-able
                    # condition, so propagate it, original stacktrace and
                    # all.
                    raise typ, value, tb

            @retry(retry_with_count(log_volume_failures_on_error))
            def put_file_helper():
                tf.seek(0)
                return uri_put_file(s3_url, tf)

            # Actually do work, retrying if necessary, and timing how long
            # it takes.
            clock_start = time.clock()
            k = put_file_helper()
            clock_finish = time.clock()

            kib_per_second = format_kib_per_second(clock_start, clock_finish,
                                                   k.size)
            logger.info(
                msg='finish uploading a base backup volume',
                detail=('Uploading to "{s3_url}" complete at '
                        '{kib_per_second}KiB/s. ')
                .format(s3_url=s3_url, kib_per_second=kib_per_second))

        return tpart
Ejemplo n.º 12
0
def do_partition_put(backup_s3_prefix, tpart, rate_limit, gpg_key):
    """
    Synchronous version of the s3-upload wrapper

    """
    logger.info(msg='beginning volume compression',
                detail='Building volume {name}.'.format(name=tpart.name))

    with tempfile.NamedTemporaryFile(mode='rwb') as tf:
        pipeline = get_upload_pipeline(PIPE,
                                       tf,
                                       rate_limit=rate_limit,
                                       gpg_key=gpg_key)
        tpart.tarfile_write(pipeline.stdin)
        pipeline.stdin.flush()
        pipeline.stdin.close()
        pipeline.finish()

        tf.flush()

        s3_url = '/'.join([
            backup_s3_prefix, 'tar_partitions',
            'part_{number}.tar.lzo'.format(number=tpart.name)
        ])

        logger.info(msg='begin uploading a base backup volume',
                    detail=('Uploading to "{s3_url}".').format(s3_url=s3_url))

        def log_volume_failures_on_error(exc_tup, exc_processor_cxt):
            def standard_detail_message(prefix=''):
                return (prefix + '  There have been {n} attempts to send the '
                        'volume {name} so far.'.format(n=exc_processor_cxt,
                                                       name=tpart.name))

            typ, value, tb = exc_tup
            del exc_tup

            # Screen for certain kinds of known-errors to retry from
            if issubclass(typ, socket.error):
                socketmsg = value[1] if isinstance(value, tuple) else value

                logger.info(msg='Retrying send because of a socket error',
                            detail=standard_detail_message(
                                "The socket error's message is '{0}'.".format(
                                    socketmsg)))
            elif (issubclass(typ, boto.exception.S3ResponseError)
                  and value.error_code == 'RequestTimeTooSkewed'):
                logger.info(msg='Retrying send because of a Request Skew time',
                            detail=standard_detail_message())

            else:
                # This type of error is unrecognized as a retry-able
                # condition, so propagate it, original stacktrace and
                # all.
                raise typ, value, tb

        @retry(retry_with_count(log_volume_failures_on_error))
        def put_file_helper():
            tf.seek(0)
            return uri_put_file(s3_url, tf)

        # Actually do work, retrying if necessary, and timing how long
        # it takes.
        clock_start = time.clock()
        k = put_file_helper()
        clock_finish = time.clock()

        kib_per_second = format_kib_per_second(clock_start, clock_finish,
                                               k.size)
        logger.info(msg='finish uploading a base backup volume',
                    detail=('Uploading to "{s3_url}" complete at '
                            '{kib_per_second}KiB/s. ').format(
                                s3_url=s3_url, kib_per_second=kib_per_second))
Ejemplo n.º 13
0
    def __call__(self, tpart):
        """
        Synchronous version of the upload wrapper

        """
        logger.info(msg="beginning volume compression", detail="Building volume {name}.".format(name=tpart.name))

        with tempfile.NamedTemporaryFile(mode="rwb") as tf:
            pipeline = get_upload_pipeline(PIPE, tf, rate_limit=self.rate_limit, gpg_key=self.gpg_key)
            tpart.tarfile_write(pipeline.stdin)
            pipeline.stdin.flush()
            pipeline.stdin.close()
            pipeline.finish()

            tf.flush()

            # TODO :: Move arbitray path construction to StorageLayout Object
            url = "{0}/tar_partitions/part_{number}.tar.lzo".format(self.backup_prefix.rstrip("/"), number=tpart.name)

            logger.info(msg="begin uploading a base backup volume", detail='Uploading to "{url}".'.format(url=url))

            def log_volume_failures_on_error(exc_tup, exc_processor_cxt):
                def standard_detail_message(prefix=""):
                    return prefix + "  There have been {n} attempts to send the " "volume {name} so far.".format(
                        n=exc_processor_cxt, name=tpart.name
                    )

                typ, value, tb = exc_tup
                del exc_tup

                # Screen for certain kinds of known-errors to retry from
                if issubclass(typ, socket.error):
                    socketmsg = value[1] if isinstance(value, tuple) else value

                    logger.info(
                        msg="Retrying send because of a socket error",
                        detail=standard_detail_message("The socket error's message is '{0}'.".format(socketmsg)),
                    )
                elif issubclass(typ, boto.exception.S3ResponseError) and value.error_code == "RequestTimeTooSkewed":
                    logger.info(msg="Retrying send because of a Request Skew time", detail=standard_detail_message())

                else:
                    # This type of error is unrecognized as a retry-able
                    # condition, so propagate it, original stacktrace and
                    # all.
                    raise typ, value, tb

            @retry(retry_with_count(log_volume_failures_on_error))
            def put_file_helper():
                tf.seek(0)
                return self.blobstore.uri_put_file(self.creds, url, tf)

            # Actually do work, retrying if necessary, and timing how long
            # it takes.
            clock_start = time.time()
            k = put_file_helper()
            clock_finish = time.time()

            kib_per_second = format_kib_per_second(clock_start, clock_finish, k.size)
            logger.info(
                msg="finish uploading a base backup volume",
                detail=(
                    'Uploading to "{url}" complete at '
                    "{kib_per_second}KiB/s. ".format(url=url, kib_per_second=kib_per_second)
                ),
            )

        return tpart
Ejemplo n.º 14
0
    def __call__(self, tpart):
        """
        Synchronous version of the upload wrapper

        """
        logger.info(msg='beginning volume compression',
                    detail='Building volume {name}.'.format(name=tpart.name))

        with tempfile.NamedTemporaryFile(
                mode='r+b', bufsize=pipebuf.PIPE_BUF_BYTES) as tf:
            with pipeline.get_upload_pipeline(PIPE, tf,
                                              rate_limit=self.rate_limit,
                                              gpg_key=self.gpg_key) as pl:
                tpart.tarfile_write(pl.stdin)

            tf.flush()

            # TODO :: Move arbitray path construction to StorageLayout Object
            url = '{0}/tar_partitions/part_{number:08d}.tar.lzo'.format(
                self.backup_prefix.rstrip('/'), number=tpart.name)

            manifest_file = StringIO(tpart.format_manifest())
            manifest_url = '{0}/manifests/part_{number:08d}.json'.format(
                self.backup_prefix.rstrip('/'), number=tpart.name)

            logger.info(msg='begin uploading a base backup volume',
                        detail='Uploading to "{url}".'.format(url=url))

            def log_volume_failures_on_error(exc_tup, exc_processor_cxt):
                def standard_detail_message(prefix=''):
                    return (prefix +
                            '  There have been {n} attempts to send the '
                            'volume {name} so far.'.format(n=exc_processor_cxt,
                                                           name=tpart.name))

                typ, value, tb = exc_tup
                del exc_tup

                # Screen for certain kinds of known-errors to retry from
                if issubclass(typ, socket.error):
                    socketmsg = value[1] if isinstance(value, tuple) else value

                    logger.info(
                        msg='Retrying send because of a socket error',
                        detail=standard_detail_message(
                            "The socket error's message is '{0}'."
                            .format(socketmsg)))
                elif (issubclass(typ, boto.exception.S3ResponseError) and
                      value.error_code == 'RequestTimeTooSkewed'):
                    logger.info(
                        msg='Retrying send because of a Request Skew time',
                        detail=standard_detail_message())

                else:
                    # This type of error is unrecognized as a retry-able
                    # condition, so propagate it, original stacktrace and
                    # all.
                    raise typ, value, tb

            @retry(retry_with_count(log_volume_failures_on_error))
            def put_file_helper(_creds, _url, _file):
                _file.seek(0)
                return self.blobstore.uri_put_file(_creds, _url, _file)

            # Actually do work, retrying if necessary, and timing how long
            # it takes.
            clock_start = time.time()
            k = put_file_helper(self.creds, url, tf)
            k2 = put_file_helper(self.creds, manifest_url, manifest_file)
            clock_finish = time.time()

            kib_per_second = format_kib_per_second(clock_start, clock_finish,
                                                   k.size + k2.size)
            logger.info(
                msg='finish uploading a base backup volume',
                detail=('Uploading to "{url}" complete at '
                        '{kib_per_second}KiB/s. '
                        .format(url=url, kib_per_second=kib_per_second)))

        return tpart