def do_lzop_put(creds, url, local_path, gpg_key): """ Compress and upload a given local path. :type url: string :param url: A (s3|wabs)://bucket/key style URL that is the destination :type local_path: string :param local_path: a path to a file to be compressed """ assert url.endswith('.lzo') blobstore = get_blobstore(storage.StorageLayout(url)) with tempfile.NamedTemporaryFile(mode='r+b') as tf: pipeline = get_upload_pipeline( open(local_path, 'r'), tf, gpg_key=gpg_key) pipeline.finish() tf.flush() clock_start = time.time() tf.seek(0) k = blobstore.uri_put_file(creds, url, tf) clock_finish = time.time() kib_per_second = format_kib_per_second( clock_start, clock_finish, k.size) return kib_per_second
def do_lzop_s3_put(s3_url, local_path, gpg_key): """ Compress and upload a given local path. :type s3_url: string :param s3_url: A s3://bucket/key style URL that is the destination :type local_path: string :param local_path: a path to a file to be compressed """ assert not s3_url.endswith('.lzo') s3_url += '.lzo' with tempfile.NamedTemporaryFile(mode='rwb') as tf: pipeline = get_upload_pipeline( open(local_path, 'r'), tf, gpg_key=gpg_key) pipeline.finish() tf.flush() clock_start = time.clock() tf.seek(0) k = uri_put_file(s3_url, tf) clock_finish = time.clock() kib_per_second = format_kib_per_second(clock_start, clock_finish, k.size) return kib_per_second
def do_lzop_s3_put(s3_url, local_path, gpg_key): """ Compress and upload a given local path. :type s3_url: string :param s3_url: A s3://bucket/key style URL that is the destination :type local_path: string :param local_path: a path to a file to be compressed """ assert not s3_url.endswith('.lzo') s3_url += '.lzo' with tempfile.NamedTemporaryFile(mode='rwb') as tf: pipeline = get_upload_pipeline(open(local_path, 'r'), tf, gpg_key=gpg_key) pipeline.finish() tf.flush() clock_start = time.clock() tf.seek(0) k = uri_put_file(s3_url, tf) clock_finish = time.clock() kib_per_second = format_kib_per_second(clock_start, clock_finish, k.size) return kib_per_second
def test_upload_download_pipeline(tmpdir, rate_limit): payload, payload_file = create_bogus_payload(tmpdir) # Upload section test_upload = tmpdir.join('upload') with open(unicode(test_upload), 'w') as upload: with open(unicode(payload_file)) as inp: pl = pipeline.get_upload_pipeline(inp, upload, rate_limit=rate_limit) pl.finish() with open(unicode(test_upload)) as completed: round_trip = completed.read() # Download section test_download = tmpdir.join('download') with open(unicode(test_upload)) as upload: with open(unicode(test_download), 'w') as download: pl = pipeline.get_download_pipeline(upload, download) pl.finish() with open(unicode(test_download)) as completed: round_trip = completed.read() assert round_trip == payload
def do_lzop_s3_put(s3_url, local_path, gpg_key): """ Compress and upload a given local path. :type s3_url: string :param s3_url: A s3://bucket/key style URL that is the destination :type local_path: string :param local_path: a path to a file to be compressed """ assert not s3_url.endswith(".lzo") s3_url += ".lzo" with tempfile.NamedTemporaryFile(mode="rwb") as tf: pipeline = get_upload_pipeline(file(local_path, "r"), tf, gpg_key=gpg_key) pipeline.finish() tf.flush() logger.info(msg="begin archiving a file", detail=('Uploading "{local_path}" to "{s3_url}".'.format(**locals()))) clock_start = time.clock() tf.seek(0) k = uri_put_file(s3_url, tf) clock_finish = time.clock() kib_per_second = format_kib_per_second(clock_start, clock_finish, k.size) logger.info( msg="completed archiving to a file ", detail=('Archiving to "{s3_url}" complete at ' "{kib_per_second}KiB/s. ").format( s3_url=s3_url, kib_per_second=kib_per_second ), )
def test_upload_download_pipeline(tmpdir, rate_limit): payload, payload_file = create_bogus_payload(tmpdir) # Upload section test_upload = tmpdir.join('upload') with open(unicode(test_upload), 'w') as upload: with open(unicode(payload_file)) as inp: pl = pipeline.get_upload_pipeline( inp, upload, rate_limit=rate_limit) pl.finish() with open(unicode(test_upload)) as completed: round_trip = completed.read() # Download section test_download = tmpdir.join('download') with open(unicode(test_upload)) as upload: with open(unicode(test_download), 'w') as download: pl = pipeline.get_download_pipeline(upload, download) pl.finish() with open(unicode(test_download)) as completed: round_trip = completed.read() assert round_trip == payload
def test_upload_download_pipeline(tmpdir, rate_limit): payload, payload_file = create_bogus_payload(tmpdir) # Upload section test_upload = tmpdir.join('upload') with open(str(test_upload), 'wb') as upload: with open(str(payload_file), 'rb') as inp: with pipeline.get_upload_pipeline(inp, upload, rate_limit=rate_limit): pass with open(str(test_upload), 'rb') as completed: round_trip = completed.read() # Download section test_download = tmpdir.join('download') with open(str(test_upload), 'rb') as upload: with open(str(test_download), 'wb') as download: with pipeline.get_download_pipeline(upload, download): pass with open(str(test_download), 'rb') as completed: round_trip = completed.read() assert round_trip == payload
def test_upload_download_pipeline(tmpdir, rate_limit): payload, payload_file = create_bogus_payload(tmpdir) # Upload section test_upload = tmpdir.join('upload') with open(str(test_upload), 'wb') as upload: with open(str(payload_file), 'rb') as inp: with pipeline.get_upload_pipeline( inp, upload, rate_limit=rate_limit): pass with open(str(test_upload), 'rb') as completed: round_trip = completed.read() # Download section test_download = tmpdir.join('download') with open(str(test_upload), 'rb') as upload: with open(str(test_download), 'wb') as download: with pipeline.get_download_pipeline(upload, download): pass with open(str(test_download), 'rb') as completed: round_trip = completed.read() assert round_trip == payload
def do_lzop_s3_put(s3_url, local_path, gpg_key): """ Compress and upload a given local path. :type s3_url: string :param s3_url: A s3://bucket/key style URL that is the destination :type local_path: string :param local_path: a path to a file to be compressed """ assert not s3_url.endswith('.lzo') s3_url += '.lzo' with tempfile.NamedTemporaryFile(mode='rwb') as tf: pipeline = get_upload_pipeline(file(local_path, 'r'), tf, gpg_key=gpg_key) pipeline.finish() tf.flush() logger.info(msg='begin archiving a file', detail=('Uploading "{local_path}" to "{s3_url}".'.format( **locals()))) clock_start = time.clock() tf.seek(0) k = uri_put_file(s3_url, tf) clock_finish = time.clock() kib_per_second = format_kib_per_second(clock_start, clock_finish, k.size) logger.info(msg='completed archiving to a file ', detail=('Archiving to "{s3_url}" complete at ' '{kib_per_second}KiB/s. ').format( s3_url=s3_url, kib_per_second=kib_per_second))
def __call__(self, tpart): """ Synchronous version of the upload wrapper """ logger.info(msg='beginning volume compression', detail='Building volume {name}.'.format(name=tpart.name)) with tempfile.NamedTemporaryFile(mode='r+b', bufsize=pipebuf.PIPE_BUF_BYTES) as tf: with pipeline.get_upload_pipeline(PIPE, tf, rate_limit=self.rate_limit, gpg_key=self.gpg_key) as pl: tpart.tarfile_write(pl.stdin) tf.flush() # TODO :: Move arbitray path construction to StorageLayout Object url = '{0}/tar_partitions/part_{number:08d}.tar.lzo'.format( self.backup_prefix.rstrip('/'), number=tpart.name) manifest_file = StringIO(tpart.format_manifest()) manifest_url = '{0}/manifests/part_{number:08d}.json'.format( self.backup_prefix.rstrip('/'), number=tpart.name) logger.info(msg='begin uploading a base backup volume', detail='Uploading to "{url}".'.format(url=url)) def log_volume_failures_on_error(exc_tup, exc_processor_cxt): def standard_detail_message(prefix=''): return (prefix + ' There have been {n} attempts to send the ' 'volume {name} so far.'.format(n=exc_processor_cxt, name=tpart.name)) typ, value, tb = exc_tup del exc_tup # Screen for certain kinds of known-errors to retry from if issubclass(typ, socket.error): socketmsg = value[1] if isinstance(value, tuple) else value logger.info( msg='Retrying send because of a socket error', detail=standard_detail_message( "The socket error's message is '{0}'.".format( socketmsg))) elif (issubclass(typ, boto.exception.S3ResponseError) and value.error_code == 'RequestTimeTooSkewed'): logger.info( msg='Retrying send because of a Request Skew time', detail=standard_detail_message()) else: # This type of error is unrecognized as a retry-able # condition, so propagate it, original stacktrace and # all. raise typ, value, tb @retry(retry_with_count(log_volume_failures_on_error)) def put_file_helper(_creds, _url, _file): _file.seek(0) return self.blobstore.uri_put_file(_creds, _url, _file) # Actually do work, retrying if necessary, and timing how long # it takes. clock_start = time.time() k = put_file_helper(self.creds, url, tf) k2 = put_file_helper(self.creds, manifest_url, manifest_file) clock_finish = time.time() kib_per_second = format_kib_per_second(clock_start, clock_finish, k.size + k2.size) logger.info(msg='finish uploading a base backup volume', detail=('Uploading to "{url}" complete at ' '{kib_per_second}KiB/s. '.format( url=url, kib_per_second=kib_per_second))) return tpart
def __call__(self, tpart): """ Synchronous version of the s3-upload wrapper """ logger.info(msg='beginning volume compression', detail='Building volume {name}.'.format(name=tpart.name)) with tempfile.NamedTemporaryFile(mode='rwb') as tf: pipeline = get_upload_pipeline(PIPE, tf, rate_limit=self.rate_limit, gpg_key=self.gpg_key) tpart.tarfile_write(pipeline.stdin) pipeline.stdin.flush() pipeline.stdin.close() pipeline.finish() tf.flush() s3_url = '/'.join([self.backup_s3_prefix, 'tar_partitions', 'part_{number}.tar.lzo' .format(number=tpart.name)]) logger.info( msg='begin uploading a base backup volume', detail=('Uploading to "{s3_url}".') .format(s3_url=s3_url)) def log_volume_failures_on_error(exc_tup, exc_processor_cxt): def standard_detail_message(prefix=''): return (prefix + ' There have been {n} attempts to send the ' 'volume {name} so far.'.format(n=exc_processor_cxt, name=tpart.name)) typ, value, tb = exc_tup del exc_tup # Screen for certain kinds of known-errors to retry from if issubclass(typ, socket.error): socketmsg = value[1] if isinstance(value, tuple) else value logger.info( msg='Retrying send because of a socket error', detail=standard_detail_message( "The socket error's message is '{0}'." .format(socketmsg))) elif (issubclass(typ, boto.exception.S3ResponseError) and value.error_code == 'RequestTimeTooSkewed'): logger.info( msg='Retrying send because of a Request Skew time', detail=standard_detail_message()) else: # This type of error is unrecognized as a retry-able # condition, so propagate it, original stacktrace and # all. raise typ, value, tb @retry(retry_with_count(log_volume_failures_on_error)) def put_file_helper(): tf.seek(0) return uri_put_file(s3_url, tf) # Actually do work, retrying if necessary, and timing how long # it takes. clock_start = time.clock() k = put_file_helper() clock_finish = time.clock() kib_per_second = format_kib_per_second(clock_start, clock_finish, k.size) logger.info( msg='finish uploading a base backup volume', detail=('Uploading to "{s3_url}" complete at ' '{kib_per_second}KiB/s. ') .format(s3_url=s3_url, kib_per_second=kib_per_second)) return tpart
def do_partition_put(backup_s3_prefix, tpart, rate_limit, gpg_key): """ Synchronous version of the s3-upload wrapper """ logger.info(msg='beginning volume compression', detail='Building volume {name}.'.format(name=tpart.name)) with tempfile.NamedTemporaryFile(mode='rwb') as tf: pipeline = get_upload_pipeline(PIPE, tf, rate_limit=rate_limit, gpg_key=gpg_key) tpart.tarfile_write(pipeline.stdin) pipeline.stdin.flush() pipeline.stdin.close() pipeline.finish() tf.flush() s3_url = '/'.join([ backup_s3_prefix, 'tar_partitions', 'part_{number}.tar.lzo'.format(number=tpart.name) ]) logger.info(msg='begin uploading a base backup volume', detail=('Uploading to "{s3_url}".').format(s3_url=s3_url)) def log_volume_failures_on_error(exc_tup, exc_processor_cxt): def standard_detail_message(prefix=''): return (prefix + ' There have been {n} attempts to send the ' 'volume {name} so far.'.format(n=exc_processor_cxt, name=tpart.name)) typ, value, tb = exc_tup del exc_tup # Screen for certain kinds of known-errors to retry from if issubclass(typ, socket.error): socketmsg = value[1] if isinstance(value, tuple) else value logger.info(msg='Retrying send because of a socket error', detail=standard_detail_message( "The socket error's message is '{0}'.".format( socketmsg))) elif (issubclass(typ, boto.exception.S3ResponseError) and value.error_code == 'RequestTimeTooSkewed'): logger.info(msg='Retrying send because of a Request Skew time', detail=standard_detail_message()) else: # This type of error is unrecognized as a retry-able # condition, so propagate it, original stacktrace and # all. raise typ, value, tb @retry(retry_with_count(log_volume_failures_on_error)) def put_file_helper(): tf.seek(0) return uri_put_file(s3_url, tf) # Actually do work, retrying if necessary, and timing how long # it takes. clock_start = time.clock() k = put_file_helper() clock_finish = time.clock() kib_per_second = format_kib_per_second(clock_start, clock_finish, k.size) logger.info(msg='finish uploading a base backup volume', detail=('Uploading to "{s3_url}" complete at ' '{kib_per_second}KiB/s. ').format( s3_url=s3_url, kib_per_second=kib_per_second))
def __call__(self, tpart): """ Synchronous version of the upload wrapper """ logger.info(msg="beginning volume compression", detail="Building volume {name}.".format(name=tpart.name)) with tempfile.NamedTemporaryFile(mode="rwb") as tf: pipeline = get_upload_pipeline(PIPE, tf, rate_limit=self.rate_limit, gpg_key=self.gpg_key) tpart.tarfile_write(pipeline.stdin) pipeline.stdin.flush() pipeline.stdin.close() pipeline.finish() tf.flush() # TODO :: Move arbitray path construction to StorageLayout Object url = "{0}/tar_partitions/part_{number}.tar.lzo".format(self.backup_prefix.rstrip("/"), number=tpart.name) logger.info(msg="begin uploading a base backup volume", detail='Uploading to "{url}".'.format(url=url)) def log_volume_failures_on_error(exc_tup, exc_processor_cxt): def standard_detail_message(prefix=""): return prefix + " There have been {n} attempts to send the " "volume {name} so far.".format( n=exc_processor_cxt, name=tpart.name ) typ, value, tb = exc_tup del exc_tup # Screen for certain kinds of known-errors to retry from if issubclass(typ, socket.error): socketmsg = value[1] if isinstance(value, tuple) else value logger.info( msg="Retrying send because of a socket error", detail=standard_detail_message("The socket error's message is '{0}'.".format(socketmsg)), ) elif issubclass(typ, boto.exception.S3ResponseError) and value.error_code == "RequestTimeTooSkewed": logger.info(msg="Retrying send because of a Request Skew time", detail=standard_detail_message()) else: # This type of error is unrecognized as a retry-able # condition, so propagate it, original stacktrace and # all. raise typ, value, tb @retry(retry_with_count(log_volume_failures_on_error)) def put_file_helper(): tf.seek(0) return self.blobstore.uri_put_file(self.creds, url, tf) # Actually do work, retrying if necessary, and timing how long # it takes. clock_start = time.time() k = put_file_helper() clock_finish = time.time() kib_per_second = format_kib_per_second(clock_start, clock_finish, k.size) logger.info( msg="finish uploading a base backup volume", detail=( 'Uploading to "{url}" complete at ' "{kib_per_second}KiB/s. ".format(url=url, kib_per_second=kib_per_second) ), ) return tpart
def __call__(self, tpart): """ Synchronous version of the upload wrapper """ logger.info(msg='beginning volume compression', detail='Building volume {name}.'.format(name=tpart.name)) with tempfile.NamedTemporaryFile( mode='r+b', bufsize=pipebuf.PIPE_BUF_BYTES) as tf: with pipeline.get_upload_pipeline(PIPE, tf, rate_limit=self.rate_limit, gpg_key=self.gpg_key) as pl: tpart.tarfile_write(pl.stdin) tf.flush() # TODO :: Move arbitray path construction to StorageLayout Object url = '{0}/tar_partitions/part_{number:08d}.tar.lzo'.format( self.backup_prefix.rstrip('/'), number=tpart.name) manifest_file = StringIO(tpart.format_manifest()) manifest_url = '{0}/manifests/part_{number:08d}.json'.format( self.backup_prefix.rstrip('/'), number=tpart.name) logger.info(msg='begin uploading a base backup volume', detail='Uploading to "{url}".'.format(url=url)) def log_volume_failures_on_error(exc_tup, exc_processor_cxt): def standard_detail_message(prefix=''): return (prefix + ' There have been {n} attempts to send the ' 'volume {name} so far.'.format(n=exc_processor_cxt, name=tpart.name)) typ, value, tb = exc_tup del exc_tup # Screen for certain kinds of known-errors to retry from if issubclass(typ, socket.error): socketmsg = value[1] if isinstance(value, tuple) else value logger.info( msg='Retrying send because of a socket error', detail=standard_detail_message( "The socket error's message is '{0}'." .format(socketmsg))) elif (issubclass(typ, boto.exception.S3ResponseError) and value.error_code == 'RequestTimeTooSkewed'): logger.info( msg='Retrying send because of a Request Skew time', detail=standard_detail_message()) else: # This type of error is unrecognized as a retry-able # condition, so propagate it, original stacktrace and # all. raise typ, value, tb @retry(retry_with_count(log_volume_failures_on_error)) def put_file_helper(_creds, _url, _file): _file.seek(0) return self.blobstore.uri_put_file(_creds, _url, _file) # Actually do work, retrying if necessary, and timing how long # it takes. clock_start = time.time() k = put_file_helper(self.creds, url, tf) k2 = put_file_helper(self.creds, manifest_url, manifest_file) clock_finish = time.time() kib_per_second = format_kib_per_second(clock_start, clock_finish, k.size + k2.size) logger.info( msg='finish uploading a base backup volume', detail=('Uploading to "{url}" complete at ' '{kib_per_second}KiB/s. ' .format(url=url, kib_per_second=kib_per_second))) return tpart