def fetch_partition(self, partition_name):
part_abs_name = self.layout.basebackup_tar_partition(
self.backup_info, partition_name)
logger.info(msg='beginning partition download',
detail='The partition being downloaded is {0}.'.format(
partition_name),
hint='The absolute S3 key is {0}.'.format(part_abs_name))
key = self.bucket.get_key(part_abs_name)
with get_download_pipeline(PIPE, PIPE, self.decrypt) as pl:
g = gevent.spawn(s3.write_and_return_error, key, pl.stdin)
TarPartition.tarfile_extract(pl.stdout, self.local_root)
# Raise any exceptions guarded by write_and_return_error.
exc = g.get()
if exc is not None:
raise exc
def fetch_partition(self, partition_name):
part_abs_name = self.layout.basebackup_tar_partition(
self.backup_info, partition_name)
logger.info(
msg='beginning partition download',
detail='The partition being downloaded is {0}.'
.format(partition_name),
hint='The absolute file key is {0}.'.format(part_abs_name))
key = self.bucket.get_key(part_abs_name)
with get_download_pipeline(PIPE, PIPE, self.decrypt) as pl:
g = gevent.spawn(file.write_and_return_error, key, pl.stdin)
TarPartition.tarfile_extract(pl.stdout, self.local_root)
# Raise any exceptions guarded by write_and_return_error.
exc = g.get()
if exc is not None:
raise exc
def fetch_partition(self, partition_name):
part_abs_name = self.layout.basebackup_tar_partition(
self.backup_info, partition_name)
logger.info(msg='beginning partition download',
detail=('The partition being downloaded is {0}.'.format(
partition_name)),
hint='The absolute S3 key is {0}.'.format(part_abs_name))
url = 'wabs://{ctr}/{path}'.format(ctr=self.layout.store_name(),
path=part_abs_name)
with get_download_pipeline(PIPE, PIPE, self.decrypt) as pl:
g = gevent.spawn(wabs.write_and_return_error, url, self.wabs_conn,
pl.stdin)
TarPartition.tarfile_extract(pl.stdout, self.local_root)
# Raise any exceptions from self._write_and_close
exc = g.get()
if exc is not None:
raise exc
def fetch_partition(self, partition_name):
part_abs_name = self.layout.basebackup_tar_partition(
self.backup_info, partition_name)
logger.info(
msg='beginning partition download',
detail='The partition being downloaded is {0}.'.format(
partition_name),
hint='The absolute GCS object is {0}.'.format(part_abs_name))
blob = self.bucket.get_blob('/' + part_abs_name)
signed = blob.generate_signed_url(datetime.datetime.utcnow() +
datetime.timedelta(minutes=10))
with get_download_pipeline(PIPE, PIPE, self.decrypt) as pl:
g = gevent.spawn(gs.write_and_return_error, signed, pl.stdin)
TarPartition.tarfile_extract(pl.stdout, self.local_root)
# Raise any exceptions guarded by write_and_return_error.
exc = g.get()
if exc is not None:
raise exc
def fetch_partition(self, partition_name):
part_abs_name = self.layout.basebackup_tar_partition(
self.backup_info, partition_name)
logger.info(
msg='beginning partition download',
detail=('The partition being downloaded is {0}.'
.format(partition_name)),
hint='The absolute S3 key is {0}.'.format(part_abs_name))
url = 'wabs://{ctr}/{path}'.format(ctr=self.layout.store_name(),
path=part_abs_name)
with get_download_pipeline(PIPE, PIPE, self.decrypt) as pl:
g = gevent.spawn(wabs.write_and_return_error,
url, self.wabs_conn, pl.stdin)
TarPartition.tarfile_extract(pl.stdout, self.local_root)
# Raise any exceptions from self._write_and_close
exc = g.get()
if exc is not None:
raise exc
def fetch_partition(self, partition_name):
part_abs_name = self.layout.basebackup_tar_partition(
self.backup_info, partition_name)
logger.info(msg='beginning partition download',
detail=('The partition being downloaded is {0}.'.format(
partition_name)),
hint='The absolute Swift object name is {0}.'.format(
part_abs_name))
url = 'swift://{ctr}/{path}'.format(ctr=self.layout.store_name(),
path=part_abs_name)
pipeline = get_download_pipeline(PIPE, PIPE, self.decrypt)
g = gevent.spawn(swift.write_and_return_error, url, self.swift_conn,
pipeline.stdin)
TarPartition.tarfile_extract(pipeline.stdout, self.local_root)
# Raise any exceptions guarded by write_and_return_error.
exc = g.get()
if exc is not None:
raise exc
pipeline.finish()
def fetch_partition(self, partition_name):
part_abs_name = self.layout.basebackup_tar_partition(
self.backup_info, partition_name)
logger.info(
msg='beginning partition download',
detail=('The partition being downloaded is {0}.'
.format(partition_name)),
hint='The absolute Swift object name is {0}.'
.format(part_abs_name))
url = 'swift://{ctr}/{path}'.format(ctr=self.layout.store_name(),
path=part_abs_name)
pipeline = get_download_pipeline(PIPE, PIPE, self.decrypt)
g = gevent.spawn(swift.write_and_return_error,
url, self.swift_conn, pipeline.stdin)
TarPartition.tarfile_extract(pipeline.stdout, self.local_root)
# Raise any exceptions guarded by write_and_return_error.
exc = g.get()
if exc is not None:
raise exc
pipeline.finish()
def fetch_manifest(self, partition_name, wale_info_dir):
part_abs_name = self.layout.basebackup_manifest(
self.backup_info, partition_name)
logger.info(
msg='beginning manifest download',
detail='The manifest being downloaded is {0}.'
.format(partition_name),
hint='The absolute S3 key is {0}.'.format(part_abs_name))
key = self.bucket.get_key(part_abs_name)
with get_download_pipeline(PIPE, PIPE,
gpg=self.decrypt,
lzop=False) as pl:
g = gevent.spawn(s3.write_and_return_error, key, pl.stdin)
TarPartition.manifest_extract(pl.stdout,
self.local_root,
partition_name,
wale_info_dir)
# Raise any exceptions guarded by write_and_return_error.
exc = g.get()
if exc is not None:
raise exc
