def do_lzo_aes_s3_get(s3_url, path, s3cmd_config_path):
"""
Get and decompress a S3 URL
This streams the s3cmd directly to lzo_aes; the compressed version is
never stored on disk.
"""
assert s3_url.endswith('.lzo.aes'), 'Expect an lzo_aes aes encrypted file'
with open(path, 'wb') as decomp_out:
popens = []
try:
popens = pipe(
dict(args=[S3CMD_BIN, '-c', s3cmd_config_path,
'get', s3_url, '-'],
bufsize=BUFSIZE_HT),
dict(args=[DECODE_BIN], stdout=decomp_out,
bufsize=BUFSIZE_HT))
pipe_wait(popens)
s3cmd_proc, decode_proc = popens
def check_exitcode(cmdname, popen):
if popen.returncode != 0:
raise UserException(
'downloading a wal file has failed',
cmdname + ' terminated with exit code: ' +
unicode(s3cmd_proc.returncode))
check_exitcode('s3cmd', s3cmd_proc)
check_exitcode('decode', decode_proc)
print >>sys.stderr, ('Got and decompressed file: '
'{s3_url} to {path}'
.format(**locals()))
except KeyboardInterrupt, keyboard_int:
for popen in popens:
try:
popen.send_signal(signal.SIGINT)
popen.wait()
except OSError, e:
# ESRCH aka "no such process"
if e.errno != errno.ESRCH:
raise e
raise keyboard_int
def do_partition_get(backup_s3_prefix, local_root, tpart_number,
s3cmd_config_path):
tar = None
try:
popens = pipe(
dict(args=[S3CMD_BIN, '-c', s3cmd_config_path, 'get',
'/'.join([backup_s3_prefix, 'tar_partitions',
'part_{0}.tar.lzo.aes'.format(tpart_number)]),
'-'],
bufsize=BUFSIZE_HT),
dict(args=[DECODE_BIN], stdout=subprocess.PIPE,
bufsize=BUFSIZE_HT))
assert len(popens) > 0
tar = tarfile.open(mode='r|', fileobj=popens[-1].stdout)
tar.extractall(local_root)
tar.close()
popens[-1].stdout.close()
pipe_wait(popens)
s3cmd_proc, decode_proc = popens
def check_exitcode(cmdname, popen):
if popen.returncode != 0:
raise UserException(
'downloading a tarfile cluster partition has failed',
cmdname + ' terminated with exit code: ' +
unicode(s3cmd_proc.returncode))
check_exitcode('s3cmd', s3cmd_proc)
check_exitcode('decode', decode_proc)
except KeyboardInterrupt, keyboard_int:
for popen in popens:
try:
popen.send_signal(signal.SIGINT)
popen.wait()
except OSError, e:
# ESRCH aka "no such process"
if e.errno != errno.ESRCH:
raise
