def do_lzo_aes_s3_get(s3_url, path, s3cmd_config_path): """ Get and decompress a S3 URL This streams the s3cmd directly to lzo_aes; the compressed version is never stored on disk. """ assert s3_url.endswith('.lzo.aes'), 'Expect an lzo_aes aes encrypted file' with open(path, 'wb') as decomp_out: popens = [] try: popens = pipe( dict(args=[S3CMD_BIN, '-c', s3cmd_config_path, 'get', s3_url, '-'], bufsize=BUFSIZE_HT), dict(args=[DECODE_BIN], stdout=decomp_out, bufsize=BUFSIZE_HT)) pipe_wait(popens) s3cmd_proc, decode_proc = popens def check_exitcode(cmdname, popen): if popen.returncode != 0: raise UserException( 'downloading a wal file has failed', cmdname + ' terminated with exit code: ' + unicode(s3cmd_proc.returncode)) check_exitcode('s3cmd', s3cmd_proc) check_exitcode('decode', decode_proc) print >>sys.stderr, ('Got and decompressed file: ' '{s3_url} to {path}' .format(**locals())) except KeyboardInterrupt, keyboard_int: for popen in popens: try: popen.send_signal(signal.SIGINT) popen.wait() except OSError, e: # ESRCH aka "no such process" if e.errno != errno.ESRCH: raise e raise keyboard_int
def do_partition_get(backup_s3_prefix, local_root, tpart_number, s3cmd_config_path): tar = None try: popens = pipe( dict(args=[S3CMD_BIN, '-c', s3cmd_config_path, 'get', '/'.join([backup_s3_prefix, 'tar_partitions', 'part_{0}.tar.lzo.aes'.format(tpart_number)]), '-'], bufsize=BUFSIZE_HT), dict(args=[DECODE_BIN], stdout=subprocess.PIPE, bufsize=BUFSIZE_HT)) assert len(popens) > 0 tar = tarfile.open(mode='r|', fileobj=popens[-1].stdout) tar.extractall(local_root) tar.close() popens[-1].stdout.close() pipe_wait(popens) s3cmd_proc, decode_proc = popens def check_exitcode(cmdname, popen): if popen.returncode != 0: raise UserException( 'downloading a tarfile cluster partition has failed', cmdname + ' terminated with exit code: ' + unicode(s3cmd_proc.returncode)) check_exitcode('s3cmd', s3cmd_proc) check_exitcode('decode', decode_proc) except KeyboardInterrupt, keyboard_int: for popen in popens: try: popen.send_signal(signal.SIGINT) popen.wait() except OSError, e: # ESRCH aka "no such process" if e.errno != errno.ESRCH: raise