def test_CommunicationWriterTGZ_single_file_default_name(
output_file, login_info):
comm = read_communication_from_file("tests/testdata/simple_1.concrete")
writer = CommunicationWriterTGZ()
try:
writer.open(output_file)
writer.write(comm)
finally:
writer.close()
assert tarfile.is_tarfile(output_file)
f = tarfile.open(output_file)
tarinfo = f.next()
assert tarinfo is not None
assert comm.id + '.concrete' == tarinfo.name
assert tarinfo.isreg()
assert tarinfo.mtime > time() - TIME_MARGIN
assert os.stat('tests/testdata/simple_1.concrete').st_size == tarinfo.size
assert 0o644 == tarinfo.mode
assert login_info['uid'] == tarinfo.uid
assert login_info['username'] == tarinfo.uname
assert login_info['gid'] == tarinfo.gid
assert login_info['groupname'] == tarinfo.gname
tarinfo = f.next()
assert tarinfo is None
f.close()
def test_CommunicationWriterTGZ_single_file_default_name(output_file,
login_info):
comm = read_communication_from_file("tests/testdata/simple_1.concrete")
writer = CommunicationWriterTGZ()
try:
writer.open(output_file)
writer.write(comm)
finally:
writer.close()
assert tarfile.is_tarfile(output_file)
f = tarfile.open(output_file)
tarinfo = f.next()
assert tarinfo is not None
assert comm.id + '.concrete' == tarinfo.name
assert tarinfo.isreg()
assert tarinfo.mtime > time() - TIME_MARGIN
assert os.stat('tests/testdata/simple_1.concrete').st_size == tarinfo.size
assert 0o644 == tarinfo.mode
assert login_info['uid'] == tarinfo.uid
assert login_info['username'] == tarinfo.uname
assert login_info['gid'] == tarinfo.gid
assert login_info['groupname'] == tarinfo.gname
tarinfo = f.next()
assert tarinfo is None
f.close()
def main():
set_stdout_encoding()
parser = argparse.ArgumentParser(
description='Read communications from AWS S3 bucket (keyed by '
'communication id) and write to a tar.gz file.',
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
parser.add_argument('bucket_name', help='name of S3 bucket to read from')
parser.add_argument('output_path',
help='path to input communications (uncompressed, '
'gz, bz2, tar, zip, etc.)')
parser.add_argument('--prefix-len',
type=int,
default=DEFAULT_S3_KEY_PREFIX_LEN,
help='S3 keys are prefixed with hashes of this length')
parser.add_argument('-l',
'--loglevel',
'--log-level',
help='Logging verbosity level threshold (to stderr)',
default='info')
concrete.version.add_argparse_argument(parser)
args = parser.parse_args()
logging.basicConfig(format='%(asctime)-15s %(levelname)s: %(message)s',
level=args.loglevel.upper())
logging.info('connecting to s3')
conn = connect_s3()
logging.info('retrieving bucket {}'.format(args.bucket_name))
bucket = conn.get_bucket(args.bucket_name)
logging.info(
'reading from s3 bucket {}, prefix length {}; writing to {}'.format(
args.bucket_name, args.prefix_len, args.output_path))
container = S3BackedCommunicationContainer(bucket, args.prefix_len)
with CommunicationWriterTGZ(args.output_path) as writer:
for comm_id in container:
logging.info('fetching {}'.format(comm_id))
writer.write(container[comm_id])