def test_untar_from_s3(check_empty=False):
bucket_name = str(uuid.uuid4())
prefix = '{0}/'.format(str(uuid.uuid4()))
tar_name = '{0}.tar.gz'.format(str(uuid.uuid4()))
conn = S3Connection()
conn.create_bucket(bucket_name)
tar = TarHelper(
os.path.join(_outdir, tar_name),
os.path.join(os.path.dirname(__file__), 'testfiles')
)
tar.tar_to_s3(bucket_name=bucket_name, prefix=prefix)
bucket = S3BucketHelper(
bucket_name=bucket_name,
prefix=prefix
)
assert bucket.get_most_recent_key() == '{0}{1}'.format(prefix, tar_name)
archive_path = os.path.join(
_outdir,
str(uuid.uuid4()),
'{0}.tar.gz'.format(str(uuid.uuid4()))
)
target_path = os.path.join(_outdir, str(uuid.uuid4()))
if check_empty:
bucket.prefix = str(uuid.uuid4())
tar2 = bucket.get_tar_helper(
most_recent=True,
archive_path=archive_path,
target_path=target_path
)
if check_empty:
assert tar2 is None
return
assert os.path.isfile(archive_path)
tar2.extract()
assert os.path.isdir(target_path)
for xfile in range(1, 10):
assert os.path.isfile(os.path.join(target_path, 'file_{0}'.format(xfile)))
for xdir in range(1, 3):
dirpath = os.path.join(target_path, 'dir_{0}'.format(xdir))
assert os.path.isdir(dirpath)
for xfile in range(1, 10):
assert os.path.isfile(os.path.join(dirpath, 'file_{0}'.format(xfile)))
def test_scripts_tar_to_s3_and_back(check_empty=False):
bucket_name = str(uuid.uuid4())
prefix = '{0}/'.format(str(uuid.uuid4()))
tar_name = '{0}.tar.gz'.format(str(uuid.uuid4()))
mock_args = namedtuple('mock_args', 'directory prefix bucket key_format')
args = mock_args(directory=os.path.join(os.path.dirname(__file__), 'testfiles'),
prefix=prefix,
bucket=bucket_name,
key_format='{0}.tar.gz'.format(tar_name))
conn = S3Connection()
conn.create_bucket(bucket_name)
TarHelper.tar_to_s3_entry(args)
bucket = S3BucketHelper(
bucket_name=bucket_name,
prefix=prefix
)
assert bucket.get_most_recent_key() == '{0}{1}.tar.gz'.format(prefix, tar_name)
target_path = os.path.join(_outdir, os.path.dirname(__file__), 'testfiles')
mock_args = namedtuple('mock_args', 'directory prefix bucket cwd delete save_as key ignore_missing')
args = mock_args(directory=target_path,
prefix=prefix if not check_empty else str(uuid.uuid4()),
bucket=bucket_name,
delete=True,
cwd=_outdir,
save_as=None,
key=None,
ignore_missing=True if check_empty else False)
result = TarHelper.untar_from_s3_entry(args)
if check_empty:
assert result is False
else:
for xdir in range(1, 3):
dirpath = os.path.join(target_path, 'dir_{0}'.format(xdir))
assert os.path.isdir(dirpath)
for xfile in range(1, 10):
assert os.path.isfile(os.path.join(dirpath, 'file_{0}'.format(xfile)))
def test_tar_to_s3():
bucket_name = str(uuid.uuid4())
prefix = '{0}/'.format(str(uuid.uuid4()))
tar_name = '{0}.tar.gz'.format(str(uuid.uuid4()))
conn = S3Connection()
conn.create_bucket(bucket_name)
tar = TarHelper(
os.path.join(_outdir, tar_name),
os.path.join(os.path.dirname(__file__), 'testfiles')
)
tar.tar_to_s3(bucket_name=bucket_name, prefix=prefix)
bucket = S3BucketHelper(
bucket_name=bucket_name,
prefix=prefix
)
assert bucket.get_most_recent_key() == '{0}{1}'.format(prefix, tar_name)