def test_untar_from_s3(check_empty=False): bucket_name = str(uuid.uuid4()) prefix = '{0}/'.format(str(uuid.uuid4())) tar_name = '{0}.tar.gz'.format(str(uuid.uuid4())) conn = S3Connection() conn.create_bucket(bucket_name) tar = TarHelper( os.path.join(_outdir, tar_name), os.path.join(os.path.dirname(__file__), 'testfiles') ) tar.tar_to_s3(bucket_name=bucket_name, prefix=prefix) bucket = S3BucketHelper( bucket_name=bucket_name, prefix=prefix ) assert bucket.get_most_recent_key() == '{0}{1}'.format(prefix, tar_name) archive_path = os.path.join( _outdir, str(uuid.uuid4()), '{0}.tar.gz'.format(str(uuid.uuid4())) ) target_path = os.path.join(_outdir, str(uuid.uuid4())) if check_empty: bucket.prefix = str(uuid.uuid4()) tar2 = bucket.get_tar_helper( most_recent=True, archive_path=archive_path, target_path=target_path ) if check_empty: assert tar2 is None return assert os.path.isfile(archive_path) tar2.extract() assert os.path.isdir(target_path) for xfile in range(1, 10): assert os.path.isfile(os.path.join(target_path, 'file_{0}'.format(xfile))) for xdir in range(1, 3): dirpath = os.path.join(target_path, 'dir_{0}'.format(xdir)) assert os.path.isdir(dirpath) for xfile in range(1, 10): assert os.path.isfile(os.path.join(dirpath, 'file_{0}'.format(xfile)))
def test_scripts_tar_to_s3_and_back(check_empty=False): bucket_name = str(uuid.uuid4()) prefix = '{0}/'.format(str(uuid.uuid4())) tar_name = '{0}.tar.gz'.format(str(uuid.uuid4())) mock_args = namedtuple('mock_args', 'directory prefix bucket key_format') args = mock_args(directory=os.path.join(os.path.dirname(__file__), 'testfiles'), prefix=prefix, bucket=bucket_name, key_format='{0}.tar.gz'.format(tar_name)) conn = S3Connection() conn.create_bucket(bucket_name) TarHelper.tar_to_s3_entry(args) bucket = S3BucketHelper( bucket_name=bucket_name, prefix=prefix ) assert bucket.get_most_recent_key() == '{0}{1}.tar.gz'.format(prefix, tar_name) target_path = os.path.join(_outdir, os.path.dirname(__file__), 'testfiles') mock_args = namedtuple('mock_args', 'directory prefix bucket cwd delete save_as key ignore_missing') args = mock_args(directory=target_path, prefix=prefix if not check_empty else str(uuid.uuid4()), bucket=bucket_name, delete=True, cwd=_outdir, save_as=None, key=None, ignore_missing=True if check_empty else False) result = TarHelper.untar_from_s3_entry(args) if check_empty: assert result is False else: for xdir in range(1, 3): dirpath = os.path.join(target_path, 'dir_{0}'.format(xdir)) assert os.path.isdir(dirpath) for xfile in range(1, 10): assert os.path.isfile(os.path.join(dirpath, 'file_{0}'.format(xfile)))
def test_tar_to_s3(): bucket_name = str(uuid.uuid4()) prefix = '{0}/'.format(str(uuid.uuid4())) tar_name = '{0}.tar.gz'.format(str(uuid.uuid4())) conn = S3Connection() conn.create_bucket(bucket_name) tar = TarHelper( os.path.join(_outdir, tar_name), os.path.join(os.path.dirname(__file__), 'testfiles') ) tar.tar_to_s3(bucket_name=bucket_name, prefix=prefix) bucket = S3BucketHelper( bucket_name=bucket_name, prefix=prefix ) assert bucket.get_most_recent_key() == '{0}{1}'.format(prefix, tar_name)