def test_untar_from_s3(check_empty=False):

    bucket_name = str(uuid.uuid4())
    prefix = '{0}/'.format(str(uuid.uuid4()))
    tar_name = '{0}.tar.gz'.format(str(uuid.uuid4()))

    conn = S3Connection()
    conn.create_bucket(bucket_name)

    tar = TarHelper(
        os.path.join(_outdir, tar_name),
        os.path.join(os.path.dirname(__file__), 'testfiles')
    )
    tar.tar_to_s3(bucket_name=bucket_name, prefix=prefix)

    bucket = S3BucketHelper(
        bucket_name=bucket_name,
        prefix=prefix
    )

    assert bucket.get_most_recent_key() == '{0}{1}'.format(prefix, tar_name)

    archive_path = os.path.join(
        _outdir,
        str(uuid.uuid4()),
        '{0}.tar.gz'.format(str(uuid.uuid4()))
    )
    target_path = os.path.join(_outdir, str(uuid.uuid4()))

    if check_empty:
        bucket.prefix = str(uuid.uuid4())

    tar2 = bucket.get_tar_helper(
        most_recent=True,
        archive_path=archive_path,
        target_path=target_path
    )

    if check_empty:
        assert tar2 is None
        return

    assert os.path.isfile(archive_path)

    tar2.extract()

    assert os.path.isdir(target_path)
    for xfile in range(1, 10):
            assert os.path.isfile(os.path.join(target_path, 'file_{0}'.format(xfile)))

    for xdir in range(1, 3):
        dirpath = os.path.join(target_path, 'dir_{0}'.format(xdir))
        assert os.path.isdir(dirpath)
        for xfile in range(1, 10):
            assert os.path.isfile(os.path.join(dirpath, 'file_{0}'.format(xfile)))
def test_scripts_tar_to_s3_and_back(check_empty=False):

    bucket_name = str(uuid.uuid4())
    prefix = '{0}/'.format(str(uuid.uuid4()))
    tar_name = '{0}.tar.gz'.format(str(uuid.uuid4()))

    mock_args = namedtuple('mock_args', 'directory prefix bucket key_format')
    args = mock_args(directory=os.path.join(os.path.dirname(__file__), 'testfiles'),
                     prefix=prefix,
                     bucket=bucket_name,
                     key_format='{0}.tar.gz'.format(tar_name))

    conn = S3Connection()
    conn.create_bucket(bucket_name)

    TarHelper.tar_to_s3_entry(args)

    bucket = S3BucketHelper(
        bucket_name=bucket_name,
        prefix=prefix
    )

    assert bucket.get_most_recent_key() == '{0}{1}.tar.gz'.format(prefix, tar_name)

    target_path = os.path.join(_outdir, os.path.dirname(__file__), 'testfiles')
    mock_args = namedtuple('mock_args', 'directory prefix bucket cwd delete save_as key ignore_missing')
    args = mock_args(directory=target_path,
                     prefix=prefix if not check_empty else str(uuid.uuid4()),
                     bucket=bucket_name,
                     delete=True,
                     cwd=_outdir,
                     save_as=None,
                     key=None,
                     ignore_missing=True if check_empty else False)

    result = TarHelper.untar_from_s3_entry(args)

    if check_empty:
        assert result is False
    else:
        for xdir in range(1, 3):
            dirpath = os.path.join(target_path, 'dir_{0}'.format(xdir))
            assert os.path.isdir(dirpath)
            for xfile in range(1, 10):
                assert os.path.isfile(os.path.join(dirpath, 'file_{0}'.format(xfile)))
def test_tar_to_s3():

    bucket_name = str(uuid.uuid4())
    prefix = '{0}/'.format(str(uuid.uuid4()))
    tar_name = '{0}.tar.gz'.format(str(uuid.uuid4()))

    conn = S3Connection()
    conn.create_bucket(bucket_name)

    tar = TarHelper(
        os.path.join(_outdir, tar_name),
        os.path.join(os.path.dirname(__file__), 'testfiles')
    )
    tar.tar_to_s3(bucket_name=bucket_name, prefix=prefix)

    bucket = S3BucketHelper(
        bucket_name=bucket_name,
        prefix=prefix
    )

    assert bucket.get_most_recent_key() == '{0}{1}'.format(prefix, tar_name)