def test_get(self, s3_client):
BUCKET = "test-bucket"
test_file = Path("test_file")
with test_file.open("w") as f:
f.write("sample text\n")
# create bucket
s3_client.create_bucket(Bucket=BUCKET)
# place file in bucket so that pegasus-s3 put can overwrite it
s3_client.upload_file(str(test_file), BUCKET, test_file.name)
# remove file
test_file.unlink()
# get file
# pegasus-s3 get s3://rynge@osgconnect/test-bucket/test_file
parser, args = s3.parse_args([
"get",
"s3://rynge@osgconnect/{}/{}".format(BUCKET, test_file.name)
])
s3.get(args)
# file should exist again
assert test_file.exists() == True
test_file.unlink()
# cleanup
s3_client.delete_object(Bucket=BUCKET, Key=test_file.name)
s3_client.delete_bucket(Bucket=BUCKET)
def test_get_as_file(self, s3_client, test_file):
BUCKET = "test-bucket"
# create bucket
s3_client.create_bucket(Bucket=BUCKET)
# place file in bucket
s3_client.upload_file(str(test_file), BUCKET, test_file.name)
# get file
# pegasus-s3 get s3://rynge@osgconnect/test-bucket/test_file new_test_file
parser, args = s3.parse_args([
"get",
"s3://rynge@osgconnect/{}/{}".format(BUCKET, test_file.name),
"new_test_file",
])
s3.get(args)
# check that test_file downloaded as new_test_file
new_test_file = Path("new_test_file")
assert new_test_file.exists() == True
# cleanup
new_test_file.unlink()
s3_client.delete_object(Bucket=BUCKET, Key=test_file.name)
s3_client.delete_bucket(Bucket=BUCKET)
def test_put_key_already_exists(self, s3_client, test_file):
BUCKET = "test-bucket"
# create bucket
s3_client.create_bucket(Bucket=BUCKET)
# place file in bucket
s3_client.upload_file(str(test_file), BUCKET, test_file.name)
# upload file (key that already exists in bucket)
# pegasus-s3 put test_file s3://rynge@osgconnect/test-bucket/test_file
paser, args = s3.parse_args([
"put",
str(test_file),
"s3://rynge@osgconnect/{}/{}".format(BUCKET, test_file.name),
])
with pytest.raises(Exception) as e:
s3.put(args)
assert "Key: test_file already exists" in str(e)
# cleanup
s3_client.delete_object(Bucket=BUCKET, Key=test_file.name)
s3_client.delete_bucket(Bucket=BUCKET)
def test_put_force(self, s3_client, test_file):
BUCKET = "test-bucket"
# create bucket
s3_client.create_bucket(Bucket=BUCKET)
# place file in bucket so that pegasus-s3 put can overwrite it
s3_client.upload_file(str(test_file), BUCKET, test_file.name)
# upload file
# pegasus-s3 put --force test_file s3://rynge@osgconnect/test-bucket/test_file
parser, args = s3.parse_args([
"put",
"--force",
str(test_file),
"s3://rynge@osgconnect/{}/{}".format(BUCKET, test_file.name),
])
s3.put(args)
try:
s3_client.head_object(Bucket=BUCKET, Key=test_file.name)
except botocore.exceptions.ClientError:
pytest.fail("Key: {} should exist in Bucket: {}".format(
test_file.name, BUCKET))
# cleanup
s3_client.delete_object(Bucket=BUCKET, Key=test_file.name)
s3_client.delete_bucket(Bucket=BUCKET)
def test_cp_force(self, s3_client, test_file):
BUCKET = "test-bucket"
BUCKET2 = "test-bucket2"
# create test buckets
s3_client.create_bucket(Bucket=BUCKET)
s3_client.create_bucket(Bucket=BUCKET2)
# add test file to buckets
s3_client.upload_file(str(test_file), BUCKET, test_file.name)
s3_client.upload_file(str(test_file), BUCKET2, test_file.name)
# pegasus-s3 cp --force s3://rynge@osgconnect/test-bucket/test_file s3://rynge@osgconnect/test-bucket2/test_file
parser, args = s3.parse_args([
"cp",
"--force",
"s3://rynge@osgconnect/{}/{}".format(BUCKET, test_file.name),
"s3://rynge@osgconnect/{}/{}".format(BUCKET2, test_file.name),
], )
s3.cp(args)
# ensure test_file copied into bucket
try:
s3_client.head_object(Bucket=BUCKET2, Key=test_file.name)
except sbotocore.exceptions.ClientError:
pytest.fail(
"s3://rynge@osgconnect/{}/{} should exist but does not".format(
BUCKET2, test_file.name))
# clean up buckets
s3_client.delete_object(Bucket=BUCKET, Key=test_file.name)
s3_client.delete_bucket(Bucket=BUCKET)
s3_client.delete_object(Bucket=BUCKET2, Key=test_file.name)
s3_client.delete_bucket(Bucket=BUCKET2)
def test_put_create_bucket(self, s3_client, test_file):
BUCKET = "test-bucket"
# ensure test bucket doesn't yet exist
try:
s3_client.head_bucket(Bucket=BUCKET)
pytest.fail(
"test-bucket should not yet exist as it is to be created by the put command"
)
except botocore.exceptions.ClientError:
pass
# pegasus-s3 put --create-bucket test_file s3://rynge@osgconnect/test-bucket/test_file
parser, args = s3.parse_args([
"put",
"--create-bucket",
str(test_file),
"s3://rynge@osgconnect/{}/{}".format(BUCKET, test_file.name),
])
s3.put(args)
# test
try:
s3_client.head_object(Bucket=BUCKET, Key=test_file.name)
except botocore.exceptions.ClientError:
pytest.fail("Key: {} should exist in Bucket: {}".format(
test_file.name, BUCKET))
# cleanup
s3_client.delete_object(Bucket=BUCKET, Key=test_file.name)
s3_client.delete_bucket(Bucket=BUCKET)
def test_rm_force(self, s3_client, test_file):
BUCKET = "test-bucket"
# create test bucket
s3_client.create_bucket(Bucket=BUCKET)
# ensure file is NOT there before test
try:
s3_client.head_object(Bucket=BUCKET, Key=test_file.name)
pytest.fail("File: {} should not be in bucket: {}".format(
test_file.name, BUCKET))
except botocore.exceptions.ClientError:
pass
# remove file
# pegasus-s3 rm s3://rynge@osgconnect/test-bucket/test_file
parser, args = s3.parse_args([
"rm",
"--force",
"s3://rynge@osgconnect/{}/{}".format(BUCKET, test_file.name),
])
s3.rm(args)
try:
s3_client.head_object(Bucket=BUCKET, Key=test_file.name)
except botocore.exceptions.ClientError:
# removed key and it should not exist
pass
# cleanup bucket
s3_client.delete_bucket(Bucket=BUCKET)
def test_rm_using_file(self, s3_client):
files = [
Path("tb1_f1"),
Path("tb1_f2"),
Path("tb2_f1"),
Path("tb2_f2")
]
# create test files
for f in files:
with f.open("w") as fp:
fp.write(f.name)
# mapping of keys to buckets
buckets_keys = {"test-bucket1": files[0:2], "test-bucket2": files[2:]}
# create test buckets and upload files
for b, keys in buckets_keys.items():
s3_client.create_bucket(Bucket=b)
for k in keys:
s3_client.upload_file(str(k), b, k.name)
# create rm file
rm_file = Path("rm_file")
with rm_file.open("w") as fp:
for b, keys in buckets_keys.items():
for k in keys:
fp.write("s3://rynge@osgconnect/{}/{}\n".format(b, k.name))
# remove file
# pegasus-s3 rm s3://rynge@osgconnect/test-bucket/test_file
parser, args = s3.parse_args(["rm", "--force", "--file", rm_file.name])
s3.rm(args)
# ensure keys were removed from bucket
for b, keys in buckets_keys.items():
for k in keys:
try:
s3_client.head_object(Bucket=b, Key=k.name)
pytest.fail(
"Key: {} should not exist in Bucket: {}".format(
k.name, b))
except botocore.exceptions.ClientError:
pass
# cleanup buckets
for b, keys in buckets_keys.items():
for k in keys:
s3_client.delete_object(Bucket=b, Key=k.name)
s3_client.delete_bucket(Bucket=b)
# remove test files
rm_file.unlink()
for f in files:
f.unlink()
def test_ls_buckets(self, s3_client, capsys):
resp = s3_client.list_buckets()
expected_buckets = {b["Name"] for b in resp["Buckets"]}
# pegasus-s3 ls s3://rynge@osgconnect
parser, args = s3.parse_args(["ls", "s3://rynge@osgconnect"])
s3.ls(args)
# get stdout
out, _ = capsys.readouterr()
retrieved_buckets = {b.strip() for b in out.split()}
assert expected_buckets == retrieved_buckets
def test_put_bucket_doesnt_exist(self, test_file):
BUCKET = "test-bucket"
# pegasus-s3 put test_file s3://rynge@osgconnect/test-bucket/test_file
parser, args = s3.parse_args([
"put",
str(test_file),
"s3://rynge@osgconnect/{}/{}".format(BUCKET, test_file.name),
])
with pytest.raises(Exception) as e:
s3.put(args)
assert "Failed to upload file: test_file" in str(e)
def test_mkdir(self, s3_client):
BUCKET = "test-bucket"
# pegasus-s3 mkdir s3://rynge@osgconnect/test-bucket
parser, args = s3.parse_args(
["mkdir", "s3://rynge@osgconnect/{}".format(BUCKET)])
s3.mkdir(args)
# ensure test-bucket has been created
resp = s3_client.list_buckets()
expected_buckets = {b["Name"] for b in resp["Buckets"]}
assert BUCKET in expected_buckets
# cleanup test-bucket
s3_client.delete_bucket(Bucket=BUCKET)
def test_mkdir_bucket_already_owned_by_user(self, s3_client, caplog):
BUCKET = "test-bucket"
# create test bucket before calling mkdir on same bucket
s3_client.create_bucket(Bucket=BUCKET)
# pegasus-s3 mkdir s3://rynge@osgconnect/test-bucket
parser, args = s3.parse_args(
["mkdir", "s3://rynge@osgconnect/{}".format(BUCKET)])
s3.mkdir(args)
# check that log message printed
assert caplog.record_tuples == [(
"root",
logging.WARNING,
"Bucket: test-bucket exists and is already owned by user: rynge@osgconnect",
)]
# cleanup test-bucket
s3_client.delete_bucket(Bucket=BUCKET)
def test_cp_create_dest_and_force(self, s3_client, test_file):
BUCKET = "test-bucket"
BUCKET2 = "test-bucket2"
# create test bucket
s3_client.create_bucket(Bucket=BUCKET)
# add test file to bucket
s3_client.upload_file(str(test_file), BUCKET, test_file.name)
# pegasus-s3 cp --create-dest --force s3://rynge@osgconnect/test-bucket/test_file s3://rynge@osgconnect/test-bucket2/test_file
parser, args = s3.parse_args([
"cp",
"--create-dest",
"--force",
"s3://rynge@osgconnect/{}/{}".format(BUCKET, test_file.name),
"s3://rynge@osgconnect/{}/{}".format(BUCKET2, test_file.name),
], )
s3.cp(args)
# ensure BUCKET2 has been created an test_file copied
resp = s3_client.list_buckets()
expected_buckets = {b["Name"] for b in resp["Buckets"]}
assert BUCKET2 in expected_buckets
try:
s3_client.head_object(Bucket=BUCKET2, Key=test_file.name)
except sbotocore.exceptions.ClientError:
pytest.fail(
"s3://rynge@osgconnect/{}/{} should exist but does not".format(
BUCKET2, test_file.name))
# clean up buckets
s3_client.delete_object(Bucket=BUCKET, Key=test_file.name)
s3_client.delete_bucket(Bucket=BUCKET)
s3_client.delete_object(Bucket=BUCKET2, Key=test_file.name)
s3_client.delete_bucket(Bucket=BUCKET2)
def test_ls_keys(self, s3_client, test_file, capsys):
BUCKET = "test-bucket"
# create test bucket
s3_client.create_bucket(Bucket=BUCKET)
# add test file to bucket
s3_client.upload_file(str(test_file), BUCKET, test_file.name)
# pegasus-s3 ls s3://rynge@osgconnect/test-bucket
parser, args = s3.parse_args(
["ls", "s3://rynge@osgconnect/test-bucket"])
s3.ls(args)
# get stdout
out, _ = capsys.readouterr()
# test
assert out.strip() == test_file.name
# cleanup bucket
s3_client.delete_object(Bucket=BUCKET, Key=test_file.name)
s3_client.delete_bucket(Bucket=BUCKET)
def test_put(self, s3_client, test_file):
BUCKET = "test-bucket"
# create test bucket
s3_client.create_bucket(Bucket=BUCKET)
# pegasus-s3 put test_file s3://rynge@osgconnect/test-bucket/test_file
paser, args = s3.parse_args([
"put",
str(test_file),
"s3://rynge@osgconnect/{}/{}".format(BUCKET, test_file.name),
], )
s3.put(args)
# test
try:
s3_client.head_object(Bucket=BUCKET, Key=test_file.name)
except botocore.exceptions.ClientError:
pytest.fail("Key: {} should exist in Bucket: {}".format(
test_file.name, BUCKET))
# cleanup bucket
s3_client.delete_object(Bucket=BUCKET, Key=test_file.name)
s3_client.delete_bucket(Bucket=BUCKET)
