def compare_acl(test_set): acl, subject, expected, unexpected = test_set conn = boto.connect_s3() proxy = RadulaProxy(connection=conn) radu = Radula(connection=conn) bucket = conn.create_bucket(TEST_BUCKET) # give something to download args = vars(_parse_args(['up'])) args.update({ "subject": TEST_FILE, "target": REMOTE_FILE }) proxy.upload(**args) # for sake of testing, ensure equality key_name = os.path.basename(REMOTE_FILE) key = bucket.get_key(key_name) key.set_canned_acl(acl) args = vars(_parse_args(['compare-acl', subject, acl])) radu.compare_acl(**args) out = [line.strip() for line in sys.stdout.getvalue().split("\n")] sys.stdout.truncate(0) for msg in expected: errmsg = "Expecting log message containing '{0}'".format(msg) assert_in(msg, out, msg=errmsg) for msg in unexpected: errmsg = "Not expecting log message containing '{0}'".format(msg) assert_not_in(msg, out, msg=errmsg)
def compare_acl(test_set): acl, subject, expected, unexpected = test_set conn = boto.connect_s3() proxy = RadulaProxy(connection=conn) radu = Radula(connection=conn) bucket = conn.create_bucket(TEST_BUCKET) # give something to download args = vars(_parse_args(['up'])) args.update({"subject": TEST_FILE, "target": REMOTE_FILE}) proxy.upload(**args) # for sake of testing, ensure equality key_name = os.path.basename(REMOTE_FILE) key = bucket.get_key(key_name) key.set_canned_acl(acl) args = vars(_parse_args(['compare-acl', subject, acl])) radu.compare_acl(**args) out = [line.strip() for line in sys.stdout.getvalue().split("\n")] sys.stdout.truncate(0) for msg in expected: errmsg = "Expecting log message containing '{0}'".format(msg) assert_in(msg, out, msg=errmsg) for msg in unexpected: errmsg = "Not expecting log message containing '{0}'".format(msg) assert_not_in(msg, out, msg=errmsg)
def allow_user(method, test_set): subject, flags, expected, unexpected = test_set conn = boto.connect_s3() proxy = RadulaProxy(connection=conn) radu = Radula(connection=conn) proxy.make_bucket(subject=TEST_BUCKET) # give something to download args = vars(_parse_args(['up'])) args.update({ "subject": TEST_FILE, "target": subject }) proxy.upload(**args) sys.stdout.truncate(0) flags = flags.split(' ') args = [arg for arg in flags + [method, ALT_USER, subject] if arg] args = vars(_parse_args(args)) getattr(radu, args.get("command").replace('-', '_'))(**args) out = [line.strip() for line in sys.stdout.getvalue().split("\n")] sys.stdout.truncate(0) for msg in expected: errmsg = "Expecting log message containing '{0}'".format(msg) assert_in(msg, out, msg=errmsg) for msg in unexpected: errmsg = "Not expecting log message containing '{0}'".format(msg) assert_not_in(msg, out, msg=errmsg)
def key_slash_test(): radu = RadulaProxy(connection=boto.connect_s3()) radu.make_bucket(subject=TEST_BUCKET) # give something to download args = vars(_parse_args(['up'])) expected = [] keys_added = 3 # some to find for i in xrange(keys_added): remote_file = os.path.join(TEST_BUCKET, "find", os.path.basename(TEST_FILE)) + str(i) expected.append(remote_file) args.update({"subject": TEST_FILE, "target": remote_file}) radu.upload(**args) # some to miss for i in xrange(keys_added): remote_file = os.path.join(TEST_BUCKET, "miss", os.path.basename(TEST_FILE)) + str(i) args.update({"subject": TEST_FILE, "target": remote_file}) radu.upload(**args) sys.stdout.truncate(0) # bucket/find/ (trailing slash) radu.keys(subject=os.path.join(TEST_BUCKET, 'find', '')) keys = [k.strip() for k in sys.stdout.getvalue().strip().split("\n")] for expected_key in expected: expected_key = os.path.join('find', os.path.basename(expected_key)) msg = "Expecting output containing '{0}'".format(expected_key) assert_in(expected_key, keys, msg=msg) assert_equal(keys_added, len(keys), "Expected to have %d keys, got %d" % (keys_added, len(keys)))
def local_md5_threads_test(): radu = RadulaProxy(connection=boto.connect_s3()) args = vars(_parse_args(["local-md5"])) args.update({ "subject": 'test_file_not_found__intentional' }) radu.local_md5(**args)
def up_method(method, test_set): handler = TestHandler(Matcher()) logger = logging.getLogger() logger.addHandler(handler) radu = RadulaProxy(connection=boto.connect_s3()) radu.make_bucket(subject=TEST_BUCKET) sys.stdout.truncate(0) args = vars(_parse_args(["-y", method])) args.update(test_set) getattr(radu, args.get("command"))(**args) assert_false(handler.matches(levelno=logging.ERROR)) assert_false(handler.matches(levelno=logging.WARNING)) assert_true(handler.matches(levelno=logging.INFO)) msgs = [ "uploading", "tests.s3.amazonaws.com/data.txt", "Checksum Verified!" ] fmt = "Expecting log message containing '{0}'" for msg in msgs: assert_true( handler.matches(message=msg), msg=fmt.format(msg) )
def set_acl_subject(test_set, acl): subject, expected, unexpected = test_set conn = boto.connect_s3() proxy = RadulaProxy(connection=conn) radu = Radula(connection=conn) proxy.make_bucket(subject=TEST_BUCKET) # give something to download args = vars(_parse_args(['up'])) args.update({ "subject": TEST_FILE, "target": subject }) proxy.upload(**args) sys.stdout.truncate(0) args = vars(_parse_args(['set-acl', subject, acl])) radu.set_acl(**args) out = [line.strip() for line in sys.stdout.getvalue().split("\n")] sys.stdout.truncate(0) for msg in expected: errmsg = "Expecting log message containing '{0}'".format(msg) assert_in(msg, out, msg=errmsg) for msg in unexpected: errmsg = "Not expecting log message containing '{0}'".format(msg) assert_not_in(msg, out, msg=errmsg)
def up_method_fail(method, test_set): radu = RadulaProxy(connection=boto.connect_s3()) bucket = TEST_BUCKET radu.make_bucket(subject=bucket) sys.stdout.truncate(0) getattr(radu, method)(**test_set)
def list_method(method, buckets): radu = RadulaProxy(connection=boto.connect_s3()) for bucket in buckets: radu.make_bucket(subject=bucket) sys.stdout.truncate(0) getattr(radu, method)() out = sys.stdout.getvalue().strip() assert_equal(sorted(buckets), sorted([b for b in out.split("\n") if b]))
def local_md5_threads_test(): # ensure original text with open(TEST_FILE, 'w') as test_file: test_file.write("data") radu = RadulaProxy(connection=boto.connect_s3()) # 'threads' needed args = vars(_parse_args(["local-md5", "-t", "2"])) args.update({"subject": TEST_FILE}) radu.local_md5(**args) out = sys.stdout.getvalue().strip() assert_equal('8d777f385d3dfec8815d20f7496026dc', out)
def local_md5_threads_test(): # ensure original text with open(TEST_FILE, 'w') as test_file: test_file.write("data") radu = RadulaProxy(connection=boto.connect_s3()) # 'threads' needed args = vars(_parse_args(["local-md5", "-t", "2"])) args.update({ "subject": TEST_FILE }) radu.local_md5(**args) out = sys.stdout.getvalue().strip() assert_equal('8d777f385d3dfec8815d20f7496026dc', out)
def dl_method_fail(method, test_set): radu = RadulaProxy(connection=boto.connect_s3()) radu.make_bucket(subject=TEST_BUCKET) # give something to download args = vars(_parse_args(['up'])) args.update({ "subject": TEST_FILE, "target": REMOTE_FILE }) radu.upload(**args) sys.stdout.truncate(0) getattr(radu, method)(**test_set)
def rm_method(method): radu = RadulaProxy(connection=boto.connect_s3()) radu.make_bucket(subject=TEST_BUCKET) # give something to rm args = vars(_parse_args(['up'])) expected = [] for i in xrange(3): remote_file = REMOTE_FILE + str(i) expected.append(remote_file) args.update({ "subject": TEST_FILE, "target": remote_file }) radu.upload(**args) while len(expected): remove_file = expected.pop() sys.stdout.truncate(0) getattr(radu, method)(subject=remove_file) radu.keys(subject=TEST_BUCKET) keys = [k.strip() for k in sys.stdout.getvalue().strip().split("\n")] absent_key = os.path.basename(remove_file) assert_not_in(absent_key, keys, msg="Expecting absence of key mention '{0}'".format(absent_key)) for expected_key in expected: expected_key = os.path.basename(expected_key) assert_in(expected_key, keys, msg="Expecting output containing '{0}'".format(expected_key))
def verify_test(): # ensure original text with open(TEST_FILE, 'w') as test_file: test_file.write("data") radu = RadulaProxy(connection=boto.connect_s3()) radu.make_bucket(subject=TEST_BUCKET) # give something to download args = vars(_parse_args(['up'])) args.update({"subject": TEST_FILE, "target": REMOTE_FILE}) radu.upload(**args) sys.stdout.truncate(0) handler = TestHandler(Matcher()) logger = logging.getLogger() logger.addHandler(handler) # 'threads' needed args = vars(_parse_args(["verify"])) args.update({"subject": TEST_FILE, "target": REMOTE_FILE}) radu.verify(**args) assert_false(handler.matches(levelno=logging.ERROR)) assert_false(handler.matches(levelno=logging.WARNING)) assert_true(handler.matches(levelno=logging.INFO)) msgs = [ "Checksum Verified!", '8d777f385d3dfec8815d20f7496026dc', ] for msg in msgs: fmt = "Expecting log message containing '{0}'" assert_true(handler.matches(message=msg), msg=fmt.format(msg))
def set_acl_canned_fail(acl): conn = boto.connect_s3() proxy = RadulaProxy(connection=conn) radu = Radula(connection=conn) proxy.make_bucket(subject=TEST_BUCKET) # give something to download args = vars(_parse_args(['up'])) args.update({ "subject": TEST_FILE, "target": REMOTE_FILE }) proxy.upload(**args) sys.stdout.truncate(0) args = vars(_parse_args(['set-acl', REMOTE_FILE, acl])) radu.set_acl(**args)
def dl_method(method, test_set): radu = RadulaProxy(connection=boto.connect_s3()) radu.make_bucket(subject=TEST_BUCKET) # give something to download args = vars(_parse_args(['up'])) args.update({ "subject": TEST_FILE, "target": REMOTE_FILE }) radu.upload(**args) sys.stdout.truncate(0) args = vars(_parse_args([method, '-f'])) args.update(test_set) getattr(radu, method)(**args) out = sys.stdout.getvalue().strip() msgs = [ "Download Progress", "100.00%", ] for msg in msgs: assert_in(msg, out, msg="Expecting log message containing '{0}'".format(msg)) target = test_set.get("target", TEST_FILE) assert_true(os.path.isfile(target)) if target != TEST_FILE: os.remove(target)
def allow_user(method, test_set): subject, flags, expected, unexpected = test_set conn = boto.connect_s3() proxy = RadulaProxy(connection=conn) radu = Radula(connection=conn) proxy.make_bucket(subject=TEST_BUCKET) # give something to download args = vars(_parse_args(['up'])) args.update({"subject": TEST_FILE, "target": subject}) proxy.upload(**args) sys.stdout.truncate(0) flags = flags.split(' ') args = [arg for arg in flags + [method, ALT_USER, subject] if arg] args = vars(_parse_args(args)) getattr(radu, args.get("command").replace('-', '_'))(**args) out = [line.strip() for line in sys.stdout.getvalue().split("\n")] sys.stdout.truncate(0) for msg in expected: errmsg = "Expecting log message containing '{0}'".format(msg) if msg not in out: p = 1 assert_in(msg, out, msg=errmsg) for msg in unexpected: errmsg = "Not expecting log message containing '{0}'".format(msg) assert_not_in(msg, out, msg=errmsg)
def set_acl_subject(test_set, acl): subject, expected, unexpected = test_set conn = boto.connect_s3() proxy = RadulaProxy(connection=conn) radu = Radula(connection=conn) proxy.make_bucket(subject=TEST_BUCKET) # give something to download args = vars(_parse_args(['up'])) args.update({"subject": TEST_FILE, "target": subject}) proxy.upload(**args) sys.stdout.truncate(0) args = vars(_parse_args(['set-acl', subject, acl])) radu.set_acl(**args) out = [line.strip() for line in sys.stdout.getvalue().split("\n")] sys.stdout.truncate(0) for msg in expected: errmsg = "Expecting log message containing '{0}'".format(msg) assert_in(msg, out, msg=errmsg) for msg in unexpected: errmsg = "Not expecting log message containing '{0}'".format(msg) assert_not_in(msg, out, msg=errmsg)
def dl_method(method, test_set): handler = TestHandler(Matcher()) logger = logging.getLogger() logger.addHandler(handler) radu = RadulaProxy(connection=boto.connect_s3()) radu.make_bucket(subject=TEST_BUCKET) # give something to download args = vars(_parse_args(['up'])) args.update({"subject": TEST_FILE, "target": REMOTE_FILE}) radu.upload(**args) sys.stdout.truncate(0) args = vars(_parse_args([method])) args.update(test_set) args.update({"force": True}) getattr(radu, args.get("command"))(**args) out = sys.stdout.getvalue().strip() msgs = [ "Download Progress", "100.00%", ] for msg in msgs: fmt = "Expecting log message containing '{0}'".format(msg) assert_true(handler.matches(message=msg), msg=fmt.format(msg)) target = test_set.get("target", TEST_FILE) assert_true(os.path.isfile(target)) if target != TEST_FILE: os.remove(target)
def copy_method(method): handler = TestHandler(Matcher()) logger = logging.getLogger() logger.addHandler(handler) radu = RadulaProxy(connection=boto.connect_s3()) radu.make_bucket(subject=TEST_BUCKET) # give something to download args = vars(_parse_args(['up'])) args.update({ "subject": TEST_FILE, "target": REMOTE_FILE }) radu.upload(**args) sys.stdout.truncate(0) # 'threads' needed args = vars(_parse_args(['-y', method])) target_file = REMOTE_FILE + '-copy' args.update({ "subject": REMOTE_FILE, "target": target_file }) radu.streaming_copy(**args) msgs = [ "Finished uploading", "tests.s3.amazonaws.com/data.txt", "Download URL", "Key data matches!" ] for msg in msgs: fmt = "Expecting log message containing '{0}'" assert_true(handler.matches(message=msg), msg=fmt.format(msg)) radu.keys(subject=TEST_BUCKET) keys = [k.strip() for k in sys.stdout.getvalue().strip().split("\n")] expected = [REMOTE_FILE, target_file] fmt = "Expecting output containing '{0}'" for expected_key in expected: expected_key = os.path.basename(expected_key) assert_in(expected_key, keys, msg=fmt.format(expected_key))
def up_method(method, test_set): handler = TestHandler(Matcher()) logger = logging.getLogger() logger.addHandler(handler) radu = RadulaProxy(connection=boto.connect_s3()) radu.make_bucket(subject=TEST_BUCKET) sys.stdout.truncate(0) args = vars(_parse_args(["-y", method])) args.update(test_set) getattr(radu, args.get("command"))(**args) assert_false(handler.matches(levelno=logging.ERROR)) assert_false(handler.matches(levelno=logging.WARNING)) assert_true(handler.matches(levelno=logging.INFO)) msgs = [ "uploading", "tests.s3.amazonaws.com/data.txt", "Checksum Verified!" ] fmt = "Expecting log message containing '{0}'" for msg in msgs: assert_true(handler.matches(message=msg), msg=fmt.format(msg))
def copy_method(method): handler = TestHandler(Matcher()) logger = logging.getLogger() logger.addHandler(handler) radu = RadulaProxy(connection=boto.connect_s3()) radu.make_bucket(subject=TEST_BUCKET) # give something to download args = vars(_parse_args(['up'])) args.update({"subject": TEST_FILE, "target": REMOTE_FILE}) radu.upload(**args) sys.stdout.truncate(0) # 'threads' needed args = vars(_parse_args(['-y', method])) target_file = REMOTE_FILE + '-copy' args.update({"subject": REMOTE_FILE, "target": target_file}) radu.streaming_copy(**args) msgs = [ "Finished uploading", "tests.s3.amazonaws.com/data.txt", "Download URL", "Checksum Verified!" ] for msg in msgs: fmt = "Expecting log message containing '{0}'" assert_true(handler.matches(message=msg), msg=fmt.format(msg)) radu.keys(subject=TEST_BUCKET) keys = [k.strip() for k in sys.stdout.getvalue().strip().split("\n")] expected = [REMOTE_FILE, target_file] fmt = "Expecting output containing '{0}'" for expected_key in expected: expected_key = os.path.basename(expected_key) assert_in(expected_key, keys, msg=fmt.format(expected_key))
def dl_method_fail(method, test_set): radu = RadulaProxy(connection=boto.connect_s3()) radu.make_bucket(subject=TEST_BUCKET) # give something to download args = vars(_parse_args(['up'])) args.update({"subject": TEST_FILE, "target": REMOTE_FILE}) radu.upload(**args) sys.stdout.truncate(0) getattr(radu, method)(**test_set)
def remote_md5_test(): # ensure original text with open(TEST_FILE, 'w') as test_file: test_file.write("data") radu = RadulaProxy(connection=boto.connect_s3()) radu.make_bucket(subject=TEST_BUCKET) # give something to download args = vars(_parse_args(['up'])) args.update({"subject": TEST_FILE, "target": REMOTE_FILE}) radu.upload(**args) sys.stdout.truncate(0) # 'threads' needed args = vars(_parse_args(["remote-md5", "-t", "2"])) args.update({"subject": REMOTE_FILE}) radu.remote_md5(**args) out = sys.stdout.getvalue().strip() assert_equal('8d777f385d3dfec8815d20f7496026dc', out)
def bucket_info_test(): radu = RadulaProxy(connection=boto.connect_s3()) radu.make_bucket(subject=TEST_BUCKET) sys.stdout.truncate(0) radu.info(subject=TEST_BUCKET) out = sys.stdout.getvalue().strip() info = json.loads(out) for k in ('keys', 'size_human', 'size'): assert_in(k, info) key_info = info.get("keys") for k in ('count', 'largest', 'newest', 'oldest'): assert_in(k, key_info)
def key_info_test(): radu = RadulaProxy(connection=boto.connect_s3()) radu.make_bucket(subject=TEST_BUCKET) # give something to info args = vars(_parse_args(['up'])) remote_file = REMOTE_FILE args.update({ "subject": TEST_FILE, "target": remote_file }) radu.upload(**args) sys.stdout.truncate(0) radu.info(subject=REMOTE_FILE) out = sys.stdout.getvalue().strip() info = json.loads(out) for k in ('content_length', 'owner', 'size', 'bucket'): assert_in(k, info) assert_equal(info.get("bucket"), TEST_BUCKET)
def key_test(): radu = RadulaProxy(connection=boto.connect_s3()) radu.make_bucket(subject=TEST_BUCKET) # give something to download args = vars(_parse_args(['up'])) expected = [] for i in xrange(3): remote_file = REMOTE_FILE + str(i) expected.append(remote_file) args.update({"subject": TEST_FILE, "target": remote_file}) radu.upload(**args) sys.stdout.truncate(0) radu.keys(subject=TEST_BUCKET) keys = [k.strip() for k in sys.stdout.getvalue().strip().split("\n")] for expected_key in expected: expected_key = os.path.basename(expected_key) msg = "Expecting output containing '{0}'".format(expected_key) assert_in(expected_key, keys, msg=msg)
def remote_md5_fail_test(): # ensure original text with open(TEST_FILE, 'w') as test_file: test_file.write("data") radu = RadulaProxy(connection=boto.connect_s3()) radu.make_bucket(subject=TEST_BUCKET) sys.stdout.truncate(0) # 'threads' needed args = vars(_parse_args(["remote-md5", "-t", "2"])) args.update({ "subject": os.path.join(TEST_BUCKET, 'test_file_not_found__intentional') }) radu.remote_md5(**args)
def set_acl_canned_fail(acl): conn = boto.connect_s3() proxy = RadulaProxy(connection=conn) radu = Radula(connection=conn) proxy.make_bucket(subject=TEST_BUCKET) # give something to download args = vars(_parse_args(['up'])) args.update({"subject": TEST_FILE, "target": REMOTE_FILE}) proxy.upload(**args) sys.stdout.truncate(0) args = vars(_parse_args(['set-acl', REMOTE_FILE, acl])) radu.set_acl(**args)
def verify_test(): # ensure original text with open(TEST_FILE, 'w') as test_file: test_file.write("data") radu = RadulaProxy(connection=boto.connect_s3()) radu.make_bucket(subject=TEST_BUCKET) # give something to download args = vars(_parse_args(['up'])) args.update({ "subject": TEST_FILE, "target": REMOTE_FILE }) radu.upload(**args) sys.stdout.truncate(0) handler = TestHandler(Matcher()) logger = logging.getLogger() logger.addHandler(handler) # 'threads' needed args = vars(_parse_args(["verify"])) args.update({ "subject": TEST_FILE, "target": REMOTE_FILE }) radu.verify(**args) assert_false(handler.matches(levelno=logging.ERROR)) assert_false(handler.matches(levelno=logging.WARNING)) assert_true(handler.matches(levelno=logging.INFO)) msgs = [ "Checksum Verified!", '8d777f385d3dfec8815d20f7496026dc', ] for msg in msgs: fmt = "Expecting log message containing '{0}'" assert_true(handler.matches(message=msg), msg=fmt.format(msg))
def key_test(): radu = RadulaProxy(connection=boto.connect_s3()) radu.make_bucket(subject=TEST_BUCKET) # give something to download args = vars(_parse_args(['up'])) expected = [] for i in xrange(3): remote_file = REMOTE_FILE + str(i) expected.append(remote_file) args.update({ "subject": TEST_FILE, "target": remote_file }) radu.upload(**args) sys.stdout.truncate(0) radu.keys(subject=TEST_BUCKET) keys = [k.strip() for k in sys.stdout.getvalue().strip().split("\n")] for expected_key in expected: expected_key = os.path.basename(expected_key) assert_in(expected_key, keys, msg="Expecting output containing '{0}'".format(expected_key))
def remote_md5_test(): # ensure original text with open(TEST_FILE, 'w') as test_file: test_file.write("data") radu = RadulaProxy(connection=boto.connect_s3()) radu.make_bucket(subject=TEST_BUCKET) # give something to download args = vars(_parse_args(['up'])) args.update({ "subject": TEST_FILE, "target": REMOTE_FILE }) radu.upload(**args) sys.stdout.truncate(0) # 'threads' needed args = vars(_parse_args(["remote-md5", "-t", "2"])) args.update({ "subject": REMOTE_FILE }) radu.remote_md5(**args) out = sys.stdout.getvalue().strip() assert_equal('8d777f385d3dfec8815d20f7496026dc', out)
def test_make_bucket_fail(): radu = RadulaProxy(connection=boto.connect_s3()) radu.make_bucket(subject="")
def test_make_bucket(): radu = RadulaProxy(connection=boto.connect_s3()) radu.make_bucket(subject=TEST_BUCKET) out = sys.stdout.getvalue().strip() assert_equal('Created bucket: tests', out)
def verify_fail_method(test_set): radu = RadulaProxy(connection=boto.connect_s3()) args = vars(_parse_args(["verify"])) args.update(test_set) radu.verify(**args)
def remote_md5_nosubject_test(): radu = RadulaProxy(connection=boto.connect_s3()) args = vars(_parse_args(["remote-md5"])) radu.remote_md5(**args)
def local_md5_nosubject_test(): radu = RadulaProxy(connection=boto.connect_s3()) args = vars(_parse_args(["local-md5"])) radu.local_md5(**args)
def bucket_info_fail_test(): RadulaProxy(connection=boto.connect_s3()).info(subject=TEST_BUCKET)
def key_info_bad_subject_test(): radu = RadulaProxy(connection=boto.connect_s3()) radu.make_bucket(subject=TEST_BUCKET) radu.info(subject=REMOTE_FILE)
def keys_fail_test(): RadulaProxy(connection=boto.connect_s3()).keys(subject=TEST_BUCKET)