def test_ps_s3_notification_records():
""" test s3 records fetching """
zones, ps_zones = init_env()
bucket_name = gen_bucket_name()
# create bucket on the first of the rados zones
bucket = zones[0].create_bucket(bucket_name)
# wait for sync
zone_meta_checkpoint(ps_zones[0].zone)
# create topic
topic_name = bucket_name + TOPIC_SUFFIX
topic_conf = PSTopic(ps_zones[0].conn, topic_name)
_, status = topic_conf.set_config()
assert_equal(status / 100, 2)
# create s3 notification
notification_name = bucket_name + NOTIFICATION_SUFFIX
topic_arn = 'arn:aws:sns:::' + topic_name
s3_notification_conf = PSNotificationS3(ps_zones[0].conn, bucket_name,
notification_name, topic_arn,
['s3:ObjectCreated:*'])
response, status = s3_notification_conf.set_config()
assert_equal(status / 100, 2)
zone_meta_checkpoint(ps_zones[0].zone)
# get auto-generated subscription
sub_conf = PSSubscription(ps_zones[0].conn, notification_name, topic_name)
_, status = sub_conf.get_config()
assert_equal(status / 100, 2)
# create objects in the bucket
number_of_objects = 10
for i in range(number_of_objects):
key = bucket.new_key(str(i))
key.set_contents_from_string('bar')
# wait for sync
zone_bucket_checkpoint(ps_zones[0].zone, zones[0].zone, bucket_name)
# get the events from the subscription
result, _ = sub_conf.get_events()
parsed_result = json.loads(result)
for record in parsed_result['Records']:
log.debug(record)
keys = list(bucket.list())
# TODO: set exact_match to true
verify_s3_records_by_elements(parsed_result['Records'],
keys,
exact_match=False)
# cleanup
_, status = s3_notification_conf.del_config()
topic_conf.del_config()
# delete the keys
for key in bucket.list():
key.delete()
zones[0].delete_bucket(bucket_name)
def test_ps_s3_notification_records():
""" test s3 records fetching """
zones, ps_zones = init_env()
bucket_name = gen_bucket_name()
# create bucket on the first of the rados zones
bucket = zones[0].create_bucket(bucket_name)
# wait for sync
zone_meta_checkpoint(ps_zones[0].zone)
# create topic
topic_name = bucket_name + TOPIC_SUFFIX
topic_conf = PSTopic(ps_zones[0].conn, topic_name)
_, status = topic_conf.set_config()
assert_equal(status/100, 2)
# create s3 notification
notification_name = bucket_name + NOTIFICATION_SUFFIX
topic_arn = 'arn:aws:sns:::' + topic_name
s3_notification_conf = PSNotificationS3(ps_zones[0].conn, bucket_name,
notification_name, topic_arn, ['s3:ObjectCreated:*'])
response, status = s3_notification_conf.set_config()
assert_equal(status/100, 2)
zone_meta_checkpoint(ps_zones[0].zone)
# get auto-generated subscription
sub_conf = PSSubscription(ps_zones[0].conn, notification_name,
topic_name)
_, status = sub_conf.get_config()
assert_equal(status/100, 2)
# create objects in the bucket
number_of_objects = 10
for i in range(number_of_objects):
key = bucket.new_key(str(i))
key.set_contents_from_string('bar')
# wait for sync
zone_bucket_checkpoint(ps_zones[0].zone, zones[0].zone, bucket_name)
# get the events from the subscription
result, _ = sub_conf.get_events()
parsed_result = json.loads(result)
for record in parsed_result['Records']:
log.debug(record)
keys = list(bucket.list())
# TODO: set exact_match to true
verify_s3_records_by_elements(parsed_result['Records'], keys, exact_match=False)
# cleanup
_, status = s3_notification_conf.del_config()
topic_conf.del_config()
# delete the keys
for key in bucket.list():
key.delete()
zones[0].delete_bucket(bucket_name)
def test_ps_subscription():
""" test set/get/delete of subscription """
zones, ps_zones = init_env()
bucket_name = gen_bucket_name()
topic_name = bucket_name + TOPIC_SUFFIX
# create topic
topic_conf = PSTopic(ps_zones[0].conn, topic_name)
topic_conf.set_config()
# create bucket on the first of the rados zones
bucket = zones[0].create_bucket(bucket_name)
# wait for sync
zone_meta_checkpoint(ps_zones[0].zone)
# create notifications
notification_conf = PSNotification(ps_zones[0].conn, bucket_name,
topic_name)
_, status = notification_conf.set_config()
assert_equal(status / 100, 2)
# create subscription
sub_conf = PSSubscription(ps_zones[0].conn, bucket_name + SUB_SUFFIX,
topic_name)
_, status = sub_conf.set_config()
assert_equal(status / 100, 2)
# get the subscription
result, _ = sub_conf.get_config()
parsed_result = json.loads(result)
assert_equal(parsed_result['topic'], topic_name)
# create objects in the bucket
number_of_objects = 10
for i in range(number_of_objects):
key = bucket.new_key(str(i))
key.set_contents_from_string('bar')
# wait for sync
zone_bucket_checkpoint(ps_zones[0].zone, zones[0].zone, bucket_name)
# get the create events from the subscription
result, _ = sub_conf.get_events()
parsed_result = json.loads(result)
for event in parsed_result['events']:
log.debug('Event: objname: "' + str(event['info']['key']['name']) +
'" type: "' + str(event['event']) + '"')
keys = list(bucket.list())
# TODO: set exact_match to true
verify_events_by_elements(parsed_result['events'], keys, exact_match=False)
# delete objects from the bucket
for key in bucket.list():
key.delete()
# wait for sync
zone_meta_checkpoint(ps_zones[0].zone)
zone_bucket_checkpoint(ps_zones[0].zone, zones[0].zone, bucket_name)
# get the delete events from the subscriptions
result, _ = sub_conf.get_events()
for event in parsed_result['events']:
log.debug('Event: objname: "' + str(event['info']['key']['name']) +
'" type: "' + str(event['event']) + '"')
# TODO: check deletions
# verify_events_by_elements(parsed_result['events'], keys, exact_match=False, deletions=True)
# we should see the creations as well as the deletions
# delete subscription
_, status = sub_conf.del_config()
assert_equal(status / 100, 2)
result, status = sub_conf.get_config()
parsed_result = json.loads(result)
assert_equal(parsed_result['topic'], '')
# TODO should return 404
# assert_equal(status, 404)
# cleanup
notification_conf.del_config()
topic_conf.del_config()
zones[0].delete_bucket(bucket_name)
def test_ps_s3_notification_low_level():
""" test low level implementation of s3 notifications """
zones, ps_zones = init_env()
bucket_name = gen_bucket_name()
# create bucket on the first of the rados zones
bucket = zones[0].create_bucket(bucket_name)
# wait for sync
zone_meta_checkpoint(ps_zones[0].zone)
# create topic
topic_name = bucket_name + TOPIC_SUFFIX
topic_conf = PSTopic(ps_zones[0].conn, topic_name)
_, status = topic_conf.set_config()
assert_equal(status / 100, 2)
# create s3 notification
notification_name = bucket_name + NOTIFICATION_SUFFIX
generated_topic_name = notification_name + '_' + topic_name
topic_arn = 'arn:aws:sns:::' + topic_name
s3_notification_conf = PSNotificationS3(ps_zones[0].conn, bucket_name,
notification_name, topic_arn,
['s3:ObjectCreated:*'])
response, status = s3_notification_conf.set_config()
assert_equal(status / 100, 2)
zone_meta_checkpoint(ps_zones[0].zone)
# get auto-generated topic
generated_topic_conf = PSTopic(ps_zones[0].conn, generated_topic_name)
result, status = generated_topic_conf.get_config()
parsed_result = json.loads(result)
assert_equal(status / 100, 2)
assert_equal(parsed_result['topic']['name'], generated_topic_name)
# get auto-generated notification
notification_conf = PSNotification(ps_zones[0].conn, bucket_name,
generated_topic_name)
result, status = notification_conf.get_config()
parsed_result = json.loads(result)
assert_equal(status / 100, 2)
assert_equal(len(parsed_result['topics']), 1)
# get auto-generated subscription
sub_conf = PSSubscription(ps_zones[0].conn, notification_name,
generated_topic_name)
result, status = sub_conf.get_config()
parsed_result = json.loads(result)
assert_equal(status / 100, 2)
assert_equal(parsed_result['topic'], generated_topic_name)
# delete s3 notification
_, status = s3_notification_conf.del_config(all_notifications=False)
assert_equal(status / 100, 2)
# delete topic
_, status = topic_conf.del_config()
assert_equal(status / 100, 2)
# verify low-level cleanup
_, status = generated_topic_conf.get_config()
assert_equal(status, 404)
result, status = notification_conf.get_config()
parsed_result = json.loads(result)
assert_equal(len(parsed_result['topics']), 0)
# TODO should return 404
# assert_equal(status, 404)
result, status = sub_conf.get_config()
parsed_result = json.loads(result)
assert_equal(parsed_result['topic'], '')
# TODO should return 404
# assert_equal(status, 404)
# cleanup
topic_conf.del_config()
# delete the bucket
zones[0].delete_bucket(bucket_name)
def test_ps_subscription():
""" test set/get/delete of subscription """
zones, ps_zones = init_env()
bucket_name = gen_bucket_name()
topic_name = bucket_name+TOPIC_SUFFIX
# create topic
topic_conf = PSTopic(ps_zones[0].conn, topic_name)
topic_conf.set_config()
# create bucket on the first of the rados zones
bucket = zones[0].create_bucket(bucket_name)
# wait for sync
zone_meta_checkpoint(ps_zones[0].zone)
# create notifications
notification_conf = PSNotification(ps_zones[0].conn, bucket_name,
topic_name)
_, status = notification_conf.set_config()
assert_equal(status/100, 2)
# create subscription
sub_conf = PSSubscription(ps_zones[0].conn, bucket_name+SUB_SUFFIX,
topic_name)
_, status = sub_conf.set_config()
assert_equal(status/100, 2)
# get the subscription
result, _ = sub_conf.get_config()
parsed_result = json.loads(result)
assert_equal(parsed_result['topic'], topic_name)
# create objects in the bucket
number_of_objects = 10
for i in range(number_of_objects):
key = bucket.new_key(str(i))
key.set_contents_from_string('bar')
# wait for sync
zone_bucket_checkpoint(ps_zones[0].zone, zones[0].zone, bucket_name)
# get the create events from the subscription
result, _ = sub_conf.get_events()
parsed_result = json.loads(result)
for event in parsed_result['events']:
log.debug('Event: objname: "' + str(event['info']['key']['name']) + '" type: "' + str(event['event']) + '"')
keys = list(bucket.list())
# TODO: set exact_match to true
verify_events_by_elements(parsed_result['events'], keys, exact_match=False)
# delete objects from the bucket
for key in bucket.list():
key.delete()
# wait for sync
zone_meta_checkpoint(ps_zones[0].zone)
zone_bucket_checkpoint(ps_zones[0].zone, zones[0].zone, bucket_name)
# get the delete events from the subscriptions
result, _ = sub_conf.get_events()
for event in parsed_result['events']:
log.debug('Event: objname: "' + str(event['info']['key']['name']) + '" type: "' + str(event['event']) + '"')
# TODO: check deletions
# verify_events_by_elements(parsed_result['events'], keys, exact_match=False, deletions=True)
# we should see the creations as well as the deletions
# delete subscription
_, status = sub_conf.del_config()
assert_equal(status/100, 2)
result, _ = sub_conf.get_config()
parsed_result = json.loads(result)
assert_equal(parsed_result['topic'], '')
# TODO should return "no-key" instead
# assert_equal(parsed_result['Code'], 'NoSuchKey')
# cleanup
notification_conf.del_config()
topic_conf.del_config()
zones[0].delete_bucket(bucket_name)
def test_ps_s3_notification_low_level():
""" test low level implementation of s3 notifications """
zones, ps_zones = init_env()
bucket_name = gen_bucket_name()
# create bucket on the first of the rados zones
bucket = zones[0].create_bucket(bucket_name)
# wait for sync
zone_meta_checkpoint(ps_zones[0].zone)
# create topic
topic_name = bucket_name + TOPIC_SUFFIX
topic_conf = PSTopic(ps_zones[0].conn, topic_name)
_, status = topic_conf.set_config()
assert_equal(status/100, 2)
# create s3 notification
notification_name = bucket_name + NOTIFICATION_SUFFIX
generated_topic_name = notification_name+'_'+topic_name
topic_arn = 'arn:aws:sns:::' + topic_name
s3_notification_conf = PSNotificationS3(ps_zones[0].conn, bucket_name,
notification_name, topic_arn, ['s3:ObjectCreated:*'])
response, status = s3_notification_conf.set_config()
assert_equal(status/100, 2)
zone_meta_checkpoint(ps_zones[0].zone)
# get auto-generated topic
generated_topic_conf = PSTopic(ps_zones[0].conn, generated_topic_name)
result, status = generated_topic_conf.get_config()
parsed_result = json.loads(result)
assert_equal(status/100, 2)
assert_equal(parsed_result['topic']['name'], generated_topic_name)
# get auto-generated notification
notification_conf = PSNotification(ps_zones[0].conn, bucket_name,
generated_topic_name)
result, status = notification_conf.get_config()
parsed_result = json.loads(result)
assert_equal(status/100, 2)
assert_equal(len(parsed_result['topics']), 1)
# get auto-generated subscription
sub_conf = PSSubscription(ps_zones[0].conn, notification_name,
generated_topic_name)
result, status = sub_conf.get_config()
parsed_result = json.loads(result)
assert_equal(status/100, 2)
assert_equal(parsed_result['topic'], generated_topic_name)
# delete s3 notification
_, status = s3_notification_conf.del_config(all_notifications=False)
assert_equal(status/100, 2)
# delete topic
_, status = topic_conf.del_config()
assert_equal(status/100, 2)
# verify low-level cleanup
_, status = generated_topic_conf.get_config()
assert_equal(status, 404)
result, status = notification_conf.get_config()
parsed_result = json.loads(result)
assert_equal(len(parsed_result['topics']), 0)
# TODO should return 404
# assert_equal(status, 404)
result, status = sub_conf.get_config()
parsed_result = json.loads(result)
assert_equal(parsed_result['topic'], '')
# TODO should return 404
# assert_equal(status, 404)
# cleanup
topic_conf.del_config()
# delete the bucket
zones[0].delete_bucket(bucket_name)
