def __init__(self, halt_event, user_identity, test_script):
self._log = logging.getLogger(user_identity.user_name)
self._halt_event = halt_event
self._user_identity = user_identity
self._test_script = test_script
self._multipart_upload_cutoff = \
2 * self._test_script["multipart-part-size"]
if "archive-failure-percent" in self._test_script:
self._archive_failure_percent = \
self._test_script["archive-failure-percent"]
else:
self._archive_failure_percent = 0
self._default_collection_name = compute_default_collection_name(
self._user_identity.user_name)
self._s3_connection = None
self._buckets = dict()
self._bucket_accounting = dict()
self._versioned_bucket_names = list()
self._unversioned_bucket_names = list()
self._dispatch_table = {
"create-bucket": self._create_bucket,
"create-versioned-bucket": self._create_versioned_bucket,
"delete-bucket": self._delete_bucket,
"archive-new-key": self._archive_new_key,
"archive-new-version": self._archive_new_version,
"archive-overwrite": self._archive_overwrite,
"retrieve-latest": self._retrieve_latest,
"retrieve-version": self._retrieve_version,
"delete-key": self._delete_key,
"delete-version": self._delete_version,
}
self._frequency_table = list()
self._bucket_name_manager = BucketNameManager(
self._user_identity.user_name,
test_script["max-bucket-count"],
)
self._key_name_manager = KeyNameManager()
self._key_name_generator = None
self.key_verification = dict()
self._error_count = 0
def __init__(self, halt_event, user_identity, test_script):
self._log = logging.getLogger(user_identity.user_name)
self._halt_event = halt_event
self._user_identity = user_identity
self._test_script = test_script
self._multipart_upload_cutoff = \
2 * self._test_script["multipart-part-size"]
if "archive-failure-percent" in self._test_script:
self._archive_failure_percent = \
self._test_script["archive-failure-percent"]
else:
self._archive_failure_percent = 0
self._default_collection_name = compute_default_collection_name(
self._user_identity.user_name
)
self._s3_connection = None
self._buckets = dict()
self._bucket_accounting = dict()
self._versioned_bucket_names = list()
self._unversioned_bucket_names = list()
self._dispatch_table = {
"create-bucket" : self._create_bucket,
"create-versioned-bucket" : self._create_versioned_bucket,
"delete-bucket" : self._delete_bucket,
"archive-new-key" : self._archive_new_key,
"archive-new-version" : self._archive_new_version,
"archive-overwrite" : self._archive_overwrite,
"retrieve-latest" : self._retrieve_latest,
"retrieve-version" : self._retrieve_version,
"delete-key" : self._delete_key,
"delete-version" : self._delete_version,
}
self._frequency_table = list()
self._bucket_name_manager = BucketNameManager(
self._user_identity.user_name,
test_script["max-bucket-count"],
)
self._key_name_manager = KeyNameManager()
self._key_name_generator = None
self.key_verification = dict()
self._error_count = 0
class BaseCustomer(object):
"""
Base class representing a single nimbus.io customer
"""
def __init__(self, halt_event, user_identity, test_script):
self._log = logging.getLogger(user_identity.user_name)
self._halt_event = halt_event
self._user_identity = user_identity
self._test_script = test_script
self._multipart_upload_cutoff = \
2 * self._test_script["multipart-part-size"]
if "archive-failure-percent" in self._test_script:
self._archive_failure_percent = \
self._test_script["archive-failure-percent"]
else:
self._archive_failure_percent = 0
self._default_collection_name = compute_default_collection_name(
self._user_identity.user_name
)
self._s3_connection = None
self._buckets = dict()
self._bucket_accounting = dict()
self._versioned_bucket_names = list()
self._unversioned_bucket_names = list()
self._dispatch_table = {
"create-bucket" : self._create_bucket,
"create-versioned-bucket" : self._create_versioned_bucket,
"delete-bucket" : self._delete_bucket,
"archive-new-key" : self._archive_new_key,
"archive-new-version" : self._archive_new_version,
"archive-overwrite" : self._archive_overwrite,
"retrieve-latest" : self._retrieve_latest,
"retrieve-version" : self._retrieve_version,
"delete-key" : self._delete_key,
"delete-version" : self._delete_version,
}
self._frequency_table = list()
self._bucket_name_manager = BucketNameManager(
self._user_identity.user_name,
test_script["max-bucket-count"],
)
self._key_name_manager = KeyNameManager()
self._key_name_generator = None
self.key_verification = dict()
self._error_count = 0
@property
def error_count(self):
return self._error_count
def _main_loop(self):
self._s3_connection = motoboto.connect_s3(identity=self._user_identity)
self._initial_inventory()
if self._test_script.get("verify-before", False):
self._verify_before()
self._load_frequency_table()
self._key_name_generator = self._key_name_manager.key_name_generator()
# do an initial delay so all customers don't start at once
self._delay()
while not self._halt_event.is_set():
# run a randomly selected test function
test_function = self._frequency_table[random.randint(0, 99)]
try:
test_function()
except Exception:
self._error_count += 1
self._log.exception("test_function error #{0}".format(
self._error_count))
self._delay()
if self._test_script.get("verify-after", False):
self._verify_after()
if self._test_script.get("audit-after", False):
self._audit_after()
self._s3_connection.close()
self._log.info("{0} errors".format(self._error_count))
def _initial_inventory(self):
"""get an initial inventory of buckets and files"""
buckets = self._s3_connection.get_all_buckets()
for bucket in buckets:
self._log.info("_initial_inventory found bucket %r" % (
bucket.name,
))
self._buckets[bucket.name] = bucket
self._bucket_name_manager.existing_bucket_name(bucket.name)
if bucket.versioning:
self._versioned_bucket_names.append(bucket.name)
else:
self._unversioned_bucket_names.append(bucket.name)
self._bucket_accounting[bucket.name] = CollectionOpsAccounting()
bucket_accounting = self._bucket_accounting[bucket.name]
bucket_accounting.increment_by("listmatch_request", 1)
keys = bucket.get_all_keys()
bucket_accounting.increment_by("listmatch_success", 1)
for key in keys:
self._log.info("_initial_inventory found key %r, %r" % (
key.name, bucket.name,
))
self._key_name_manager.existing_key_name(key.name)
def _verify_key(self, bucket, key, data_size, md5_digest):
verification_key = (bucket.name, key.name, key.version_id, )
try:
expected_data_size, expected_md5_digest = \
self.key_verification[verification_key]
except KeyError:
self._error_count += 1
self._log.error("_verify_key key not found {0} error #{1}".format(
verification_key, self._error_count))
return
if data_size != expected_data_size:
self._error_count += 1
self._log.error("size mistmatch {0} {1} {2} error #{3}".format(
data_size,
expected_data_size,
verification_key,
self._error_count))
if expected_md5_digest is not None and \
md5_digest != expected_md5_digest:
self._error_count += 1
self._log.error("md5 mismatch {0} error #{1}".format(
verification_key, self._error_count))
def _verify_key_final(self, bucket, key, data_size, md5_digest):
"""
remove key when verifying, so we can track unmatched keys
"""
verification_key = (bucket.name, key.name, key.version_id, )
try:
expected_data_size, expected_md5_digest = \
self.key_verification.pop(verification_key)
except KeyError:
self._error_count += 1
self._log.error("key not found {0} error #{1}".format(
verification_key, self._error_count))
return
if data_size != expected_data_size:
self._error_count += 1
self._log.error("size mistmatch {0} {1} {2} error #{3}".format(
data_size,
expected_data_size,
verification_key,
self._error_count))
if expected_md5_digest is not None and \
md5_digest != expected_md5_digest:
self._error_count += 1
self._log.error("md5 mismatch {0} error #{1}".format(
verification_key, self._error_count))
def _verify_before(self):
"""
retrieve all known keys to verify that they are reachable
store md5 digests
"""
self._log.info("verifying retrieves before")
buckets = self._s3_connection.get_all_buckets()
for bucket in buckets:
bucket_accounting = self._bucket_accounting[bucket.name]
if bucket.versioning:
bucket_accounting.increment_by("listmatch_request", 1)
for key in bucket.get_all_versions():
result = key.get_contents_as_string(
version_id=key.version_id
)
verification_key = (bucket.name, key.name, key.version_id)
md5_sum = hashlib.md5(result)
if verification_key in self.key_verification:
self._log.error("duplicate key (versioning) %s" % (
verification_key, ))
self.key_verification[verification_key] = \
(len(result), md5_sum.digest(), )
bucket_accounting.increment_by("listmatch_success", 1)
else:
bucket_accounting.increment_by("listmatch_request", 1)
for key in bucket.get_all_keys():
result = key.get_contents_as_string()
md5_sum = hashlib.md5(result)
verification_key = (bucket.name, key.name, key.version_id)
if verification_key in self.key_verification:
self._log.error("duplicate key %s" % (
verification_key, ))
self.key_verification[verification_key] = \
(len(result), md5_sum.digest(), )
bucket_accounting.increment_by("listmatch_success", 1)
def _verify_after(self):
"""
retrieve all known keys to verify that they are reachable
check md5 digests if they exist
"""
self._log.info("verifying retrieves after")
buckets = self._s3_connection.get_all_buckets()
for bucket in buckets:
bucket_accounting = self._bucket_accounting[bucket.name]
if bucket.versioning:
bucket_accounting.increment_by("listmatch_request", 1)
for key in bucket.get_all_versions():
result = key.get_contents_as_string(
version_id=key.version_id
)
md5_sum = hashlib.md5(result)
self._verify_key_final(bucket,
key,
len(result),
md5_sum.digest())
bucket_accounting.increment_by("listmatch_success", 1)
else:
bucket_accounting.increment_by("listmatch_request", 1)
for key in bucket.get_all_keys():
result = key.get_contents_as_string()
md5_sum = hashlib.md5(result)
self._verify_key_final(bucket,
key,
len(result),
md5_sum.digest())
bucket_accounting.increment_by("listmatch_success", 1)
if len(self.key_verification) > 0:
self._log.info("{0} unreachable keys".format(
len(self.key_verification)))
for key, value in self.key_verification.items():
self._error_count += 1
self._log.error("unreachable key {0} {1}".format(key, value))
def _audit_after(self):
"""
retrieve the space_usage for each bucket and compare it to our value
"""
self._log.info("audit_after begin")
audit_error_count = 0
buckets = self._s3_connection.get_all_buckets()
for bucket in buckets:
result = bucket.get_space_used()
if not result["success"]:
audit_error_count += 1
self._log.error("audit {0} {1}".format(bucket.name,
result["error_message"]))
continue
# XXX: can't handle more than one day
if len(result["operational_stats"]) > 1:
audit_error_count += 1
self._log.error("audit {0} need a single day {1}".format(
bucket.name, result["operational_stats"]))
continue
if len(result["operational_stats"]) == 0:
self._log.info("audit {0} no server data".format(bucket.name))
server_audit = {"archive_success" : 0,
"success_bytes_in" : 0,
"retrieve_success" : 0,
"success_bytes_out" : 0,
"delete_success" : 0,
"listmatch_success" : 0, }
else:
server_audit = result["operational_stats"][0]
our_audit = self._bucket_accounting[bucket.name]
if our_audit["archive_success"] != server_audit["archive_success"]:
audit_error_count += 1
self._log.error("audit {0} archive_success {1} {2}".format(
bucket.name,
our_audit["archive_success"],
server_audit["archive_success"]))
if our_audit["success_bytes_in"] != server_audit["success_bytes_in"]:
audit_error_count += 1
self._log.error("audit {0} success_bytes_in {1} {2}".format(
bucket.name,
our_audit["success_bytes_in"],
server_audit["success_bytes_in"]))
if our_audit["retrieve_success"] != server_audit["retrieve_success"]:
audit_error_count += 1
self._log.error("audit {0} retrieve_success {1} {2}".format(
bucket.name,
our_audit["retrieve_success"],
server_audit["retrieve_success"]))
if our_audit["success_bytes_out"] != server_audit["success_bytes_out"]:
audit_error_count += 1
self._log.error("audit {0} success_bytes_out {1} {2}".format(
bucket.name,
our_audit["success_bytes_out"],
server_audit["success_bytes_out"]))
if our_audit["delete_success"] != server_audit["delete_success"]:
audit_error_count += 1
self._log.error("audit {0} delete_success {1} {2}".format(
bucket.name,
our_audit["delete_success"],
server_audit["delete_success"]))
if our_audit["listmatch_success"] != server_audit["listmatch_success"]:
audit_error_count += 1
self._log.error("audit {0} listmatch_success {1} {2}".format(
bucket.name,
our_audit["listmatch_success"],
server_audit["listmatch_success"]))
self._log.info("audit_after found {0} errors".format(audit_error_count))
self._error_count += audit_error_count
def _load_frequency_table(self):
"""
for each action specfied in the distribution append n instances
of the corresponding function object. We will choose a random number
between 0 and 99, to select a test action
"""
for key in self._test_script["distribution"].keys():
count = self._test_script["distribution"][key]
for _ in xrange(count):
self._frequency_table.append(self._dispatch_table[key])
assert len(self._frequency_table) == 100
def _delay(self):
"""wait for a (delimited) random time"""
delay_size = random.uniform(
self._test_script["low-delay"], self._test_script["high-delay"]
)
self._halt_event.wait(timeout=delay_size)
def _create_unversioned_bucket(self):
if len(self._buckets) >= self._test_script["max-bucket-count"]:
self._log.info("ignore _create_bucket: already have %s buckets" % (
len(self._buckets),
))
return None
bucket_name = self._bucket_name_manager.next()
if bucket_name is None:
self._log.info("ignore _create_bucket")
return
new_bucket = self._s3_connection.create_bucket(bucket_name)
self._buckets[new_bucket.name] = new_bucket
self._bucket_accounting[new_bucket.name] = CollectionOpsAccounting()
return new_bucket
def _create_bucket(self):
bucket = self._create_unversioned_bucket()
if bucket is None:
return
self._log.info("create unversioned bucket {0} {1}".format(
bucket.name, bucket.versioning, ))
self._unversioned_bucket_names.append(bucket.name)
def _create_versioned_bucket(self):
bucket = self._create_unversioned_bucket()
if bucket is None:
return
bucket.configure_versioning(True)
self._log.info("create versioned bucket {0} {1}".format(
bucket.name, bucket.versioning, ))
self._versioned_bucket_names.append(bucket.name)
def _delete_bucket(self):
eligible_bucket_names = [
k for k in self._buckets.keys() \
if k != self._default_collection_name
]
if len(eligible_bucket_names) == 0:
self._log.warn("no buckets eligible for deletion")
return
bucket_name = random.choice(eligible_bucket_names)
bucket = self._buckets.pop(bucket_name)
self._log.info("delete bucket {0} versioned={1}".format(
bucket.name, bucket.versioning))
self._bucket_accounting[bucket_name].mark_end()
if bucket.versioning:
self._clear_versioned_bucket(bucket)
else:
self._clear_unversioned_bucket(bucket)
self._bucket_name_manager.deleted_bucket_name(bucket.name)
self._s3_connection.delete_bucket(bucket.name)
def _clear_versioned_bucket(self, bucket):
try:
i = self._versioned_bucket_names.index(bucket.name)
except ValueError:
self._log.error("not found in versioned buckets {0}".format(
bucket.name))
else:
del self._versioned_bucket_names[i]
bucket_accounting = self._bucket_accounting[bucket.name]
# delete all the keys (or versions) for the bucket
bucket_accounting.increment_by("listmatch_request", 1)
for key in bucket.get_all_versions():
verification_key = (bucket.name, key.name, key.version_id)
self._log.info("_delete_bucket deleting version {0}".format(
verification_key))
retry_count = 0
while not self._halt_event.is_set():
bucket_accounting.increment_by("delete_request", 1)
try:
key.delete(version_id=key.version_id)
except LumberyardRetryableHTTPError, instance:
bucket_accounting.increment_by("delete_error", 1)
if retry_count >= _max_delete_retries:
raise
self._log.warn("%s: retry in %s seconds" % (
instance, instance.retry_after,
))
self._halt_event.wait(timeout=instance.retry_after)
retry_count += 1
self._log.warn("retry #%s" % (retry_count, ))
else:
break
bucket_accounting.increment_by("delete_success", 1)
try:
del self.key_verification[verification_key]
except KeyError:
self._log.error(
"_delete_bucket verification key not found %s" % (
verification_key, ))
bucket_accounting.increment_by("listmatch_success", 1)
class BaseCustomer(object):
"""
Base class representing a single nimbus.io customer
"""
def __init__(self, halt_event, user_identity, test_script):
self._log = logging.getLogger(user_identity.user_name)
self._halt_event = halt_event
self._user_identity = user_identity
self._test_script = test_script
self._multipart_upload_cutoff = \
2 * self._test_script["multipart-part-size"]
if "archive-failure-percent" in self._test_script:
self._archive_failure_percent = \
self._test_script["archive-failure-percent"]
else:
self._archive_failure_percent = 0
self._default_collection_name = compute_default_collection_name(
self._user_identity.user_name)
self._s3_connection = None
self._buckets = dict()
self._bucket_accounting = dict()
self._versioned_bucket_names = list()
self._unversioned_bucket_names = list()
self._dispatch_table = {
"create-bucket": self._create_bucket,
"create-versioned-bucket": self._create_versioned_bucket,
"delete-bucket": self._delete_bucket,
"archive-new-key": self._archive_new_key,
"archive-new-version": self._archive_new_version,
"archive-overwrite": self._archive_overwrite,
"retrieve-latest": self._retrieve_latest,
"retrieve-version": self._retrieve_version,
"delete-key": self._delete_key,
"delete-version": self._delete_version,
}
self._frequency_table = list()
self._bucket_name_manager = BucketNameManager(
self._user_identity.user_name,
test_script["max-bucket-count"],
)
self._key_name_manager = KeyNameManager()
self._key_name_generator = None
self.key_verification = dict()
self._error_count = 0
@property
def error_count(self):
return self._error_count
def _main_loop(self):
self._s3_connection = motoboto.connect_s3(identity=self._user_identity)
self._initial_inventory()
if self._test_script.get("verify-before", False):
self._verify_before()
self._load_frequency_table()
self._key_name_generator = self._key_name_manager.key_name_generator()
# do an initial delay so all customers don't start at once
self._delay()
while not self._halt_event.is_set():
# run a randomly selected test function
test_function = self._frequency_table[random.randint(0, 99)]
try:
test_function()
except Exception:
self._error_count += 1
self._log.exception("test_function error #{0}".format(
self._error_count))
self._delay()
if self._test_script.get("verify-after", False):
self._verify_after()
if self._test_script.get("audit-after", False):
self._audit_after()
self._s3_connection.close()
self._log.info("{0} errors".format(self._error_count))
def _initial_inventory(self):
"""get an initial inventory of buckets and files"""
buckets = self._s3_connection.get_all_buckets()
for bucket in buckets:
self._log.info("_initial_inventory found bucket %r" %
(bucket.name, ))
self._buckets[bucket.name] = bucket
self._bucket_name_manager.existing_bucket_name(bucket.name)
if bucket.versioning:
self._versioned_bucket_names.append(bucket.name)
else:
self._unversioned_bucket_names.append(bucket.name)
self._bucket_accounting[bucket.name] = CollectionOpsAccounting()
bucket_accounting = self._bucket_accounting[bucket.name]
bucket_accounting.increment_by("listmatch_request", 1)
keys = bucket.get_all_keys()
bucket_accounting.increment_by("listmatch_success", 1)
for key in keys:
self._log.info("_initial_inventory found key %r, %r" % (
key.name,
bucket.name,
))
self._key_name_manager.existing_key_name(key.name)
def _verify_key(self, bucket, key, data_size, md5_digest):
verification_key = (
bucket.name,
key.name,
key.version_id,
)
try:
expected_data_size, expected_md5_digest = \
self.key_verification[verification_key]
except KeyError:
self._error_count += 1
self._log.error("_verify_key key not found {0} error #{1}".format(
verification_key, self._error_count))
return
if data_size != expected_data_size:
self._error_count += 1
self._log.error("size mistmatch {0} {1} {2} error #{3}".format(
data_size, expected_data_size, verification_key,
self._error_count))
if expected_md5_digest is not None and \
md5_digest != expected_md5_digest:
self._error_count += 1
self._log.error("md5 mismatch {0} error #{1}".format(
verification_key, self._error_count))
def _verify_key_final(self, bucket, key, data_size, md5_digest):
"""
remove key when verifying, so we can track unmatched keys
"""
verification_key = (
bucket.name,
key.name,
key.version_id,
)
try:
expected_data_size, expected_md5_digest = \
self.key_verification.pop(verification_key)
except KeyError:
self._error_count += 1
self._log.error("key not found {0} error #{1}".format(
verification_key, self._error_count))
return
if data_size != expected_data_size:
self._error_count += 1
self._log.error("size mistmatch {0} {1} {2} error #{3}".format(
data_size, expected_data_size, verification_key,
self._error_count))
if expected_md5_digest is not None and \
md5_digest != expected_md5_digest:
self._error_count += 1
self._log.error("md5 mismatch {0} error #{1}".format(
verification_key, self._error_count))
def _verify_before(self):
"""
retrieve all known keys to verify that they are reachable
store md5 digests
"""
self._log.info("verifying retrieves before")
buckets = self._s3_connection.get_all_buckets()
for bucket in buckets:
bucket_accounting = self._bucket_accounting[bucket.name]
if bucket.versioning:
bucket_accounting.increment_by("listmatch_request", 1)
for key in bucket.get_all_versions():
result = key.get_contents_as_string(
version_id=key.version_id)
verification_key = (bucket.name, key.name, key.version_id)
md5_sum = hashlib.md5(result)
if verification_key in self.key_verification:
self._log.error("duplicate key (versioning) %s" %
(verification_key, ))
self.key_verification[verification_key] = \
(len(result), md5_sum.digest(), )
bucket_accounting.increment_by("listmatch_success", 1)
else:
bucket_accounting.increment_by("listmatch_request", 1)
for key in bucket.get_all_keys():
result = key.get_contents_as_string()
md5_sum = hashlib.md5(result)
verification_key = (bucket.name, key.name, key.version_id)
if verification_key in self.key_verification:
self._log.error("duplicate key %s" %
(verification_key, ))
self.key_verification[verification_key] = \
(len(result), md5_sum.digest(), )
bucket_accounting.increment_by("listmatch_success", 1)
def _verify_after(self):
"""
retrieve all known keys to verify that they are reachable
check md5 digests if they exist
"""
self._log.info("verifying retrieves after")
buckets = self._s3_connection.get_all_buckets()
for bucket in buckets:
bucket_accounting = self._bucket_accounting[bucket.name]
if bucket.versioning:
bucket_accounting.increment_by("listmatch_request", 1)
for key in bucket.get_all_versions():
result = key.get_contents_as_string(
version_id=key.version_id)
md5_sum = hashlib.md5(result)
self._verify_key_final(bucket, key, len(result),
md5_sum.digest())
bucket_accounting.increment_by("listmatch_success", 1)
else:
bucket_accounting.increment_by("listmatch_request", 1)
for key in bucket.get_all_keys():
result = key.get_contents_as_string()
md5_sum = hashlib.md5(result)
self._verify_key_final(bucket, key, len(result),
md5_sum.digest())
bucket_accounting.increment_by("listmatch_success", 1)
if len(self.key_verification) > 0:
self._log.info("{0} unreachable keys".format(
len(self.key_verification)))
for key, value in self.key_verification.items():
self._error_count += 1
self._log.error("unreachable key {0} {1}".format(key, value))
def _audit_after(self):
"""
retrieve the space_usage for each bucket and compare it to our value
"""
self._log.info("audit_after begin")
audit_error_count = 0
buckets = self._s3_connection.get_all_buckets()
for bucket in buckets:
result = bucket.get_space_used()
if not result["success"]:
audit_error_count += 1
self._log.error("audit {0} {1}".format(
bucket.name, result["error_message"]))
continue
# XXX: can't handle more than one day
if len(result["operational_stats"]) > 1:
audit_error_count += 1
self._log.error("audit {0} need a single day {1}".format(
bucket.name, result["operational_stats"]))
continue
if len(result["operational_stats"]) == 0:
self._log.info("audit {0} no server data".format(bucket.name))
server_audit = {
"archive_success": 0,
"success_bytes_in": 0,
"retrieve_success": 0,
"success_bytes_out": 0,
"delete_success": 0,
"listmatch_success": 0,
}
else:
server_audit = result["operational_stats"][0]
our_audit = self._bucket_accounting[bucket.name]
if our_audit["archive_success"] != server_audit["archive_success"]:
audit_error_count += 1
self._log.error("audit {0} archive_success {1} {2}".format(
bucket.name, our_audit["archive_success"],
server_audit["archive_success"]))
if our_audit["success_bytes_in"] != server_audit[
"success_bytes_in"]:
audit_error_count += 1
self._log.error("audit {0} success_bytes_in {1} {2}".format(
bucket.name, our_audit["success_bytes_in"],
server_audit["success_bytes_in"]))
if our_audit["retrieve_success"] != server_audit[
"retrieve_success"]:
audit_error_count += 1
self._log.error("audit {0} retrieve_success {1} {2}".format(
bucket.name, our_audit["retrieve_success"],
server_audit["retrieve_success"]))
if our_audit["success_bytes_out"] != server_audit[
"success_bytes_out"]:
audit_error_count += 1
self._log.error("audit {0} success_bytes_out {1} {2}".format(
bucket.name, our_audit["success_bytes_out"],
server_audit["success_bytes_out"]))
if our_audit["delete_success"] != server_audit["delete_success"]:
audit_error_count += 1
self._log.error("audit {0} delete_success {1} {2}".format(
bucket.name, our_audit["delete_success"],
server_audit["delete_success"]))
if our_audit["listmatch_success"] != server_audit[
"listmatch_success"]:
audit_error_count += 1
self._log.error("audit {0} listmatch_success {1} {2}".format(
bucket.name, our_audit["listmatch_success"],
server_audit["listmatch_success"]))
self._log.info(
"audit_after found {0} errors".format(audit_error_count))
self._error_count += audit_error_count
def _load_frequency_table(self):
"""
for each action specfied in the distribution append n instances
of the corresponding function object. We will choose a random number
between 0 and 99, to select a test action
"""
for key in self._test_script["distribution"].keys():
count = self._test_script["distribution"][key]
for _ in xrange(count):
self._frequency_table.append(self._dispatch_table[key])
assert len(self._frequency_table) == 100
def _delay(self):
"""wait for a (delimited) random time"""
delay_size = random.uniform(self._test_script["low-delay"],
self._test_script["high-delay"])
self._halt_event.wait(timeout=delay_size)
def _create_unversioned_bucket(self):
if len(self._buckets) >= self._test_script["max-bucket-count"]:
self._log.info("ignore _create_bucket: already have %s buckets" %
(len(self._buckets), ))
return None
bucket_name = self._bucket_name_manager.next()
if bucket_name is None:
self._log.info("ignore _create_bucket")
return
new_bucket = self._s3_connection.create_bucket(bucket_name)
self._buckets[new_bucket.name] = new_bucket
self._bucket_accounting[new_bucket.name] = CollectionOpsAccounting()
return new_bucket
def _create_bucket(self):
bucket = self._create_unversioned_bucket()
if bucket is None:
return
self._log.info("create unversioned bucket {0} {1}".format(
bucket.name,
bucket.versioning,
))
self._unversioned_bucket_names.append(bucket.name)
def _create_versioned_bucket(self):
bucket = self._create_unversioned_bucket()
if bucket is None:
return
bucket.configure_versioning(True)
self._log.info("create versioned bucket {0} {1}".format(
bucket.name,
bucket.versioning,
))
self._versioned_bucket_names.append(bucket.name)
def _delete_bucket(self):
eligible_bucket_names = [
k for k in self._buckets.keys() \
if k != self._default_collection_name
]
if len(eligible_bucket_names) == 0:
self._log.warn("no buckets eligible for deletion")
return
bucket_name = random.choice(eligible_bucket_names)
bucket = self._buckets.pop(bucket_name)
self._log.info("delete bucket {0} versioned={1}".format(
bucket.name, bucket.versioning))
self._bucket_accounting[bucket_name].mark_end()
if bucket.versioning:
self._clear_versioned_bucket(bucket)
else:
self._clear_unversioned_bucket(bucket)
self._bucket_name_manager.deleted_bucket_name(bucket.name)
self._s3_connection.delete_bucket(bucket.name)
def _clear_versioned_bucket(self, bucket):
try:
i = self._versioned_bucket_names.index(bucket.name)
except ValueError:
self._log.error("not found in versioned buckets {0}".format(
bucket.name))
else:
del self._versioned_bucket_names[i]
bucket_accounting = self._bucket_accounting[bucket.name]
# delete all the keys (or versions) for the bucket
bucket_accounting.increment_by("listmatch_request", 1)
for key in bucket.get_all_versions():
verification_key = (bucket.name, key.name, key.version_id)
self._log.info(
"_delete_bucket deleting version {0}".format(verification_key))
retry_count = 0
while not self._halt_event.is_set():
bucket_accounting.increment_by("delete_request", 1)
try:
key.delete(version_id=key.version_id)
except LumberyardRetryableHTTPError, instance:
bucket_accounting.increment_by("delete_error", 1)
if retry_count >= _max_delete_retries:
raise
self._log.warn("%s: retry in %s seconds" % (
instance,
instance.retry_after,
))
self._halt_event.wait(timeout=instance.retry_after)
retry_count += 1
self._log.warn("retry #%s" % (retry_count, ))
else:
break
bucket_accounting.increment_by("delete_success", 1)
try:
del self.key_verification[verification_key]
except KeyError:
self._log.error(
"_delete_bucket verification key not found %s" %
(verification_key, ))
bucket_accounting.increment_by("listmatch_success", 1)
