def start_retrieve_slice_tests(state):
"""
start a deferred request for retrieve of slices
"""
global _pending_retrieve_slice_test_count
for key in state["slice-data"].keys():
log.msg("retrieving slice '%s' (%s, %s)" % (
key,
state["slice-data"][key]["offset"],
state["slice-data"][key]["size"], ),
logLevel=logging.DEBUG)
consumer = BufferedConsumer()
path = compute_retrieve_path(key)
range_header_tuple = \
compute_range_header_tuple(state["slice-data"][key]["offset"],
state["slice-data"][key]["size"])
headers = dict([range_header_tuple])
expected_status = frozenset([httplib.PARTIAL_CONTENT, ])
deferred = start_collection_request(state["identity"],
"GET",
state["collection-name"],
path,
response_consumer=consumer,
additional_headers=headers,
valid_http_status=expected_status)
deferred.addCallback(_retrieve_slice_data, state, key, consumer)
deferred.addErrback(_retrieve_slice_error, state, key)
_pending_retrieve_slice_test_count += 1
def start_single_part_archives(state):
"""
start a group of deferred archive requests
"""
global _pending_archive_count
log.msg("starting user_name = %s collection = %s" % (
state["identity"].user_name,
state["collection-name"], ),
logLevel=logging.DEBUG)
# start all the keys archiving
for i in range(state["args"].number_of_single_part_keys):
prefix = random.choice(state["prefixes"])
key = "".join([prefix, state["separator"],
"single_part_key_%05d" % (i+1, )])
log.msg("starting archive for %r" % (key, ), logLevel=logging.DEBUG)
consumer = BufferedConsumer()
path = compute_archive_path(key)
length = random.randint(state["args"].min_single_part_file_size,
state["args"].max_single_part_file_size)
producer = PassThruProducer(key, length)
state["key-data"][key] = {"length" : length,
"md5" : md5(),
"version-identifier" : None}
deferred = start_collection_request(state["identity"],
"POST",
state["collection-name"],
path,
response_consumer=consumer,
body_producer=producer)
deferred.addCallback(_archive_result, state, key, consumer)
deferred.addErrback(_archive_error, state, key)
_pending_archive_count += 1
# loop on callLater until all archive is complete
feed_delay = random.uniform(state["args"].min_feed_delay,
state["args"].max_feed_delay)
reactor.callLater(feed_delay, _feed_producer, key, producer, state)
def start_head_tests(state):
"""
start a deferred request for HEAD on every key
"""
global _pending_head_test_count
for key in state["key-data"].keys():
log.msg("starting HEAD for %r" % (key, ), logLevel=logging.DEBUG)
path = compute_head_path(key)
deferred = start_collection_request(state["identity"],
"HEAD",
state["collection-name"],
path)
deferred.addCallback(_head_result, state, key)
deferred.addErrback(_head_error, state, key)
_pending_head_test_count += 1
def start_conjoined_archives(state):
"""
start a group of deferred archive requests
"""
global _pending_start_conjoined_count
log.msg("start conjoined user_name = %s collection = %s" % (
state["identity"].user_name,
state["collection-name"], ),
logLevel=logging.DEBUG)
# start all the conjoined archives
for i in range(state["args"].number_of_conjoined_keys):
prefix = random.choice(state["prefixes"])
key = "".join([prefix, state["separator"],
"conjoined_key_%05d" % (i+1, )])
log.msg("starting conjoined archive for %r" % (key, ),
logLevel=logging.DEBUG)
consumer = BufferedConsumer()
path = compute_start_conjoined_path(key)
length = random.randint(state["args"].min_conjoined_file_size,
state["args"].max_conjoined_file_size)
state["key-data"][key] = {"length" : length,
"md5" : md5(),
"version-identifier" : None}
state["conjoined-data"][key] = {"conjoined-identifier": None}
deferred = start_collection_request(state["identity"],
"POST",
state["collection-name"],
path,
response_consumer=consumer)
deferred.addCallback(_start_conjoined_result, state, key, consumer)
deferred.addErrback(_start_conjoined_error, state, key)
_pending_start_conjoined_count += 1
def start_retrieve_stream_tests(state):
"""
start a deferred request for retrieve
"""
global _pending_retrieve_stream_test_count
for key in state["key-data"].keys():
log.msg("retrieving key '%s'" % (key, ), logLevel=logging.DEBUG)
consumer = TestStreamConsumer()
path = compute_retrieve_path(key)
deferred = start_collection_request(state["identity"],
"GET",
state["collection-name"],
path,
response_consumer=consumer)
deferred.addCallback(_retrieve_data, state, key, consumer)
deferred.addErrback(_retrieve_error, state, key)
_pending_retrieve_stream_test_count += 1
def _finish_conjoined(state):
global _pending_finish_conjoined_count
# finish all the conjoined archives
for key, entry in state["conjoined-data"].items():
log.msg("finishing conjoined archive for %r %s" %
(key, entry["conjoined-identifier"], ),
logLevel=logging.DEBUG)
consumer = BufferedConsumer()
path = compute_finish_conjoined_path(key,
entry["conjoined-identifier"])
deferred = start_collection_request(state["identity"],
"POST",
state["collection-name"],
path,
response_consumer=consumer)
deferred.addCallback(_finish_conjoined_result, state, key, consumer)
deferred.addErrback(_finish_conjoined_error, state, key)
_pending_finish_conjoined_count += 1
def start_list_versions_tests(state):
"""
start a deferred request for list_versions, with and without key prefixes
"""
global _pending_list_versions_test_count
prefixes = ["", ]
prefixes.extend(state["prefixes"])
for prefix in prefixes:
log.msg("listing versions for prefix '%s'" % (prefix, ),
logLevel=logging.DEBUG)
consumer = BufferedConsumer()
path = compute_list_versions_path(prefix=prefix)
deferred = start_collection_request(state["identity"],
"GET",
state["collection-name"],
path,
response_consumer=consumer)
deferred.addCallback(_list_versions_result, state, prefix, consumer)
deferred.addErrback(_list_versions_error, state, prefix)
_pending_list_versions_test_count += 1
def _archive_conjoined(state):
global _pending_archive_count
for key, entry in state["conjoined-data"].items():
conjoined_identifier = entry["conjoined-identifier"]
length = random.randint(state["args"].min_conjoined_file_size,
state["args"].max_conjoined_file_size)
_key_content[key] = \
"".join([random.choice(printable) for _ in range(length)])\
state["key-data"][key] = {"length" : length,
"md5" : md5(_key_content[key]),
"version-identifier": None}
slice_start = 0
slice_end = slice_start + state["args"].max_conjoined_part_size
conjoined_part = 0
while slice_start < length:
conjoined_part += 1
producer_name = "%s_%03d" % (key, conjoined_part)
if slice_end <= length:
producer_length = state["args"].max_conjoined_part_size
else:
producer_length = length - slice_start
consumer = BufferedConsumer()
producer = PassThruProducer(producer_name, producer_length)
path = \
compute_archive_path(key,
conjoined_identifier=conjoined_identifier,\
conjoined_part=conjoined_part)
deferred = start_collection_request(state["identity"],
"POST",
state["collection-name"],
path,
response_consumer=consumer,
body_producer=producer)
deferred.addCallback(_archive_result,
state,
key,
conjoined_part,
consumer)
deferred.addErrback(_archive_error, state, key, conjoined_part)
_pending_archive_count += 1
# loop on callLater until all archive is complete
feed_delay = random.uniform(state["args"].min_feed_delay,
state["args"].max_feed_delay)
reactor.callLater(feed_delay,
_feed_producer,
key,
conjoined_part,
producer,
slice_start,
slice_end,
state)
slice_start = slice_end
slice_end = slice_start + state["args"].max_conjoined_part_size