def start_single_part_archives(state): """ start a group of deferred archive requests """ global _pending_archive_count log.msg("starting user_name = %s collection = %s" % ( state["identity"].user_name, state["collection-name"], ), logLevel=logging.DEBUG) # start all the keys archiving for i in range(state["args"].number_of_single_part_keys): prefix = random.choice(state["prefixes"]) key = "".join([prefix, state["separator"], "single_part_key_%05d" % (i+1, )]) log.msg("starting archive for %r" % (key, ), logLevel=logging.DEBUG) consumer = BufferedConsumer() path = compute_archive_path(key) length = random.randint(state["args"].min_single_part_file_size, state["args"].max_single_part_file_size) producer = PassThruProducer(key, length) state["key-data"][key] = {"length" : length, "md5" : md5(), "version-identifier" : None} deferred = start_collection_request(state["identity"], "POST", state["collection-name"], path, response_consumer=consumer, body_producer=producer) deferred.addCallback(_archive_result, state, key, consumer) deferred.addErrback(_archive_error, state, key) _pending_archive_count += 1 # loop on callLater until all archive is complete feed_delay = random.uniform(state["args"].min_feed_delay, state["args"].max_feed_delay) reactor.callLater(feed_delay, _feed_producer, key, producer, state)
def _archive_conjoined(state): global _pending_archive_count for key, entry in state["conjoined-data"].items(): conjoined_identifier = entry["conjoined-identifier"] length = random.randint(state["args"].min_conjoined_file_size, state["args"].max_conjoined_file_size) _key_content[key] = \ "".join([random.choice(printable) for _ in range(length)])\ state["key-data"][key] = {"length" : length, "md5" : md5(_key_content[key]), "version-identifier": None} slice_start = 0 slice_end = slice_start + state["args"].max_conjoined_part_size conjoined_part = 0 while slice_start < length: conjoined_part += 1 producer_name = "%s_%03d" % (key, conjoined_part) if slice_end <= length: producer_length = state["args"].max_conjoined_part_size else: producer_length = length - slice_start consumer = BufferedConsumer() producer = PassThruProducer(producer_name, producer_length) path = \ compute_archive_path(key, conjoined_identifier=conjoined_identifier,\ conjoined_part=conjoined_part) deferred = start_collection_request(state["identity"], "POST", state["collection-name"], path, response_consumer=consumer, body_producer=producer) deferred.addCallback(_archive_result, state, key, conjoined_part, consumer) deferred.addErrback(_archive_error, state, key, conjoined_part) _pending_archive_count += 1 # loop on callLater until all archive is complete feed_delay = random.uniform(state["args"].min_feed_delay, state["args"].max_feed_delay) reactor.callLater(feed_delay, _feed_producer, key, conjoined_part, producer, slice_start, slice_end, state) slice_start = slice_end slice_end = slice_start + state["args"].max_conjoined_part_size