def create_loading_tasks(params):
log = logger.Logger.get_logger()
task = {
"ops": params["ops"],
"seed": params["seed"],
"bucket": params["bucket"],
"docs": [],
"sizes": params["sizes"]
}
log.info("params : {0}".format(params))
for size in params["sizes"]:
info = RebalanceDataGenerator.job_info()
if "kv_template" not in params:
kv_template = {
"name": "employee-${prefix}-${seed}",
"sequence": "${seed}",
"join_yr": 2007,
"join_mo": 10,
"join_day": 20,
"email": "${prefix}@couchbase.com",
"job_title": "Software Engineer-${padding}",
"desc": info["desc"].encode("utf-8", "ignore")
}
else:
kv_template = params["kv_template"]
options = {"size": size, "seed": params["seed"]}
if "padding" in params:
options["padding"] = params["padding"]
docs = DocumentGenerator.make_docs(
params["count"] / len(params["sizes"]), kv_template, options)
task["docs"].append(docs)
return task
def add_doc_gen_task(tm, rest, count, bucket="default", kv_store=None, store_enabled=True,
kv_template=None, seed=None, sizes=None, expiration=None,
loop=False, monitor=False, doc_generators=None):
doc_generators = doc_generators or DocumentGenerator.get_doc_generators(count, kv_template, seed, sizes)
_t = task.LoadDocGeneratorTask(rest, doc_generators, bucket=bucket, kv_store=kv_store,
store_enabled=store_enabled, expiration=expiration, loop=loop)
return RebalanceTaskHelper.schedule_task_helper(tm, _t, monitor)
def create_loading_tasks(params):
log = logger.Logger.get_logger()
task = {
"ops": params["ops"],
"seed": params["seed"],
"bucket": params["bucket"],
"docs": [],
"sizes": params["sizes"],
}
log.info("params : {0}".format(params))
for size in params["sizes"]:
info = RebalanceDataGenerator.job_info()
if "kv_template" not in params:
kv_template = {
"name": "employee-${prefix}-${seed}",
"sequence": "${seed}",
"join_yr": 2007,
"join_mo": 10,
"join_day": 20,
"email": "${prefix}@couchbase.com",
"job_title": "Software Engineer-${padding}",
"desc": info["desc"].encode("utf-8", "ignore"),
}
else:
kv_template = params["kv_template"]
options = {"size": size, "seed": params["seed"]}
if "padding" in params:
options["padding"] = params["padding"]
docs = DocumentGenerator.make_docs(params["count"] / len(params["sizes"]), kv_template, options)
task["docs"].append(docs)
return task
def load_bucket_task_helper(rest, task_manager, bucket, ram_load_ratio,
kv_store=None, distribution=None,
keys_count= -1, seed=None, monitor=True):
log = logger.Logger().get_logger()
tasks = []
if keys_count == -1:
# create document generators based on value_size_distrobution
doc_gen_configs = \
DocumentGenerator.get_doc_generators_by_load_ratio(rest,
bucket,
ram_load_ratio,
distribution)
for config in doc_gen_configs:
doc_gen = config['value']
how_many = config['how_many']
size = config['size']
seed = config['seed']
# start bucket loading task
msg = "start task to send {0} items with value of size : {1}"
log.info(msg.format(how_many, size))
doc_gen_task = RebalanceTaskHelper.add_doc_gen_task(task_manager, rest,
how_many, bucket,
kv_store=kv_store,
doc_generators=[doc_gen],
monitor=monitor)
tasks.append({'task': doc_gen_task, 'seed': seed})
else:
msg = "start task to send {0} items to bucket {1}"
log.info(msg.format(keys_count, bucket))
doc_gen_task = RebalanceTaskHelper.add_doc_gen_task(task_manager,
rest,
keys_count,
bucket,
kv_store,
seed=seed,
monitor=monitor)
tasks.append({'task': doc_gen_task, 'seed': seed})
return tasks
def _iterative_load(self, info, tc, view, loads_per_iteration, verify_docs_loaded):
loaded_docs = []
try:
smart = VBucketAwareMemcached(self.rest, self.bucket)
for i in range(1,self.years + 1):
for j in range(1, self.months + 1):
doc_sets = []
for k in range(1, self.days + 1):
kv_template = {"name": "employee-${prefix}-${seed}",
"join_yr" : 2007+i, "join_mo" : j, "join_day" : k,
"email": "${prefix}@couchbase.com",
"job_title" : info["title"].encode("utf-8","ignore"),
"type" : info["type"].encode("utf-8","ignore"),
"desc" : info["desc"].encode("utf-8", "ignore")}
options = {"size": 256, "seed": str(uuid.uuid4())[:7]}
docs = DocumentGenerator.make_docs(loads_per_iteration, kv_template, options)
doc_sets.append(docs)
#load docs
self._load_chunk(smart, doc_sets)
except Exception as ex:
view.results.addError(tc, sys.exc_info())
raise ex
