def create_loading_tasks(params): log = logger.Logger.get_logger() task = { "ops": params["ops"], "seed": params["seed"], "bucket": params["bucket"], "docs": [], "sizes": params["sizes"] } log.info("params : {0}".format(params)) for size in params["sizes"]: info = RebalanceDataGenerator.job_info() if "kv_template" not in params: kv_template = { "name": "employee-${prefix}-${seed}", "sequence": "${seed}", "join_yr": 2007, "join_mo": 10, "join_day": 20, "email": "${prefix}@couchbase.com", "job_title": "Software Engineer-${padding}", "desc": info["desc"].encode("utf-8", "ignore") } else: kv_template = params["kv_template"] options = {"size": size, "seed": params["seed"]} if "padding" in params: options["padding"] = params["padding"] docs = DocumentGenerator.make_docs( params["count"] / len(params["sizes"]), kv_template, options) task["docs"].append(docs) return task
def add_doc_gen_task(tm, rest, count, bucket="default", kv_store=None, store_enabled=True, kv_template=None, seed=None, sizes=None, expiration=None, loop=False, monitor=False, doc_generators=None): doc_generators = doc_generators or DocumentGenerator.get_doc_generators(count, kv_template, seed, sizes) _t = task.LoadDocGeneratorTask(rest, doc_generators, bucket=bucket, kv_store=kv_store, store_enabled=store_enabled, expiration=expiration, loop=loop) return RebalanceTaskHelper.schedule_task_helper(tm, _t, monitor)
def create_loading_tasks(params): log = logger.Logger.get_logger() task = { "ops": params["ops"], "seed": params["seed"], "bucket": params["bucket"], "docs": [], "sizes": params["sizes"], } log.info("params : {0}".format(params)) for size in params["sizes"]: info = RebalanceDataGenerator.job_info() if "kv_template" not in params: kv_template = { "name": "employee-${prefix}-${seed}", "sequence": "${seed}", "join_yr": 2007, "join_mo": 10, "join_day": 20, "email": "${prefix}@couchbase.com", "job_title": "Software Engineer-${padding}", "desc": info["desc"].encode("utf-8", "ignore"), } else: kv_template = params["kv_template"] options = {"size": size, "seed": params["seed"]} if "padding" in params: options["padding"] = params["padding"] docs = DocumentGenerator.make_docs(params["count"] / len(params["sizes"]), kv_template, options) task["docs"].append(docs) return task
def load_bucket_task_helper(rest, task_manager, bucket, ram_load_ratio, kv_store=None, distribution=None, keys_count= -1, seed=None, monitor=True): log = logger.Logger().get_logger() tasks = [] if keys_count == -1: # create document generators based on value_size_distrobution doc_gen_configs = \ DocumentGenerator.get_doc_generators_by_load_ratio(rest, bucket, ram_load_ratio, distribution) for config in doc_gen_configs: doc_gen = config['value'] how_many = config['how_many'] size = config['size'] seed = config['seed'] # start bucket loading task msg = "start task to send {0} items with value of size : {1}" log.info(msg.format(how_many, size)) doc_gen_task = RebalanceTaskHelper.add_doc_gen_task(task_manager, rest, how_many, bucket, kv_store=kv_store, doc_generators=[doc_gen], monitor=monitor) tasks.append({'task': doc_gen_task, 'seed': seed}) else: msg = "start task to send {0} items to bucket {1}" log.info(msg.format(keys_count, bucket)) doc_gen_task = RebalanceTaskHelper.add_doc_gen_task(task_manager, rest, keys_count, bucket, kv_store, seed=seed, monitor=monitor) tasks.append({'task': doc_gen_task, 'seed': seed}) return tasks
def _iterative_load(self, info, tc, view, loads_per_iteration, verify_docs_loaded): loaded_docs = [] try: smart = VBucketAwareMemcached(self.rest, self.bucket) for i in range(1,self.years + 1): for j in range(1, self.months + 1): doc_sets = [] for k in range(1, self.days + 1): kv_template = {"name": "employee-${prefix}-${seed}", "join_yr" : 2007+i, "join_mo" : j, "join_day" : k, "email": "${prefix}@couchbase.com", "job_title" : info["title"].encode("utf-8","ignore"), "type" : info["type"].encode("utf-8","ignore"), "desc" : info["desc"].encode("utf-8", "ignore")} options = {"size": 256, "seed": str(uuid.uuid4())[:7]} docs = DocumentGenerator.make_docs(loads_per_iteration, kv_template, options) doc_sets.append(docs) #load docs self._load_chunk(smart, doc_sets) except Exception as ex: view.results.addError(tc, sys.exc_info()) raise ex