def insert_multi(self, keys, ttl=0, format=None, persist_to=0, replicate_to=0): import bulk_doc_operations.doc_ops as doc_op docs = [] for key, value in keys.items(): docs.append(self.__translate_to_json_document(key, value, ttl)) doc_op().bulkSet(self.cb, docs)
def call(self): self.thread_used = threading.currentThread().getName() self.started = time.time() try: temp=0 docs=[] keys=[] for i in xrange(self.num_items): if self.op_type == "create": # for j in xrange(3): var = str(json.dumps(self.generate_GleambookMessages(i+self.start_from , self.start_from))) user = JsonTranscoder().stringToJsonObject(var); doc = JsonDocument.create(str(i+self.start_from), user); docs.append(doc) temp+=1 if temp == self.batch_size: try: doc_op().bulkSet(self.msg_bucket, docs) except: print "Sleeping for 20 secs" time.sleep(20) try: doc_op().bulkUpsert(self.msg_bucket, docs) except: print "GleambookMessages_Docloader:skipping %s documents create"%len(docs) pass temp = 0 docs=[] # global_vars.message_id += 1 # end_message_id = global_vars.message_id elif self.op_type == "update": var = str(json.dumps(self.generate_GleambookMessages(i+self.start_from , i+self.start_from))) user = JsonTranscoder().stringToJsonObject(var); doc = JsonDocument.create(str(i+self.start_from), user); docs.append(doc) if temp == self.batch_size: try: doc_op().bulkUpsert(self.msg_bucket, docs) except: print "Sleeping for 20 secs" time.sleep(20) try: doc_op().bulkUpsert(self.msg_bucket, docs) except: print "GleambookMessages_Docloader:skipping %s documents upload"%len(docs) pass temp = 0 docs=[] elif self.op_type == "delete": try: response = self.msg_bucket.remove(str(i+self.start_from)); except: pass self.loaded += 1 except Exception, ex: import traceback traceback.print_exc() exc_info = sys.exc_info() traceback.print_exception(*exc_info) self.exception = ex
def call(self): self.thread_used = threading.currentThread().getName() self.started = time.time() try: temp=0 docs=[] for i in xrange(self.num_items): start_message_id = global_vars.message_id if self.op_type == "create": for j in xrange(random.randint(1,10)): var = str(json.dumps(self.generate_GleambookMessages(i+self.start_from , global_vars.message_id))) user = JsonTranscoder().stringToJsonObject(var) # print i+self.start_from,global_vars.message_id doc = JsonDocument.create(str(global_vars.message_id), user) docs.append(doc) temp += 1 if temp == self.batch_size: try: doc_op().bulkSet(self.msg_bucket, docs) except: sleep(20, "Exception in Java SDK - create") try: doc_op().bulkUpsert(self.msg_bucket, docs) except: print "skipping %s documents upload"%len(docs) pass temp = 0 docs=[] global_vars.message_id += 1 end_message_id = global_vars.message_id elif self.op_type == "update": var = str(json.dumps(self.generate_GleambookMessages(i+self.start_from , i+start_message_id))) user = JsonTranscoder().stringToJsonObject(var) doc = JsonDocument.create(str(i+start_message_id), user) docs.append(doc) if temp == self.batch_size: try: doc_op().bulkUpsert(self.msg_bucket, docs) except: sleep(20, "Exception in Java SDK - create") try: doc_op().bulkUpsert(self.msg_bucket, docs) except: print "skipping %s documents upload"%len(docs) pass temp = 0 docs=[] elif self.op_type == "delete": try: response = self.msg_bucket.remove(str(i+start_message_id)) except: pass self.loaded += 1 except Exception, ex: import traceback traceback.print_exc() exc_info = sys.exc_info() traceback.print_exception(*exc_info) self.exception = ex
def call(self): self.thread_used = threading.currentThread().getName() self.started = time.time() try: docs=[] keys=[] temp=0 for i in xrange(self.num_items): if self.op_type == "create": var = str(json.dumps(self.generate_GleambookUser(i+self.start_from))) user = JsonTranscoder().stringToJsonObject(var); doc = JsonDocument.create(str(i+self.start_from), user); docs.append(doc) temp += 1 if temp == self.batch_size: try: doc_op().bulkSet(self.bucket, docs) except: print "Exception from Java SDK - create" time.sleep(20) try: doc_op().bulkUpsert(self.bucket, docs) except: print "GleambookUser_Docloader: skipping %s documents create"%len(docs) pass temp = 0 docs=[] # response = self.bucket.insert(doc); elif self.op_type == "update": var = str(json.dumps(self.generate_GleambookUser(i+self.start_from))) user = JsonTranscoder().stringToJsonObject(var); doc = JsonDocument.create(str(i+self.start_from), user); docs.append(doc) temp += 1 if temp == self.batch_size: try: doc_op().bulkUpsert(self.bucket, docs) except: print "Exception from Java SDK - create" time.sleep(20) try: doc_op().bulkUpsert(self.bucket, docs) except: print "GleambookUser_Docloader: skipping %s documents upload"%len(docs) pass temp = 0 docs=[] elif self.op_type == "delete": try: response = self.bucket.remove(str(i+self.start_from)); except: print "Exception from Java SDK - remove" self.loaded += 1 except Exception, ex: import traceback traceback.print_exc() exc_info = sys.exc_info() traceback.print_exception(*exc_info) self.exception = ex
def set_multi(self, keys, ttl=0, format=None, persist_to=0, replicate_to=0): import bulk_doc_operations.doc_ops as doc_op docs = [] for key, value in keys.items(): docs.append(self.__translate_to_json_document(key, value, ttl)) try: doc_op().bulkSet(self.cb, docs) except: time.sleep(20) doc_op().bulkUpsert(self.cb, docs) log.info( "Calling close inside SDK due to an exception during bulkSet.")
def get_multi(self, keys, ttl=0, quiet=True, replica=False, no_format=False): import bulk_doc_operations.doc_ops as doc_op try: data = doc_op().bulkGet(self.cb, keys) # data = self.cb.get_multi(keys, ttl=ttl, quiet=quiet, replica=replica, no_format=no_format) return self.__translate_get_multi(data) except CouchbaseException as e: try: time.sleep(10) data = self.cb.get_multi(keys, ttl=ttl, quiet=quiet, replica=replica, no_format=no_format) return self.__translate_get_multi(data) except CouchbaseException as e: raise