def insert_multi(self,
keys,
ttl=0,
format=None,
persist_to=0,
replicate_to=0):
import bulk_doc_operations.doc_ops as doc_op
docs = []
for key, value in keys.items():
docs.append(self.__translate_to_json_document(key, value, ttl))
doc_op().bulkSet(self.cb, docs)
def call(self):
self.thread_used = threading.currentThread().getName()
self.started = time.time()
try:
temp=0
docs=[]
keys=[]
for i in xrange(self.num_items):
if self.op_type == "create":
# for j in xrange(3):
var = str(json.dumps(self.generate_GleambookMessages(i+self.start_from , self.start_from)))
user = JsonTranscoder().stringToJsonObject(var);
doc = JsonDocument.create(str(i+self.start_from), user);
docs.append(doc)
temp+=1
if temp == self.batch_size:
try:
doc_op().bulkSet(self.msg_bucket, docs)
except:
print "Sleeping for 20 secs"
time.sleep(20)
try:
doc_op().bulkUpsert(self.msg_bucket, docs)
except:
print "GleambookMessages_Docloader:skipping %s documents create"%len(docs)
pass
temp = 0
docs=[]
# global_vars.message_id += 1
# end_message_id = global_vars.message_id
elif self.op_type == "update":
var = str(json.dumps(self.generate_GleambookMessages(i+self.start_from , i+self.start_from)))
user = JsonTranscoder().stringToJsonObject(var);
doc = JsonDocument.create(str(i+self.start_from), user);
docs.append(doc)
if temp == self.batch_size:
try:
doc_op().bulkUpsert(self.msg_bucket, docs)
except:
print "Sleeping for 20 secs"
time.sleep(20)
try:
doc_op().bulkUpsert(self.msg_bucket, docs)
except:
print "GleambookMessages_Docloader:skipping %s documents upload"%len(docs)
pass
temp = 0
docs=[]
elif self.op_type == "delete":
try:
response = self.msg_bucket.remove(str(i+self.start_from));
except:
pass
self.loaded += 1
except Exception, ex:
import traceback
traceback.print_exc()
exc_info = sys.exc_info()
traceback.print_exception(*exc_info)
self.exception = ex
def call(self):
self.thread_used = threading.currentThread().getName()
self.started = time.time()
try:
temp=0
docs=[]
for i in xrange(self.num_items):
start_message_id = global_vars.message_id
if self.op_type == "create":
for j in xrange(random.randint(1,10)):
var = str(json.dumps(self.generate_GleambookMessages(i+self.start_from , global_vars.message_id)))
user = JsonTranscoder().stringToJsonObject(var)
# print i+self.start_from,global_vars.message_id
doc = JsonDocument.create(str(global_vars.message_id), user)
docs.append(doc)
temp += 1
if temp == self.batch_size:
try:
doc_op().bulkSet(self.msg_bucket, docs)
except:
sleep(20, "Exception in Java SDK - create")
try:
doc_op().bulkUpsert(self.msg_bucket, docs)
except:
print "skipping %s documents upload"%len(docs)
pass
temp = 0
docs=[]
global_vars.message_id += 1
end_message_id = global_vars.message_id
elif self.op_type == "update":
var = str(json.dumps(self.generate_GleambookMessages(i+self.start_from , i+start_message_id)))
user = JsonTranscoder().stringToJsonObject(var)
doc = JsonDocument.create(str(i+start_message_id), user)
docs.append(doc)
if temp == self.batch_size:
try:
doc_op().bulkUpsert(self.msg_bucket, docs)
except:
sleep(20, "Exception in Java SDK - create")
try:
doc_op().bulkUpsert(self.msg_bucket, docs)
except:
print "skipping %s documents upload"%len(docs)
pass
temp = 0
docs=[]
elif self.op_type == "delete":
try:
response = self.msg_bucket.remove(str(i+start_message_id))
except:
pass
self.loaded += 1
except Exception, ex:
import traceback
traceback.print_exc()
exc_info = sys.exc_info()
traceback.print_exception(*exc_info)
self.exception = ex
def call(self):
self.thread_used = threading.currentThread().getName()
self.started = time.time()
try:
docs=[]
keys=[]
temp=0
for i in xrange(self.num_items):
if self.op_type == "create":
var = str(json.dumps(self.generate_GleambookUser(i+self.start_from)))
user = JsonTranscoder().stringToJsonObject(var);
doc = JsonDocument.create(str(i+self.start_from), user);
docs.append(doc)
temp += 1
if temp == self.batch_size:
try:
doc_op().bulkSet(self.bucket, docs)
except:
print "Exception from Java SDK - create"
time.sleep(20)
try:
doc_op().bulkUpsert(self.bucket, docs)
except:
print "GleambookUser_Docloader: skipping %s documents create"%len(docs)
pass
temp = 0
docs=[]
# response = self.bucket.insert(doc);
elif self.op_type == "update":
var = str(json.dumps(self.generate_GleambookUser(i+self.start_from)))
user = JsonTranscoder().stringToJsonObject(var);
doc = JsonDocument.create(str(i+self.start_from), user);
docs.append(doc)
temp += 1
if temp == self.batch_size:
try:
doc_op().bulkUpsert(self.bucket, docs)
except:
print "Exception from Java SDK - create"
time.sleep(20)
try:
doc_op().bulkUpsert(self.bucket, docs)
except:
print "GleambookUser_Docloader: skipping %s documents upload"%len(docs)
pass
temp = 0
docs=[]
elif self.op_type == "delete":
try:
response = self.bucket.remove(str(i+self.start_from));
except:
print "Exception from Java SDK - remove"
self.loaded += 1
except Exception, ex:
import traceback
traceback.print_exc()
exc_info = sys.exc_info()
traceback.print_exception(*exc_info)
self.exception = ex
def set_multi(self,
keys,
ttl=0,
format=None,
persist_to=0,
replicate_to=0):
import bulk_doc_operations.doc_ops as doc_op
docs = []
for key, value in keys.items():
docs.append(self.__translate_to_json_document(key, value, ttl))
try:
doc_op().bulkSet(self.cb, docs)
except:
time.sleep(20)
doc_op().bulkUpsert(self.cb, docs)
log.info(
"Calling close inside SDK due to an exception during bulkSet.")
def get_multi(self,
keys,
ttl=0,
quiet=True,
replica=False,
no_format=False):
import bulk_doc_operations.doc_ops as doc_op
try:
data = doc_op().bulkGet(self.cb, keys)
# data = self.cb.get_multi(keys, ttl=ttl, quiet=quiet, replica=replica, no_format=no_format)
return self.__translate_get_multi(data)
except CouchbaseException as e:
try:
time.sleep(10)
data = self.cb.get_multi(keys,
ttl=ttl,
quiet=quiet,
replica=replica,
no_format=no_format)
return self.__translate_get_multi(data)
except CouchbaseException as e:
raise
