class CouchbaseWriter(migrator.Writer): def __init__(self, destination): # username:[email protected]:8091/bucket m = re.match('^([^:]+):([^@]*)@([^:]+):([^/]+)/(.+)$', destination) self.username = m.group(1) self.password = m.group(2) self.host = m.group(3) self.port = m.group(4) self.bucket = m.group(5) self.bucket_port = 11211 self.bucket_password = '' self.verbose = False # todo: use server username/password to query the bucket password/port if needed self.server = "http://{0}:{1}/pools/default".format(self.host, self.port) self.client = VBucketAwareCouchbaseClient(self.server, self.bucket, self.password, self.verbose) def write(self, record): for i in range(5): try: # todo: check for timeout and flags self.client.set(str(record['id'].encode('utf-8')), 0, 0, json.dumps(record['value'])) return except MemcachedTimeoutException as e: pass except: self.client.done() self.client = VBucketAwareCouchbaseClient(self.server, self.bucket, self.password, self.verbose) print 'unable to set key {0}'.format(str(record['id'].encode('utf-8'))) def close(self): self.client.done()
def _run(self): v = None try: options = self._options v = VBucketAwareCouchbaseClient(self._server, options.bucket, options.password, options.verbose) number_of_items = (int(options.items) / int(options.num_of_threads)) value = StringUtil.create_value("*", int(options.value_size)) for i in range(0, number_of_items): if self.shut_down: break key = "{0}-{1}".format(options.key_prefix, str(uuid.uuid4())[:5]) if options.load_json: document = "\"name\":\"pymc-{0}\"".format(key, key) document = document + ",\"age\":{0}".format(random.randint(0, 1000)) document = document + ",\"index\":{0}".format(i) document = "{" + document + "}" try: self._profile_before() v.set(key, 0, 0, document) self._profile_after() except MemcachedTimeoutException: self._stats["timeouts"] += 1 else: try: self._profile_before() v.set(key, 0, 0, value) self._profile_after() except MemcachedTimeoutException: self._stats["timeouts"] += 1 self._bar.update() v.done() v = None except: if v: v.done()
def _run(self): v = None try: options = self._options if not options.password: options.password = "" v = VBucketAwareCouchbaseClient(self._server, options.bucket, options.password, options.verbose) number_of_items = (int(options.items) / int(options.num_of_threads)) value = StringUtil.create_value("*", int(options.value_size)) for i in range(0, number_of_items): if self.shut_down: break key = "{0}-{1}".format(options.key_prefix, str(uuid.uuid4())[:5]) if options.load_json: document = "\"name\":\"pymc-{0}\"".format(key, key) document = document + ",\"age\":{0}".format( random.randint(0, 1000)) document = document + ",\"index\":{0}".format(i) document = "{" + document + "}" try: self._profile_before() v.set(key, 0, 0, document) self._profile_after() except MemcachedTimeoutException: self._stats["timeouts"] += 1 else: try: self._profile_before() v.set(key, 0, 0, value) self._profile_after() except MemcachedTimeoutException: self._stats["timeouts"] += 1 self._bar.update() v.done() v = None except: if v: v.done()
class CouchbaseClientTest(unittest.TestCase): def setUp(self): self.url = config['node-1']['url'] self.bucket = config['node-1']['bucket'] self.client = VBucketAwareCouchbaseClient(self.url, self.bucket, "", True) def tearDown(self): self.client.done() def test_set_and_get(self): kvs = [(str(uuid.uuid4()), str(uuid.uuid4())) for i in range(0, 100)] for k, v in kvs: self.client.set(k, 0, 0, v) for k, v in kvs: self.client.get(k) def test_set_and_delete(self): kvs = [(str(uuid.uuid4()), str(uuid.uuid4())) for i in range(0, 100)] for k, v in kvs: self.client.set(k, 0, 0, v) for k, v in kvs: self.client.delete(k)