def _run(self):
v = None
try:
options = self._options
v = VBucketAwareCouchbaseClient(self._server, options.bucket, options.password, options.verbose)
number_of_items = (int(options.items) / int(options.num_of_threads))
value = StringUtil.create_value("*", int(options.value_size))
for i in range(0, number_of_items):
if self.shut_down:
break
key = "{0}-{1}".format(options.key_prefix, str(uuid.uuid4())[:5])
if options.load_json:
document = "\"name\":\"pymc-{0}\"".format(key, key)
document = document + ",\"age\":{0}".format(random.randint(0, 1000))
document = document + ",\"index\":{0}".format(i)
document = "{" + document + "}"
try:
self._profile_before()
v.set(key, 0, 0, document)
self._profile_after()
except MemcachedTimeoutException:
self._stats["timeouts"] += 1
else:
try:
self._profile_before()
v.set(key, 0, 0, value)
self._profile_after()
except MemcachedTimeoutException:
self._stats["timeouts"] += 1
self._bar.update()
v.done()
v = None
except:
if v:
v.done()
def write(self, record):
for i in range(5):
try:
# todo: check for timeout and flags
self.client.set(str(record['id'].encode('utf-8')), 0, 0, json.dumps(record['value']))
return
except MemcachedTimeoutException as e:
pass
except:
self.client.done()
self.client = VBucketAwareCouchbaseClient(self.server, self.bucket, self.password, self.verbose)
print 'unable to set key {0}'.format(str(record['id'].encode('utf-8')))
def add(request):
couchbase = VBucketAwareCouchbaseClient("http://localhost:8091/pools/default", "default", "")
a,b,c = couchbase.get('allrssfeeds')
rss_dict = json.loads(c)
new_feed = request.raw_post_data
found = False
for item in rss_dict:
if item['url'] == new_feed:
found = True
break
if not found:
rss_dict.append({'url':new_feed, 'status':'valid'})
couchbase.set('allrssfeeds', 0, 0,json.dumps(rss_dict))
return render_to_response('template/home.html',
{'title': 'allrssfeeds', 'feeds': _filter_invalid(rss_dict)})
def remove(request):
couchbase = VBucketAwareCouchbaseClient("http://localhost:8091/pools/default", "default", "")
which_rss = request.raw_post_data
a,b,c = couchbase.get('allrssfeeds')
print c
rss_dict = json.loads(c)
for item in rss_dict:
if item['url'] == which_rss:
item['status'] = 'invalid'
break
print rss_dict
print which_rss
couchbase.set('allrssfeeds', 0, 0,json.dumps(rss_dict))
a,b,c = couchbase.get('allrssfeeds')
return render_to_response('template/home.html',
{'title': 'allrssfeeds', 'feeds':_filter_invalid(rss_dict)})
def __init__(self, destination):
# username:[email protected]:8091/bucket
m = re.match('^([^:]+):([^@]*)@([^:]+):([^/]+)/(.+)$', destination)
self.username = m.group(1)
self.password = m.group(2)
self.host = m.group(3)
self.port = m.group(4)
self.bucket = m.group(5)
self.bucket_port = 11211
self.bucket_password = ''
self.verbose = False
# todo: use server username/password to query the bucket password/port if needed
self.server = "http://{0}:{1}/pools/default".format(self.host, self.port)
self.client = VBucketAwareCouchbaseClient(self.server, self.bucket, self.password, self.verbose)
def test_vbucketawarecouchbaseclient_object_construction(self):
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
# Trigger a warning.
cb = VBucketAwareCouchbaseClient(
"http://" + self.host + ':' + self.port + "/pools/default",
self.bucket_name, "")
self.assertIsInstance(cb.servers, types.ListType)
# Verify some things
self.assertTrue(len(w) == 1)
self.assertTrue("deprecated" in str(w[-1].message))
def _run(self):
v = None
try:
options = self._options
if not options.password:
options.password = ""
v = VBucketAwareCouchbaseClient(self._server, options.bucket,
options.password, options.verbose)
number_of_items = (int(options.items) /
int(options.num_of_threads))
value = StringUtil.create_value("*", int(options.value_size))
for i in range(0, number_of_items):
if self.shut_down:
break
key = "{0}-{1}".format(options.key_prefix,
str(uuid.uuid4())[:5])
if options.load_json:
document = "\"name\":\"pymc-{0}\"".format(key, key)
document = document + ",\"age\":{0}".format(
random.randint(0, 1000))
document = document + ",\"index\":{0}".format(i)
document = "{" + document + "}"
try:
self._profile_before()
v.set(key, 0, 0, document)
self._profile_after()
except MemcachedTimeoutException:
self._stats["timeouts"] += 1
else:
try:
self._profile_before()
v.set(key, 0, 0, value)
self._profile_after()
except MemcachedTimeoutException:
self._stats["timeouts"] += 1
self._bar.update()
v.done()
v = None
except:
if v:
v.done()
class CouchbaseWriter(migrator.Writer):
def __init__(self, destination):
# username:[email protected]:8091/bucket
m = re.match('^([^:]+):([^@]*)@([^:]+):([^/]+)/(.+)$', destination)
self.username = m.group(1)
self.password = m.group(2)
self.host = m.group(3)
self.port = m.group(4)
self.bucket = m.group(5)
self.bucket_port = 11211
self.bucket_password = ''
self.verbose = False
# todo: use server username/password to query the bucket password/port if needed
self.server = "http://{0}:{1}/pools/default".format(self.host, self.port)
self.client = VBucketAwareCouchbaseClient(self.server, self.bucket, self.password, self.verbose)
def write(self, record):
for i in range(5):
try:
# todo: check for timeout and flags
self.client.set(str(record['id'].encode('utf-8')), 0, 0, json.dumps(record['value']))
return
except MemcachedTimeoutException as e:
pass
except:
self.client.done()
self.client = VBucketAwareCouchbaseClient(self.server, self.bucket, self.password, self.verbose)
print 'unable to set key {0}'.format(str(record['id'].encode('utf-8')))
def close(self):
self.client.done()
class CouchbaseClientTest(unittest.TestCase):
def setUp(self):
self.url = config['node-1']['url']
self.bucket = config['node-1']['bucket']
self.client = VBucketAwareCouchbaseClient(self.url, self.bucket, "", True)
def tearDown(self):
self.client.done()
def test_set_and_get(self):
kvs = [(str(uuid.uuid4()), str(uuid.uuid4())) for i in range(0, 100)]
for k, v in kvs:
self.client.set(k, 0, 0, v)
for k, v in kvs:
self.client.get(k)
def test_set_and_delete(self):
kvs = [(str(uuid.uuid4()), str(uuid.uuid4())) for i in range(0, 100)]
for k, v in kvs:
self.client.set(k, 0, 0, v)
for k, v in kvs:
self.client.delete(k)
def setUp(self):
self.url = config['node-1']['url']
self.bucket = config['node-1']['bucket']
self.client = VBucketAwareCouchbaseClient(self.url, self.bucket, "", True)
def load(request):
couchbase = VBucketAwareCouchbaseClient("http://localhost:8091/pools/default", "default", "")
a,b,c = couchbase.get('allrssfeeds')
rss_dict = json.loads(c)
return render_to_response('template/home.html',
{'title': 'allrssfeeds', 'feeds': _filter_invalid(rss_dict)})
