def gitx(request):
'''
get all commits for this project
for top 20 , do view call to find out the builds
'''
logger = logging.getLogger('dashboard')
project = request.GET.get('project')
limit = request.GET.get('limit')
couchbase = CouchbaseClient("http://localhost:8091/pools/default", "default", "")
cbclient= CbClient("localhost", 8091, "Administrator", "password")
a,b,value = couchbase.get(project.encode('ascii','ignore'))
commits = json.loads(value)["tree"]
counter = 0
data = list()
for commit in commits:
commit["builds"]=[]
counter = counter + 1
filter_value = "\"%s-%s\"" % (project, commit["revision"])
params={"startkey":filter_value,"endkey":filter_value}
qr = cbclient.query(bucket='default', ddoc='builds', view='commits', limit=100,params=params)
for item in qr["rows"]:
logger.error(commit["builds"])
fullversion = item["value"][1].encode('ascii','ignore')
if fullversion not in commit["builds"]:
commit["builds"].append(fullversion)
data.append(commit)
end_index = commit["message"].find("\n") or len(commit["message"] - 1)
commit["message"] = commit["message"][0:end_index]
if counter == limit:
break
return render_to_response('../templates/dashboard/commits.html',
{'title': 'commits', 'data': data})
class CouchbaseClientTest(Base):
def setUp(self):
super(CouchbaseClientTest, self).setUp()
self.client = CouchbaseClient(self.url, self.bucket_name, "", True)
def tearDown(self):
self.client.done()
@nottest
def setup_memcached_bucket(self):
self.memcached_bucket = 'testing-memcached'
self.rest_client = RestConnection({'ip': self.host,
'port': self.port,
'username': self.username,
'password': self.password})
self.rest_client.create_bucket(self.memcached_bucket,
bucketType='memcached',
authType='sasl', ramQuotaMB=64)
self.client_for_memcached_bucket = \
CouchbaseClient(self.url, self.memcached_bucket, verbose=True)
@nottest
def teardown_memcached_bucket(self):
self.rest_client.delete_bucket(self.memcached_bucket)
@attr(cbv="1.0.0")
def test_set_integer_value(self):
self.client.set('int', 0, 0, 10)
self.assertEqual(self.client.get('int')[2], 10,
'value should be the integer 10')
self.client.incr('int')
self.assertEqual(self.client.get('int')[2], 11,
'value should be the integer 11')
self.client.delete('int')
def test_two_client_incr(self):
"""http://www.couchbase.com/issues/browse/PYCBC-16"""
key = 'test_two_client_incr'
client_one = self.client
client_two = CouchbaseClient(self.url, self.bucket_name, "", True)
# Client one sets a numeric key
client_one.set(key, 0, 0, 20)
# Client two tries to increment this numeric key
(i, cas) = client_two.incr(key)
self.assertEqual(i, 21)
# Client two should be able to keep incrementing this key
(i, cas) = client_two.incr(key)
self.assertEqual(i, 22)
(_, cas, i) = client_two.get(key)
self.assertEqual(i, 22)
(i, cas) = client_two.incr(key)
self.assertEqual(i, 23)
def test_bucket_of_type_memcached(self):
self.setup_memcached_bucket()
self.assertIsInstance(self.client_for_memcached_bucket,
CouchbaseClient)
self.teardown_memcached_bucket()
def create(self, request):
logger = logging.getLogger('dashboard')
couchbase = CouchbaseClient("http://localhost:8091/pools/default", "default", "")
kv_data = {}
json_data = json.loads(request.raw_post_data)
for object in json_data:
name = json_data[object]
skip_tokens = ["CHANGES", "1.7", "1.8.1r", "preview-", "-manifest"]
if not RawBuildNameDecodedHelper.is_substring(skip_tokens, name):
data = RawBuildNameDecodedHelper.populate_build_meta(name)
kv_data[name] = data
result = {}
num_updated_rows = 0
for key in kv_data:
item = kv_data[key]
if "package" not in item or "fullversion" not in item:
continue
item = kv_data[key]
_id = "%s-%s-%s-%s" % (item["package"], item["arch"],
item["os"], item["fullversion"])
if item["package"] == "manifest":
content = self._download(item["url"])
to_json = self._to_json(content, item["fullversion"])
item["commits"] = to_json
couchbase.set(_id.encode("ascii", "ignore"), 0, 0, json.dumps(item).encode("ascii", "ignore"))
num_updated_rows += 1
logger.info('updated %d rows in the database' % (num_updated_rows))
return {"status": "ok"}
def __init__(self, name, server):
self.server = server
self.name = name
rest = server._rest()
self.info = rest.get_bucket(self.name)
self.password = self.info.saslPassword
ip, port, rest_username, rest_password = server._rest_info()
formatter_uri = "http://%s:%s/pools/default"
self.mc_client = CouchbaseClient(formatter_uri % (ip, port), self.name,
self.password)
def populate_all_manifests(request):
logger = logging.getLogger('dashboard')
couchbase = CouchbaseClient("http://localhost:8091/pools/default", "default", "")
cbclient= CbClient("localhost", 8091, "Administrator", "password")
qr = cbclient.query(bucket='default', ddoc='builds', view='manifests', limit=20000)
for item in qr["rows"]:
doc = item["value"]
if doc["name"].find("-manifest.xml") == -1:
# if doc["name"].find("-manifest.xml") == -1 and 'commits' not in doc:
manifest_commits = ManifestPopulate(doc, doc["url"]).json()
doc['commits'] = manifest_commits
couchbase.set(doc['name'].encode('ascii','ignore'), 0, 0,
json.dumps(doc).encode('ascii','ignore'))
logger.info("retreived git commits for %s" % (doc['name']))
return render_to_response('../templates/dashboard/empty.html')
def populate_all_manifests(request):
logger = logging.getLogger('dashboard')
couchbase = CouchbaseClient("http://localhost:8091/pools/default",
"default", "")
cbclient = CbClient("localhost", 8091, "Administrator", "password")
qr = cbclient.query(bucket='default',
ddoc='builds',
view='manifests',
limit=20000)
for item in qr["rows"]:
doc = item["value"]
if doc["name"].find("-manifest.xml") == -1:
# if doc["name"].find("-manifest.xml") == -1 and 'commits' not in doc:
manifest_commits = ManifestPopulate(doc, doc["url"]).json()
doc['commits'] = manifest_commits
couchbase.set(doc['name'].encode('ascii', 'ignore'), 0, 0,
json.dumps(doc).encode('ascii', 'ignore'))
logger.info("retreived git commits for %s" % (doc['name']))
return render_to_response('../templates/dashboard/empty.html')
def test_two_client_incr(self):
"""http://www.couchbase.com/issues/browse/PYCBC-16"""
key = 'test_two_client_incr'
client_one = self.client
client_two = CouchbaseClient(self.url, self.bucket_name, "", True)
# Client one sets a numeric key
client_one.set(key, 0, 0, 20)
# Client two tries to increment this numeric key
(i, cas) = client_two.incr(key)
self.assertEqual(i, 21)
# Client two should be able to keep incrementing this key
(i, cas) = client_two.incr(key)
self.assertEqual(i, 22)
(_, cas, i) = client_two.get(key)
self.assertEqual(i, 22)
(i, cas) = client_two.incr(key)
self.assertEqual(i, 23)
def __init__(self, name, server):
self.server = server
self.name = name
rest = server._rest()
self.password = rest.get_bucket(self.name).saslPassword
ip, port, rest_username, rest_password = server._rest_info()
formatter_uri = "http://%s:%s/pools/default"
self.mc_client = CouchbaseClient(formatter_uri % (ip, port), self.name,
self.password)
class CouchbaseClientTest(Base):
def setUp(self):
super(CouchbaseClientTest, self).setUp()
self.client = CouchbaseClient(self.url, self.bucket_name, "", True)
def tearDown(self):
self.client.flush()
self.client.done()
@nottest
def setup_memcached_bucket(self):
self.memcached_bucket = 'testing-memcached'
self.rest_client = RestConnection({'ip': self.host,
'port': self.port,
'username': self.username,
'password': self.password})
self.rest_client.create_bucket(self.memcached_bucket,
bucketType='memcached',
authType='sasl', ramQuotaMB=64)
self.client_for_memcached_bucket = \
CouchbaseClient(self.url, self.memcached_bucket, verbose=True)
@nottest
def teardown_memcached_bucket(self):
self.rest_client.delete_bucket(self.memcached_bucket)
@attr(cbv="1.0.0")
def test_set_integer_value(self):
self.client.set('int', 0, 0, 10)
self.assertEqual(self.client.get('int')[2], 10,
'value should be the integer 10')
self.client.incr('int')
self.assertEqual(self.client.get('int')[2], 11,
'value should be the integer 11')
@attr(cbv="1.0.0")
def test_bucket_of_type_memcached(self):
self.setup_memcached_bucket()
self.assertIsInstance(self.client_for_memcached_bucket,
CouchbaseClient)
self.teardown_memcached_bucket()
def test_old_set_get_create_example(self):
from couchbase.couchbaseclient import CouchbaseClient
from couchbase.rest_client import RestConnection, RestHelper
client = CouchbaseClient(self.url, self.bucket_name, "", False)
client.set("key1", 0, 0, "value1")
client.get("key1")
server_info = {
"ip": self.host,
"port": self.port,
"username": self.username,
"password": self.password
}
rest = RestConnection(server_info)
rest.create_bucket(bucket='newbucket',
ramQuotaMB=100,
authType='none',
saslPassword='',
replicaNumber=1,
proxyPort=11215,
bucketType='membase')
self.assertTrue(RestHelper(rest).bucket_exists('newbucket'))
rest.delete_bucket('newbucket')
def gitx(request):
'''
get all commits for this project
for top 20 , do view call to find out the builds
'''
logger = logging.getLogger('dashboard')
project = request.GET.get('project')
limit = request.GET.get('limit')
couchbase = CouchbaseClient("http://localhost:8091/pools/default",
"default", "")
cbclient = CbClient("localhost", 8091, "Administrator", "password")
a, b, value = couchbase.get(project.encode('ascii', 'ignore'))
commits = json.loads(value)["tree"]
counter = 0
data = list()
for commit in commits:
commit["builds"] = []
counter = counter + 1
filter_value = "\"%s-%s\"" % (project, commit["revision"])
params = {"startkey": filter_value, "endkey": filter_value}
qr = cbclient.query(bucket='default',
ddoc='builds',
view='commits',
limit=100,
params=params)
for item in qr["rows"]:
logger.error(commit["builds"])
fullversion = item["value"][1].encode('ascii', 'ignore')
if fullversion not in commit["builds"]:
commit["builds"].append(fullversion)
data.append(commit)
end_index = commit["message"].find("\n") or len(commit["message"] - 1)
commit["message"] = commit["message"][0:end_index]
if counter == limit:
break
return render_to_response('../templates/dashboard/commits.html', {
'title': 'commits',
'data': data
})
def test_old_set_get_create_example(self):
from couchbase.couchbaseclient import CouchbaseClient
from couchbase.rest_client import RestConnection, RestHelper
client = CouchbaseClient(self.url, self.bucket_name, "", False)
client.set("key1", 0, 0, "value1")
client.get("key1")
server_info = {"ip": self.host, "port": self.port, "username": self.username, "password": self.password}
rest = RestConnection(server_info)
rest.create_bucket(
bucket="newbucket",
ramQuotaMB=100,
authType="none",
saslPassword="",
replicaNumber=1,
proxyPort=11215,
bucketType="membase",
)
self.assertTrue(RestHelper(rest).bucket_exists("newbucket"))
rest.delete_bucket("newbucket")
def setup_memcached_bucket(self):
self.memcached_bucket = 'testing-memcached'
self.rest_client = RestConnection({
'ip': self.host,
'port': self.port,
'username': self.username,
'password': self.password
})
self.rest_client.create_bucket(self.memcached_bucket,
bucketType='memcached',
authType='sasl',
ramQuotaMB=64)
self.client_for_memcached_bucket = \
CouchbaseClient(self.url, self.memcached_bucket, verbose=True)
def create(self, request, *args, **kwargs):
couchbase = CouchbaseClient("http://localhost:8091/pools/default", "default", "")
if request.content_type:
data = request.data
_id = "%s-%s-%s-%s" % (data["package"], data["arch"], data["os"], data["fullversion"])
_id = _id.encode("ascii", "ignore")
couchbase.set(_id, 0, 0, json.dumps(data).encode("ascii", "ignore"))
couchbase.done()
return {"status": "ok"}
else:
return {"status": "failed", "data": request.data["os"]}
def populate_commits(request):
"""
for each commit create an entry in couchbase
"""
logger = logging.getLogger('dashboard')
project = request.GET.get('project')
refs = request.GET.get('refs')
couchbase = CouchbaseClient("http://localhost:8091/pools/default",
"default", "")
refs = request.GET.get('refs')
gp = GitPopulate("/space/repo/couchbase-2.0/", project, refs)
commits = gp.fetch_commits()
for commit in commits["tree"]:
key = "%s-%s" % (commit["project"], commit["revision"])
value = json.dumps(commit).encode('ascii', 'ignore')
key = key.encode('ascii', 'ignore')
couchbase.set(key, 0, 0, value)
key = '%s' % (project)
couchbase.set(key.encode('ascii', 'ignore'), 0, 0,
json.dumps(commits).encode('ascii', 'ignore'))
couchbase.done()
return render_to_response('../templates/dashboard/empty.html')
def populate_commits(request):
"""
for each commit create an entry in couchbase
"""
logger = logging.getLogger('dashboard')
project = request.GET.get('project')
refs = request.GET.get('refs')
couchbase = CouchbaseClient("http://localhost:8091/pools/default", "default", "")
refs = request.GET.get('refs')
gp = GitPopulate("/space/repo/couchbase-2.0/", project, refs)
commits = gp.fetch_commits()
for commit in commits["tree"]:
key = "%s-%s" % (commit["project"], commit["revision"])
value = json.dumps(commit).encode('ascii', 'ignore')
key = key.encode('ascii', 'ignore')
couchbase.set(key, 0, 0, value)
key = '%s' % (project)
couchbase.set(key.encode('ascii', 'ignore'), 0, 0,
json.dumps(commits).encode('ascii', 'ignore'))
couchbase.done()
return render_to_response('../templates/dashboard/empty.html')
def test_two_client_incr(self):
"""http://www.couchbase.com/issues/browse/PYCBC-16"""
key = 'test_two_client_incr'
client_one = self.client
client_two = CouchbaseClient(self.url, self.bucket_name, "", True)
# Client one sets a numeric key
client_one.set(key, 0, 0, 20)
# Client two tries to increment this numeric key
(i, cas) = client_two.incr(key)
self.assertEqual(i, 21)
# Client two should be able to keep incrementing this key
(i, cas) = client_two.incr(key)
self.assertEqual(i, 22)
(_, cas, i) = client_two.get(key)
self.assertEqual(i, 22)
(i, cas) = client_two.incr(key)
self.assertEqual(i, 23)
class CouchbaseClientTest(Base):
def setUp(self):
super(CouchbaseClientTest, self).setUp()
self.client = CouchbaseClient(self.url, self.bucket_name, "", True)
def tearDown(self):
self.client.done()
@nottest
def setup_memcached_bucket(self):
self.memcached_bucket = 'testing-memcached'
self.rest_client = RestConnection({
'ip': self.host,
'port': self.port,
'username': self.username,
'password': self.password
})
self.rest_client.create_bucket(self.memcached_bucket,
bucketType='memcached',
authType='sasl',
ramQuotaMB=64)
self.client_for_memcached_bucket = \
CouchbaseClient(self.url, self.memcached_bucket, verbose=True)
@nottest
def teardown_memcached_bucket(self):
self.rest_client.delete_bucket(self.memcached_bucket)
@attr(cbv="1.0.0")
def test_set_integer_value(self):
self.client.set('int', 0, 0, 10)
self.assertEqual(
self.client.get('int')[2], 10, 'value should be the integer 10')
self.client.incr('int')
self.assertEqual(
self.client.get('int')[2], 11, 'value should be the integer 11')
self.client.delete('int')
def test_two_client_incr(self):
"""http://www.couchbase.com/issues/browse/PYCBC-16"""
key = 'test_two_client_incr'
client_one = self.client
client_two = CouchbaseClient(self.url, self.bucket_name, "", True)
# Client one sets a numeric key
client_one.set(key, 0, 0, 20)
# Client two tries to increment this numeric key
(i, cas) = client_two.incr(key)
self.assertEqual(i, 21)
# Client two should be able to keep incrementing this key
(i, cas) = client_two.incr(key)
self.assertEqual(i, 22)
(_, cas, i) = client_two.get(key)
self.assertEqual(i, 22)
(i, cas) = client_two.incr(key)
self.assertEqual(i, 23)
def test_bucket_of_type_memcached(self):
self.setup_memcached_bucket()
self.assertIsInstance(self.client_for_memcached_bucket,
CouchbaseClient)
self.teardown_memcached_bucket()
class Bucket(object):
"""Handles Bucket management as well as key/value access for a specific
bucket."""
def __init__(self, name, server):
self.server = server
self.name = name
rest = server._rest()
self.password = rest.get_bucket(self.name).saslPassword
ip, port, rest_username, rest_password = server._rest_info()
formatter_uri = "http://%s:%s/pools/default"
self.mc_client = CouchbaseClient(formatter_uri % (ip, port), self.name,
self.password)
def append(self, key, value, cas=0):
return self.mc_client.append(key, value, cas)
def prepend(self, key, value, cas=0):
return self.mc_client.prepend(key, value, cas)
def incr(self, key, amt=1, init=0, exp=0):
return self.mc_client.incr(key, amt, init, exp)
def decr(self, key, amt=1, init=0, exp=0):
return self.mc_client.decr(key, amt, init, exp)
def set(self, key, expiration, flags, value):
return self.mc_client.set(key, expiration, flags, value)
def add(self, key, exp, flags, val):
return self.mc_client.add(key, exp, flags, val)
def replace(self, key, exp, flags, val):
return self.mc_client.replace(key, exp, flags, val)
def get(self, key):
return self.mc_client.get(key)
def getl(self, key, exp=15):
return self.mc_client.getl(key, exp)
def cas(self, key, exp, flags, oldVal, val):
return self.mc_client.cas(key, exp, flags, oldVal, val)
def touch(self, key, exp):
return self.mc_client.touch(key, exp)
def gat(self, key, exp):
return self.mc_client.gat(key, exp)
def stats(self, sub=''):
return self.mc_client.stats(sub)
def delete(self, key, cas=0):
if key.startswith('_design/'):
# this is a design doc, we need to handle it differently
view = key.split('/')[1]
rest = self.server._rest()
rest.delete_view(self.name, view)
else:
return self.mc_client.delete(key, cas)
def save(self, document):
value = deepcopy(document)
if '_id' in value:
key = value['_id']
del value['_id']
else:
key = str(uuid.uuid4())
if '$flags' in value:
flags = value['$flags']
del value['$flags']
else:
flags = 0
if '$expiration' in value:
expiration = value['$expiration']
del value['$expiration']
else:
expiration = 0
if key.startswith('_design/'):
# this is a design doc, we need to handle it differently
view = key.split('/')[1]
rest = self.server._rest()
rest.create_design_doc(self.name, view, json.dumps(value))
else:
if '_rev' in value:
# couchbase works in clobber mode so for "set" _rev is useless
del value['_rev']
self.set(key, expiration, flags, json.dumps(value))
return key
def __setitem__(self, key, value):
if isinstance(value, dict):
self.set(key, value['expiration'], value['flags'], value['value'])
else:
self.set(key, 0, 0, value)
def __getitem__(self, key):
return self.get(key)
def view(self, view, **options):
params = deepcopy(options)
limit = None
if 'limit' in params:
limit = params['limit']
del params['limit']
if view.startswith("_design/"):
view_s = view.split('/')
view_doc = view_s[1]
view_map = view_s[3]
else:
view_doc = view
view_map = None
rest = self.server._rest()
results = rest.view_results(self.name, view_doc, view_map,
params, limit)
if 'rows' in results:
return results['rows']
else:
return None
class Bucket(object):
"""Handles Bucket management as well as key/value access for a specific
bucket."""
def __init__(self, name, server):
self.server = server
self.name = name
rest = server._rest()
self.info = rest.get_bucket(self.name)
self.password = self.info.saslPassword
ip, port, rest_username, rest_password = server._rest_info()
formatter_uri = "http://%s:%s/pools/default"
self.mc_client = CouchbaseClient(formatter_uri % (ip, port), self.name,
self.password)
def __del__(self):
self.mc_client.done()
def append(self, key, value, cas=0):
return self.mc_client.append(key, value, cas)
def prepend(self, key, value, cas=0):
return self.mc_client.prepend(key, value, cas)
def incr(self, key, amt=1, init=0, exp=0):
return self.mc_client.incr(key, amt, init, exp)
def decr(self, key, amt=1, init=0, exp=0):
return self.mc_client.decr(key, amt, init, exp)
def set(self, key, expiration, flags, value):
if isinstance(value, dict):
value = json.dumps(value)
return self.mc_client.set(key, expiration, flags, value)
def add(self, key, exp, flags, val):
return self.mc_client.add(key, exp, flags, val)
def replace(self, key, exp, flags, val):
return self.mc_client.replace(key, exp, flags, val)
def get(self, key):
return self.mc_client.get(key)
def getl(self, key, exp=15):
return self.mc_client.getl(key, exp)
def cas(self, key, exp, flags, oldVal, val):
return self.mc_client.cas(key, exp, flags, oldVal, val)
def touch(self, key, exp):
return self.mc_client.touch(key, exp)
def gat(self, key, exp):
return self.mc_client.gat(key, exp)
def stats(self, sub=''):
return self.mc_client.stats(sub)
def delete(self, key, cas=0):
if key.startswith('_design/'):
# this is a design doc, we need to handle it differently
view = key.split('/')[1]
rest = self.server._rest()
rest.delete_view(self.name, view)
else:
return self.mc_client.delete(key, cas)
def save(self, document):
warnings.warn("save is deprecated; use set instead",
DeprecationWarning)
value = deepcopy(document)
if '_id' in value:
key = value['_id']
del value['_id']
else:
key = str(uuid.uuid4())
if '$flags' in value:
flags = value['$flags']
del value['$flags']
else:
flags = 0
if '$expiration' in value:
expiration = value['$expiration']
del value['$expiration']
else:
expiration = 0
if key.startswith('_design/'):
self[key] = value
else:
if '_rev' in value:
# couchbase works in clobber mode so for "set" _rev is useless
del value['_rev']
self.set(key, expiration, flags, json.dumps(value))
return key
def __setitem__(self, key, value):
if isinstance(value, dict):
if 'expiration' in value or 'flags' in value:
assert 'value' in value
if isinstance(value['value'], dict):
v = json.dumps(value['value'])
else:
v = value['value']
self.set(key, value.get('expiration', 0),
value.get('flags', 0), v)
elif key.startswith('_design/'):
rest = self.server._rest()
rest.create_design_doc(self.name, key[8:], json.dumps(value))
else:
self.set(key, 0, 0, json.dumps(value))
else:
self.set(key, 0, 0, value)
def __getitem__(self, key):
if key.startswith("_design/"):
rest = self.server._rest()
doc = rest.get_design_doc(self.name, key[8:])
return DesignDoc(key[8:], doc, self)
else:
return self.get(key)
def view(self, view, **options):
params = deepcopy(options)
limit = None
if 'limit' in params:
limit = params['limit']
del params['limit']
if view.startswith("_design/"):
view_s = view.split('/')
view_doc = view_s[1]
view_map = view_s[3]
else:
view_doc = view
view_map = None
rest = self.server._rest()
results = rest.view_results(self.name, view_doc, view_map,
params, limit)
if 'rows' in results:
return results['rows']
else:
return None
def design_docs(self):
"""List all design documents and return DesignDoc objects for each"""
(ip, port, _, _) = self.server._rest_info()
api = ''.join(['http://{0}:{1}'.format(ip, port),
self.info.ddocs['uri']])
r = requests.get(api, auth=(self.server.rest_username,
self.server.rest_password))
ddocs = []
for ddoc in r.json().get('rows'):
ddocs.append(DesignDoc(ddoc['doc']['meta']['id'],
ddoc['doc']['json'], bucket=self))
return ddocs
def flush(self):
"""RESTful Bucket flushing - will destory all the data in a bucket."""
ip, port, rest_username, rest_password = self.server._rest_info()
api = ''.join(["http://{0}:{1}".format(ip, port),
self.info.controllers['flush']])
response = requests.post(api, auth=(rest_username, rest_password))
if response.status_code is 503:
raise Exception("Only buckets of type 'memcached' support flush.")
elif response.status_code is 404:
raise BucketUnavailableException
elif response.status_code is 200:
return True
data["version"] = "2.0.0"
if re.search(r'\d\.\d\.\d-\d*-\w\w\w', name):
data["fullversion"] = re.search(r'\d\.\d\.\d-\d*-\w\w\w', name).group()
data["url"] = "http://builds.hq.northscale.net/latestbuilds/%s" % (data["name"])
if "arch" not in data:
data["arch"] = "unknown"
kv_data[data["name"]] = data
result = {}
for item in kv_data:
if "package" not in kv_data[item] or "fullversion" not in kv_data[item]:
continue
result[item] = kv_data[item]
with open('/tmp/jsonified.json', 'wb') as fp:
json.dump(result, fp)
c = CouchbaseClient("http://localhost:8091/pools/default", "default", "")
#id will be product-architechture-package-fullversion
print len(result)
for item in result:
_id = "%s-%s-%s-%s" % (result[item]["package"], result[item]["arch"],
result[item]["os"], result[item]["fullversion"])
c.set(_id.encode("ascii", "ignore"), 0, 0, json.dumps(result[item]).encode("ascii", "ignore"))
all_names = {"list":[]}
for item in result:
all_names["list"].append(item.encode("ascii", "ignore"))
c.set("all-builds", 0, 0, json.dumps(all_names).encode("ascii", "ignore"))
def setUp(self):
super(CouchbaseClientTest, self).setUp()
self.client = CouchbaseClient(self.url, self.bucket_name, "", True)
class Bucket(object):
"""Handles Bucket management as well as key/value access for a specific
bucket."""
def __init__(self, name, server):
self.server = server
self.name = name
rest = server._rest()
self.info = rest.get_bucket(self.name)
self.password = self.info.saslPassword
ip, port, rest_username, rest_password = server._rest_info()
formatter_uri = "http://%s:%s/pools/default"
self.mc_client = CouchbaseClient(formatter_uri % (ip, port), self.name,
self.password)
def __del__(self):
self.mc_client.done()
def append(self, key, value, cas=0):
return self.mc_client.append(key, value, cas)
def prepend(self, key, value, cas=0):
return self.mc_client.prepend(key, value, cas)
def incr(self, key, amt=1, init=0, exp=0):
return self.mc_client.incr(key, amt, init, exp)
def decr(self, key, amt=1, init=0, exp=0):
return self.mc_client.decr(key, amt, init, exp)
def set(self, key, expiration, flags, value):
if isinstance(value, dict):
value = json.dumps(value)
return self.mc_client.set(key, expiration, flags, value)
def add(self, key, exp, flags, val):
return self.mc_client.add(key, exp, flags, val)
def replace(self, key, exp, flags, val):
return self.mc_client.replace(key, exp, flags, val)
def get(self, key):
return self.mc_client.get(key)
def getl(self, key, exp=15):
return self.mc_client.getl(key, exp)
def cas(self, key, exp, flags, oldVal, val):
return self.mc_client.cas(key, exp, flags, oldVal, val)
def touch(self, key, exp):
return self.mc_client.touch(key, exp)
def gat(self, key, exp):
return self.mc_client.gat(key, exp)
def stats(self, sub=''):
return self.mc_client.stats(sub)
def delete(self, key, cas=0):
if key.startswith('_design/'):
# this is a design doc, we need to handle it differently
view = key.split('/')[1]
rest = self.server._rest()
rest.delete_view(self.name, view)
else:
return self.mc_client.delete(key, cas)
def save(self, document):
warnings.warn("save is deprecated; use set instead",
DeprecationWarning)
value = deepcopy(document)
if '_id' in value:
key = value['_id']
del value['_id']
else:
key = str(uuid.uuid4())
if '$flags' in value:
flags = value['$flags']
del value['$flags']
else:
flags = 0
if '$expiration' in value:
expiration = value['$expiration']
del value['$expiration']
else:
expiration = 0
if key.startswith('_design/'):
self[key] = value
else:
if '_rev' in value:
# couchbase works in clobber mode so for "set" _rev is useless
del value['_rev']
self.set(key, expiration, flags, json.dumps(value))
return key
def __setitem__(self, key, value):
if isinstance(value, dict):
if 'expiration' in value or 'flags' in value:
assert 'value' in value
if isinstance(value['value'], dict):
v = json.dumps(value['value'])
else:
v = value['value']
self.set(key, value.get('expiration', 0),
value.get('flags', 0), v)
elif key.startswith('_design/'):
rest = self.server._rest()
rest.create_design_doc(self.name, key[8:], json.dumps(value))
else:
self.set(key, 0, 0, json.dumps(value))
else:
self.set(key, 0, 0, value)
def __getitem__(self, key):
if key.startswith("_design/"):
rest = self.server._rest()
doc = rest.get_design_doc(self.name, key[8:])
return DesignDoc(key[8:], doc, self)
else:
return self.get(key)
def view(self, view, **options):
params = deepcopy(options)
limit = None
if 'limit' in params:
limit = params['limit']
del params['limit']
if view.startswith("_design/"):
view_s = view.split('/')
view_doc = view_s[1]
view_map = view_s[3]
else:
view_doc = view
view_map = None
rest = self.server._rest()
results = rest.view_results(self.name, view_doc, view_map, params,
limit)
if 'rows' in results:
return results['rows']
else:
return None
def design_docs(self):
"""List all design documents and return DesignDoc objects for each"""
(ip, port, _, _) = self.server._rest_info()
api = ''.join(
['http://{0}:{1}'.format(ip, port), self.info.ddocs['uri']])
r = requests.get(api,
auth=(self.server.rest_username,
self.server.rest_password))
ddocs = []
for ddoc in r.json().get('rows'):
ddocs.append(
DesignDoc(ddoc['doc']['meta']['id'],
ddoc['doc']['json'],
bucket=self))
return ddocs
def flush(self):
"""RESTful Bucket flushing - will destory all the data in a bucket."""
ip, port, rest_username, rest_password = self.server._rest_info()
api = ''.join([
"http://{0}:{1}".format(ip, port), self.info.controllers['flush']
])
response = requests.post(api, auth=(rest_username, rest_password))
if response.status_code is 503:
raise Exception("Only buckets of type 'memcached' support flush.")
elif response.status_code is 404:
raise BucketUnavailableException
elif response.status_code is 200:
return True
def __init__(self):
from couchbase.couchbaseclient import CouchbaseClient
self.client = \
CouchbaseClient('http://{0}:8091/pools/default'.format(HOST),
'default')
if re.search(r'\d\.\d\.\d-\d*-\w\w\w', name):
data["fullversion"] = re.search(r'\d\.\d\.\d-\d*-\w\w\w',
name).group()
data["url"] = "http://builds.hq.northscale.net/latestbuilds/%s" % (
data["name"])
if "arch" not in data:
data["arch"] = "unknown"
kv_data[data["name"]] = data
result = {}
for item in kv_data:
if "package" not in kv_data[item] or "fullversion" not in kv_data[item]:
continue
result[item] = kv_data[item]
with open('/tmp/jsonified.json', 'wb') as fp:
json.dump(result, fp)
c = CouchbaseClient("http://localhost:8091/pools/default", "default", "")
#id will be product-architechture-package-fullversion
print len(result)
for item in result:
_id = "%s-%s-%s-%s" % (result[item]["package"], result[item]["arch"],
result[item]["os"], result[item]["fullversion"])
c.set(_id.encode("ascii", "ignore"), 0, 0,
json.dumps(result[item]).encode("ascii", "ignore"))
all_names = {"list": []}
for item in result:
all_names["list"].append(item.encode("ascii", "ignore"))
c.set("all-builds", 0, 0, json.dumps(all_names).encode("ascii", "ignore"))
def setUp(self):
super(CouchbaseClientTest, self).setUp()
self.client = CouchbaseClient(self.url, self.bucket_name, "", True)
