def test_key_query(self):
q = Query()
q.mapkey_single = ["abbaye_de_maredsous"]
ret = self.cb.query("beer", "brewery_beers", query=q)
rows = list(ret)
self.assertEqual(len(rows), 1)
q.mapkey_single = UNSPEC
q.mapkey_multi = [["abbaye_de_maredsous"], ["abbaye_de_maredsous", "abbaye_de_maredsous-8"]]
ret = self.cb.query("beer", "brewery_beers", query=q)
rows = list(ret)
self.assertEqual(len(rows), 2)
def test_key_query(self):
q = Query()
q.mapkey_single = ["abbaye_de_maredsous"]
ret = self.cb.query("beer", "brewery_beers", query=q)
rows = list(ret)
self.assertEqual(len(rows), 1)
q.mapkey_single = UNSPEC
q.mapkey_multi = [["abbaye_de_maredsous"],
["abbaye_de_maredsous", "abbaye_de_maredsous-8"]]
ret = self.cb.query("beer", "brewery_beers", query=q)
rows = list(ret)
self.assertEqual(len(rows), 2)
def delete_dataset(dataset, bucket, ratio):
q = Query()
q.mapkey_single = dataset
v = View(bucket, "views", "keys", query=q)
keys = (x.value for x in v if x.key == dataset)
def pack_in_groups(keys, n, ratio):
group = []
for k in keys:
if ratio > random.random():
#print k
if len(group) >= n:
#print(group[0])
yield group
group = []
group.append(k)
if len(group) >= 0:
yield group
nremoved = 0
for kg in pack_in_groups(keys, 500, ratio):
try:
if kg:
bucket.remove_multi(kg, quiet=True)
except NotFoundError as error:
print(error)
else:
nremoved += len(kg)
if nremoved and nremoved % 10000 == 0:
print(nremoved)
return nremoved
def show_customer(customer):
doc = db.get(customer, quiet=True)
if not doc.success:
return "No such customer {0}".format(customer), 404
obj = CustomerRow(name=doc.value['name'], value=None, id=customer, doc=doc.value)
rp = Node_SnapshotListRowProcessor()
q = Query()
q.mapkey_single = customer
q.limit = ENTRIES_PER_PAGE
log_rows = db.query("node_snapshot", "by_customer",
row_processor=rp,
query=q,
include_docs=True)
logs = []
for log in log_rows:
logs.append({'id' : log.id,
'name' : log.name})
rp = RowProcessor(rowclass=CustomerRow)
cluster_rows = db.query("cluster", "by_customer",
row_processor=rp,
query=q,
include_docs=True)
clusters = []
for cluster in cluster_rows:
clusters.append({'id' : cluster.id,
'name' : cluster.name})
return render_template('/customer/show.html', customer=obj, logs=logs, clusters=clusters)
def listDataset(argv):
from couchbase.views.iterator import View
from couchbase.views.params import Query
from striped.client import CouchBaseBackend
from couchbase.exceptions import KeyExistsError, TemporaryFailError, TimeoutError, NotFoundError
Usage = """
python listDataset.py -c <CouchBase config file> [-f|-l] <bucket name> <dataset name>
"""
config_file = None
opts, args = getopt.getopt(argv, "c:lfn")
opts = dict(opts)
config_file = opts.get("-c")
files_only = "-f" in opts
long_print = "-l" in opts
counter = "-n" in opts
if len(sys.argv) < 2:
print(Usage)
sys.exit(1)
bucket_name, dataset_name = args
backend = CouchBaseBackend(bucket_name, config=config_file)
bucket = backend.bucket
if False:
q = Query()
q.mapkey_single = dataset_name
v = View(bucket, "views", "RGInfos", query=q)
infos = [x.value for x in v if x.key == dataset_name]
infos = backend.RGInfos(dataset_name)
infos = sorted(infos, key = lambda info: info["RGID"])
if long_print:
print("RGID NEvents File(s)")
print("------- ---------- -------")
nevents = 0
files = {}
rgids = set()
for info in infos:
fn = info["Segments"][0]["FileName"]
print("%7d %10d %s" % (info["RGID"], info["NEvents"], fn))
rgids.add(info["RGID"])
files[fn] = 1
for s in info["Segments"][1:]:
print("%19s %s" % (" ", s["FileName"]))
files[s["FileName"]] = 1
nevents += info["NEvents"]
print("------- ---------- -------")
print("%7d %10d %d" % (len(infos), nevents, len(files)))
maxrgid = max(rgids)
if len(rgids) != maxrgid+1:
print("Missing RGIDs (%d):" % (maxrgid+1 - len(rgids),))
for rgid in range(maxrgid):
if not rgid in rgids:
print(rgid, end=' ')
print()
elif files_only:
files = {} # filename -> nevents
for info in infos:
for s in info["Segments"]:
fn = s["FileName"]
files[fn] = files.get(fn, 0) + s["NEvents"]
for fn in sorted(files.keys()):
print(fn)
else:
files = set()
rgids = set()
nevents = 0
try:
counter = backend.counter("%s:@@nextRGID" % (dataset_name,), delta=0).value
except NotFoundError:
counter = None
for info in infos:
rgids.add(info["RGID"])
for s in info["Segments"]:
files.add(s["FileName"])
nevents += info["NEvents"]
print("Next FrameID: ", counter)
print("Files: ", len(files))
print("Frames: ", len(rgids))
print("Events: ", nevents)
if len(rgids):
print("Max farme id: ", max(rgids))
print("Events/frame: ", int(float(nevents)/float(len(rgids))+0.5))
maxrgid = max(rgids)
if len(rgids) < maxrgid + 1:
print("Missing RGIDs (%d):" % (maxrgid+1 - len(rgids),))
for rgid in range(maxrgid):
if not rgid in rgids:
print(rgid, end=' ')
print()
