def test_key_query(self): q = Query() q.mapkey_single = ["abbaye_de_maredsous"] ret = self.cb.query("beer", "brewery_beers", query=q) rows = list(ret) self.assertEqual(len(rows), 1) q.mapkey_single = UNSPEC q.mapkey_multi = [["abbaye_de_maredsous"], ["abbaye_de_maredsous", "abbaye_de_maredsous-8"]] ret = self.cb.query("beer", "brewery_beers", query=q) rows = list(ret) self.assertEqual(len(rows), 2)
def test_key_query(self): q = Query() q.mapkey_single = ["abbaye_de_maredsous"] ret = self.cb.query("beer", "brewery_beers", query=q) rows = list(ret) self.assertEqual(len(rows), 1) q.mapkey_single = UNSPEC q.mapkey_multi = [["abbaye_de_maredsous"], ["abbaye_de_maredsous", "abbaye_de_maredsous-8"]] ret = self.cb.query("beer", "brewery_beers", query=q) rows = list(ret) self.assertEqual(len(rows), 2)
def delete_dataset(dataset, bucket, ratio): q = Query() q.mapkey_single = dataset v = View(bucket, "views", "keys", query=q) keys = (x.value for x in v if x.key == dataset) def pack_in_groups(keys, n, ratio): group = [] for k in keys: if ratio > random.random(): #print k if len(group) >= n: #print(group[0]) yield group group = [] group.append(k) if len(group) >= 0: yield group nremoved = 0 for kg in pack_in_groups(keys, 500, ratio): try: if kg: bucket.remove_multi(kg, quiet=True) except NotFoundError as error: print(error) else: nremoved += len(kg) if nremoved and nremoved % 10000 == 0: print(nremoved) return nremoved
def show_customer(customer): doc = db.get(customer, quiet=True) if not doc.success: return "No such customer {0}".format(customer), 404 obj = CustomerRow(name=doc.value['name'], value=None, id=customer, doc=doc.value) rp = Node_SnapshotListRowProcessor() q = Query() q.mapkey_single = customer q.limit = ENTRIES_PER_PAGE log_rows = db.query("node_snapshot", "by_customer", row_processor=rp, query=q, include_docs=True) logs = [] for log in log_rows: logs.append({'id' : log.id, 'name' : log.name}) rp = RowProcessor(rowclass=CustomerRow) cluster_rows = db.query("cluster", "by_customer", row_processor=rp, query=q, include_docs=True) clusters = [] for cluster in cluster_rows: clusters.append({'id' : cluster.id, 'name' : cluster.name}) return render_template('/customer/show.html', customer=obj, logs=logs, clusters=clusters)
def listDataset(argv): from couchbase.views.iterator import View from couchbase.views.params import Query from striped.client import CouchBaseBackend from couchbase.exceptions import KeyExistsError, TemporaryFailError, TimeoutError, NotFoundError Usage = """ python listDataset.py -c <CouchBase config file> [-f|-l] <bucket name> <dataset name> """ config_file = None opts, args = getopt.getopt(argv, "c:lfn") opts = dict(opts) config_file = opts.get("-c") files_only = "-f" in opts long_print = "-l" in opts counter = "-n" in opts if len(sys.argv) < 2: print(Usage) sys.exit(1) bucket_name, dataset_name = args backend = CouchBaseBackend(bucket_name, config=config_file) bucket = backend.bucket if False: q = Query() q.mapkey_single = dataset_name v = View(bucket, "views", "RGInfos", query=q) infos = [x.value for x in v if x.key == dataset_name] infos = backend.RGInfos(dataset_name) infos = sorted(infos, key = lambda info: info["RGID"]) if long_print: print("RGID NEvents File(s)") print("------- ---------- -------") nevents = 0 files = {} rgids = set() for info in infos: fn = info["Segments"][0]["FileName"] print("%7d %10d %s" % (info["RGID"], info["NEvents"], fn)) rgids.add(info["RGID"]) files[fn] = 1 for s in info["Segments"][1:]: print("%19s %s" % (" ", s["FileName"])) files[s["FileName"]] = 1 nevents += info["NEvents"] print("------- ---------- -------") print("%7d %10d %d" % (len(infos), nevents, len(files))) maxrgid = max(rgids) if len(rgids) != maxrgid+1: print("Missing RGIDs (%d):" % (maxrgid+1 - len(rgids),)) for rgid in range(maxrgid): if not rgid in rgids: print(rgid, end=' ') print() elif files_only: files = {} # filename -> nevents for info in infos: for s in info["Segments"]: fn = s["FileName"] files[fn] = files.get(fn, 0) + s["NEvents"] for fn in sorted(files.keys()): print(fn) else: files = set() rgids = set() nevents = 0 try: counter = backend.counter("%s:@@nextRGID" % (dataset_name,), delta=0).value except NotFoundError: counter = None for info in infos: rgids.add(info["RGID"]) for s in info["Segments"]: files.add(s["FileName"]) nevents += info["NEvents"] print("Next FrameID: ", counter) print("Files: ", len(files)) print("Frames: ", len(rgids)) print("Events: ", nevents) if len(rgids): print("Max farme id: ", max(rgids)) print("Events/frame: ", int(float(nevents)/float(len(rgids))+0.5)) maxrgid = max(rgids) if len(rgids) < maxrgid + 1: print("Missing RGIDs (%d):" % (maxrgid+1 - len(rgids),)) for rgid in range(maxrgid): if not rgid in rgids: print(rgid, end=' ') print()