def lookupRange(lookupInformation: LookupInformation, rng: RangeLookup, output) -> None: index_table_ops = lookupInformation.getTableOps() indexScanner = index_table_ops.createScanner(lookupInformation.getAuths(), 1) fv = rng.getValue() if fv.endswith("*"): base = fv.replace('*', '') indexrange = pysharkbite.Range(base, True, base + "\uffff", False) else: indexrange = pysharkbite.Range(rng.getValue().lower()) indexScanner.addRange(indexrange) if not rng.getField() is None: indexScanner.fetchColumn(rng.getField().upper(), "") indexSet = indexScanner.getResultSet() try: for indexKeyValue in indexSet: value = indexKeyValue.getValue() protobuf = Uid_pb2.List() protobuf.ParseFromString(value.get().encode()) for uidvalue in protobuf.UID: shard = indexKeyValue.getKey().getColumnQualifier().split( "\u0000")[0] datatype = indexKeyValue.getKey().getColumnQualifier().split( "\u0000")[1] output.put(Range(datatype, shard, uidvalue)) except: pass indexScanner.close()
def __iter__(self): if self._indexLookupInformation is None: return self indexTableOps = lookupInformation.getTableOps() self._rngs = [None] * len(rangeQueue) self._scnrs = [None] * len(rangeQueue) self._iters = [None] * len(rangeQueue) count=0 for rng in rangeQueue: if isinstance(rng,LookupIterator): self._iters[count]=rng self._scnrs=None else: scnrs[count] = indexTableOps.createScanner(lookupInformation.getAuths(),1) indexrange = pysharkbite.Range(rng.getValue()) scnrs[count].addRange(indexrange) if not rng.getField() is None: scnrs[count].fetchColumn(rng.getField().upper(),"") self._iters=ForwardIterator(scnrs[count].getResultSet()) count=count+1 self._iter=combineScanners(self._iters) return self
def get(self, request, *args, **kwargs): shard = request.GET.get('shard') datatype = request.GET.get('datatype') uid = request.GET.get('uid') query = request.GET.get('query') authstring = request.GET.get('auths') url = "/search/?q=" + query auths = pysharkbite.Authorizations() if not authstring is None and len(authstring) > 0: auths.addAuthorization(authstring) user = pysharkbite.AuthInfo(AccumuloCluster.objects.first().user,AccumuloCluster.objects.first().password, ZkInstance().get().getInstanceId()) connector = pysharkbite.AccumuloConnector(user, ZkInstance().get()) tableOps = connector.tableOps(AccumuloCluster.objects.first().dataTable) scanner = tableOps.createScanner(auths,1) startKey = pysharkbite.Key(row=shard) endKey = pysharkbite.Key(row=shard) docid = datatype + "\x00" + uid; startKey.setColumnFamily(docid) endKey.setColumnFamily(docid + "\xff") rng = pysharkbite.Range(startKey,True,endKey,True) scanner.addRange(rng) writer = tableOps.createWriter(auths,10) deletes = pysharkbite.Mutation(shard) for keyValue in scanner.getResultSet(): key = keyValue.getKey() deletes.putDelete( key.getColumnFamily(), key.getColumnQualifier(), key.getColumnVisibility(), key.getTimestamp()) ## scan for the original document writer.addMutation(deletes) writer.close() for auth in authstring.split("|"): url = url + "&auths=" + auth return HttpResponseRedirect(url)
def populateMetadata(): import pysharkbite conf = pysharkbite.Configuration() conf.set("FILE_SYSTEM_ROOT", "/accumulo") model = apps.get_model(app_label='query', model_name='AccumuloCluster') accumulo_cluster = model.objects.first() if accumulo_cluster is None: return zk = pysharkbite.ZookeeperInstance(accumulo_cluster.instance, accumulo_cluster.zookeeper, 1000, conf) user = pysharkbite.AuthInfo("root", "secret", zk.getInstanceId()) connector = pysharkbite.AccumuloConnector(user, zk) indexTableOps = connector.tableOps("DatawaveMetadata") auths = pysharkbite.Authorizations() indexScanner = indexTableOps.createScanner(auths, 100) indexrange = pysharkbite.Range() indexScanner.addRange(indexrange) indexScanner.fetchColumn("f", "") combinertxt = "" ## load the combiner from the file system and send it to accumulo with open('countgatherer.py', 'r') as file: combinertxt = file.read() combiner = pysharkbite.PythonIterator("MetadataCounter", combinertxt, 200) indexScanner.addIterator(combiner) indexSet = indexScanner.getResultSet() import json counts = 0 mapping = {} for indexKeyValue in indexSet: value = indexKeyValue.getValue() key = indexKeyValue.getKey() if key.getColumnFamily() == "f": day = key.getColumnQualifier().split("\u0000")[1] dt = key.getColumnQualifier().split("\u0000")[0] if day in mapping: if key.getRow() in mapping[day]: try: mapping[day][key.getRow()] += int(value.get()) except: pass else: try: mapping[day][key.getRow()] = int(value.get()) except: pass else: mapping[day] = {} try: mapping[day][key.getRow()] = int(value.get()) except: pass caches['metadata'].set("field", json.dumps(mapping), 3600 * 48) return json.dumps(mapping)
def mthd(self): import pysharkbite tableOperations = super().getTableOperations() if not tableOperations.exists(False): print("Creating table") if not tableOperations.create(False): print("Could not create table") else: tableOperations.remove() time.sleep(2) tableOperations.create(False) print("Table already exists, so not creating it") tableOperations = super().newtableOperations() splits = {"row", "row5", "row9", "sow"} tableOperations.addSplits(splits) auths = pysharkbite.Authorizations() """ Add authorizations """ """ mutation.put("cf","cq","cv",1569786960) """ writer = tableOperations.createWriter(auths, 10) rng = range(0, 500) for i in rng: row = ("row%i" % (i + 5)) mutation = pysharkbite.Mutation(row) rng2 = range(0, 100) for j in rng: mutation.put("cf%i" % j, "cq", "", 1569786960, "value") writer.addMutation(mutation) print("wrote %i entries" % ((i + 1) * (j + 1))) writer.close() tableOperations.compact("a", "s", True) print("written") scanner = tableOperations.createScanner(auths, 2) time.sleep(1) accumuloRange = pysharkbite.Range("a", True, "sow", False) scanner.addRange(accumuloRange) resultset = scanner.getResultSet() count = 0 for keyvalue in resultset: key = keyvalue.getKey() count = count + 1 print("count is ", count) """ delete your table if user did not create temp """
def lookupRanges(lookupInformation: LookupInformation, ranges: list, output) -> None: index_table_ops = lookupInformation.getTableOps() rngs = [None] * len(ranges) scnrs = [None] * len(ranges) itrs = [None] * len(ranges) count = 0 for rng in ranges: if isinstance(rng, LookupIterator): itrs[count] = ForwardIterator(rng) scnrs[count] = None count = count + 1 else: scnrs[count] = index_table_ops.createScanner( lookupInformation.getAuths(), 1) fv = rng.getValue() if fv.endswith("*"): base = fv.replace('*', '') indexrange = pysharkbite.Range(base, True, base + "\uffff", False) else: indexrange = pysharkbite.Range(fv.lower()) scnrs[count].addRange(indexrange) if not rng.getField() is None: scnrs[count].fetchColumn(rng.getField().upper(), "") itrs[count] = scnrs[count].getResultSet() count = count + 1 try: for indexKeyValue in intersect_sets(itrs): output.put(indexKeyValue) except StopIteration: pass except: traceback.print_exc() raise for scnr in scnrs: if not scnr is None: scnr.close()
def get_uploads(): model = apps.get_model(app_label='query', model_name='FileUpload') objs = model.objects.filter(status="NEW") haveNew = False for obj in objs: if obj.status == "NEW": caches['eventcount'].set("ingestcomplete", 95) haveNew = True if not haveNew: caches['eventcount'].set("ingestcomplete", 100) return import pysharkbite conf = pysharkbite.Configuration() conf.set("FILE_SYSTEM_ROOT", "/accumulo") model = apps.get_model(app_label='query', model_name='AccumuloCluster') accumulo_cluster = model.objects.first() if accumulo_cluster is None: return zk = pysharkbite.ZookeeperInstance(accumulo_cluster.instance, accumulo_cluster.zookeeper, 1000, conf) user = pysharkbite.AuthInfo("root", "secret", zk.getInstanceId()) connector = pysharkbite.AccumuloConnector(user, zk) auths = pysharkbite.Authorizations() auths.addAuthorization("PROV") indexTableOps = connector.tableOps("provenanceIndex") indexScanner = indexTableOps.createScanner(auths, 10) indexrange = pysharkbite.Range() indexScanner.addRange(indexrange) indexScanner.fetchColumn("CONTENTURI", "") indexScanner.fetchColumn("TRANSITURI", "") indexSet = indexScanner.getResultSet() count = 0 usercount = 0 try: for indexKeyValue in indexSet: if indexKeyValue.getKey().getColumnFamily() == "CONTENTURI": count = count + 1 else: usercount = usercount + 1 if count > 0: caches['eventcount'].set("ingestcount", count, 3600 * 48) if usercount > 0: caches['eventcount'].set("useruploads", usercount, 3600 * 48) indexScanner.close() except: pass ## user does not have PROV
def mthd(self): import pysharkbite pysharkbite.LoggingConfiguration.enableTraceLogger() self.writerfile() readrfile = pysharkbite.RFileOperations.sequentialRead("blah.rf") accrange = pysharkbite.Range() seekable = pysharkbite.Seekable(accrange) readrfile.seek( seekable) i = 0 while readrfile.hasNext(): assert( str("row" + str(i)) == readrfile.getTopKey().getRow() ) i=i+1 readrfile.next() assert(i==1000)
def check(): model = apps.get_model(app_label='query', model_name='FileUpload') objs = model.objects.filter(status="NEW") for obj in objs: if obj.status == "NEW": import pysharkbite conf = pysharkbite.Configuration() conf.set("FILE_SYSTEM_ROOT", "/accumulo") model = apps.get_model(app_label='query', model_name='AccumuloCluster') accumulo_cluster = model.objects.first() print("Checking " + str(obj.uuid)) if accumulo_cluster is None: return print("Checking " + str(obj.uuid)) zk = pysharkbite.ZookeeperInstance(accumulo_cluster.instance, accumulo_cluster.zookeeper, 1000, conf) user = pysharkbite.AuthInfo("root", "secret", zk.getInstanceId()) connector = pysharkbite.AccumuloConnector(user, zk) indexTableOps = connector.tableOps("provenanceIndex") auths = pysharkbite.Authorizations() auths.addAuthorization("PROV") indexScanner = indexTableOps.createScanner(auths, 2) indexrange = pysharkbite.Range(str(obj.uuid)) indexScanner.addRange(indexrange) indexSet = indexScanner.getResultSet() rangelist = list() provops = connector.tableOps("provenance") scanner = provops.createScanner(auths, 10) for indexKeyValue in indexSet: value = indexKeyValue.getValue() protobuf = Uid_pb2.List() protobuf.ParseFromString(value.get().encode()) for uidvalue in protobuf.UID: shard = indexKeyValue.getKey().getColumnQualifier().split( "\u0000")[0] datatype = indexKeyValue.getKey().getColumnQualifier( ).split("\u0000")[1] startKey = pysharkbite.Key() stopKey = pysharkbite.Key() startKey.setRow(shard) stopKey.setRow(shard) startKey.setColumnFamily(datatype + "\x00" + uidvalue) stopKey.setColumnFamily(datatype + "\x00" + uidvalue + "\xff") rangelist.append( pysharkbite.Range(startKey, True, stopKey, False)) scanner = provops.createScanner(auths, 10) scanner.addRange( pysharkbite.Range(startKey, True, stopKey, False)) resultset = scanner.getResultSet() for keyvalue in resultset: key = keyvalue.getKey() value = keyvalue.getValue() eventid = key.getColumnFamily().split("\u0000")[1] fieldname = key.getColumnQualifier().split("\u0000")[0] fieldvalue = key.getColumnQualifier().split( "\u0000")[1] if (fieldname == "EVENTTYPE"): if fieldvalue == "DROP": obj.status = "COMPLETE" obj.save() break scanner.close() indexScanner.close()
def mthd(self): import pysharkbite securityOps = super().getSecurityOperations() auths = pysharkbite.Authorizations() auths.addAuthorization("blah1") auths.addAuthorization("blah2") securityOps.grantAuthorizations(auths, "root") tableOperations = super().getTableOperations() tableOperations.create(False) ## validate that we DO see the permissions """ Add authorizations """ """ mutation.put("cf","cq","cv",1569786960) """ with tableOperations.createWriter(auths, 10) as writer: mutation = pysharkbite.Mutation("row2") mutation.put("cf", "cq", "blah1", 1569786960, "value") mutation.put("cf2", "cq2", "blah1", 1569786960, "value2") """ no value """ mutation.put("cf3", "cq3", "blah2", 1569786960, "") writer.addMutation(mutation) mutation = pysharkbite.Mutation("row4") mutation.put("cf", "cq", "blah1", 1569786960, "value") mutation.put("cf2", "cq2", "blah1", 1569786960, "value2") """ no value """ mutation.put("cf3", "cq3", "blah2", 1569786960, "") writer.addMutation(mutation) auths = pysharkbite.Authorizations() auths.addAuthorization("blah1") auths.addAuthorization("blah2") count = 0 with tableOperations.createScanner(auths, 2).withRange( pysharkbite.Range("row", True, "row3", False)) as scanner: resultset = scanner.getResultSet() for keyvalue in resultset: count = count + 1 key = keyvalue.getKey() assert ("row2" == key.getRow()) value = keyvalue.getValue() if "cf" == key.getColumnFamily(): assert ("value" == value.get()) if ("cf2" == key.getColumnFamily()): assert ("value2" == value.get()) with tableOperations.createScanner(auths, 2).withRange( pysharkbite.Range("row4", True, "row5", False)) as scanner: resultset = scanner.getResultSet() for keyvalue in resultset: count = count + 1 key = keyvalue.getKey() print("%r %r" % (keyvalue.getKey(), str(keyvalue.getValue()))) assert ("row4" == key.getRow()) value = keyvalue.getValue() if "cf" == key.getColumnFamily(): assert ("value" == value.get()) if ("cf2" == key.getColumnFamily()): assert ("value2" == value.get()) """ delete your table if user did not create temp """ tableOperations.remove() print("count is " + str(count)) assert (count == 6)
def mthd(self): import pysharkbite tableOperations = super().getTableOperations() if not tableOperations.exists(False): print("Creating table") if not tableOperations.create(False): print("Could not create table") else: print("Table already exists, so not creating it") auths = pysharkbite.Authorizations() """ Add authorizations """ """ mutation.put("cf","cq","cv",1569786960) """ writer = tableOperations.createWriter(auths, 10) mutation = pysharkbite.Mutation("sow2") mutation.put("cf", "cq", "", 1569786960, "value") mutation.put("cf2", "cq2", "", 1569786960, "value2") """ no value """ mutation.put("cf3", "cq3", "", 1569786960, "") writer.addMutation(mutation) writer.close() writer = tableOperations.createWriter(auths, 10) rng = range(0, 1000) for i in rng: row = ("row%i" % (i + 5)) mutation = pysharkbite.Mutation(row) mutation.put("cf", "cq", "", 1569786960, "value") writer.addMutation(mutation) writer.close() print("written") """ auths.addAuthorization("cv") """ scanner = tableOperations.createScanner(auths, 2) startKey = pysharkbite.Key() endKey = pysharkbite.Key() startKey.setRow("sow") endKey.setRow("sow3") accumuloRange = pysharkbite.Range(startKey, True, endKey, False) scanner.addRange(accumuloRange) resultset = scanner.getResultSet() for keyvalue in resultset: key = keyvalue.getKey() assert ("sow2" == key.getRow()) value = keyvalue.getValue() if "cf" == key.getColumnFamily(): assert ("value" == value.get()) if ("cf2" == key.getColumnFamily()): assert ("value2" == value.get()) if ("cf3" == key.getColumnFamily()): assert ("" == value.get()) scanner = tableOperations.createScanner(auths, 2) accumuloRange = pysharkbite.Range("row", True, "", False) scanner.addRange(accumuloRange) resultset = scanner.getResultSet() count = 0 for keyvalue in resultset: key = keyvalue.getKey() count = count + 1 print("count is ", count) assert (count == 1003) # all rows + sows """ delete your table if user did not create temp """ tableOperations.remove()
args = parser.parse_args() file = args.file import pysharkbite try: rfile = pysharkbite.RFileOperations.open(file) # list of column families cf = list() ## range rng = pysharkbite.Range() ## seekable is an object that replaces the function call ## of seek(range,cf,boolean). This exists because the extension point ## in the C++ Api allows varying seekable objects that change how seeks occur seek = pysharkbite.Seekable(rng, cf, False) rfile.seek(seek) ## print the row while rfile.hasNext(): kv = rfile.getTop() print(kv.getKey()) rfile.next() except RuntimeError as e:
def getDocuments(cancellationtoken: CancellationToken, name: int, lookupInformation: LookupInformation, input: queue.SimpleQueue, outputQueue: queue.SimpleQueue): count = 0 while cancellationtoken.running(): docInfo = None try: try: if input.empty(): pass else: docInfo = input.get(timeout=1) except: pass # Handle empty queue here if not docInfo is None: tableOps = lookupInformation.getTableOps() scanner = tableOps.createScanner(lookupInformation.getAuths(), 5) startKey = pysharkbite.Key() endKey = pysharkbite.Key() startKey.setRow(docInfo.getShard()) docid = docInfo.getDataType() + "\x00" + docInfo.getDocId() startKey.setColumnFamily(docid) endKey.setRow(docInfo.getShard()) endKey.setColumnFamily(docid + "\xff") print("Searching for " + docInfo.getShard()) rng = pysharkbite.Range(startKey, True, endKey, True) scanner.addRange(rng) rangecount = 1 while rangecount < 10: try: docInfo = input.get(False) startKey = pysharkbite.Key() endKey = pysharkbite.Key() startKey.setRow(docInfo.getShard()) docid = docInfo.getDataType( ) + "\x00" + docInfo.getDocId() startKey.setColumnFamily(docid) endKey.setRow(docInfo.getShard()) endKey.setColumnFamily(docid + "\xff") rng = pysharkbite.Range(startKey, True, endKey, True) print("Searching for " + docInfo.getShard()) scanner.addRange(rng) rangecount = rangecount + 1 except: rangecount = 11 with open('jsoncombiner.py', 'r') as file: combinertxt = file.read() combiner = pysharkbite.PythonIterator( "PythonCombiner", combinertxt, 100) scanner.addIterator(combiner) try: count = count + scanDoc(scanner, outputQueue) except: pass else: time.sleep(0.5) except: e = sys.exc_info()[0] return True
def pouplateEventCountMetadata(): import pysharkbite import time conf = pysharkbite.Configuration() conf.set("FILE_SYSTEM_ROOT", "/accumulo") model = apps.get_model(app_label='query', model_name='AccumuloCluster') accumulo_cluster = model.objects.first() if accumulo_cluster is None: return zoo_keeper = pysharkbite.ZookeeperInstance(accumulo_cluster.instance, accumulo_cluster.zookeeper, 1000, conf) user = pysharkbite.AuthInfo("root", "secret", zoo_keeper.getInstanceId()) connector = pysharkbite.AccumuloConnector(user, zoo_keeper) queryRanges = list() #last 15 days for dateinrange in getDateRange(-15): shardbegin = dateinrange.strftime("%Y%m%d") if caches['eventcount'].get(shardbegin) is None or caches[ 'eventcount'].get(shardbegin) == 0: queryRanges.append(shardbegin) else: pass # don't add to range if len(queryRanges) > 0: ## all is cached user = pysharkbite.AuthInfo("root", "secret", zoo_keeper.getInstanceId()) connector = pysharkbite.AccumuloConnector(user, zoo_keeper) indexTableOps = connector.tableOps("DatawaveMetrics") auths = pysharkbite.Authorizations() auths.addAuthorization("MTRCS") indexScanner = indexTableOps.createScanner(auths, 100) start = time.time() for dt in queryRanges: indexrange = pysharkbite.Range(dt, True, dt + "\uffff", False) indexScanner.addRange(indexrange) indexScanner.fetchColumn("EVENT_COUNT", "") combinertxt = "" ## load the combiner from the file system and send it to accumulo with open('metricscombiner.py', 'r') as file: combinertxt = file.read() combiner = pysharkbite.PythonIterator("MetadataCounter", combinertxt, 200) indexScanner.addIterator(combiner) indexSet = indexScanner.getResultSet() counts = 0 mapping = {} try: for indexKeyValue in indexSet: value = indexKeyValue.getValue() key = indexKeyValue.getKey() if key.getColumnFamily() == "EVENT_COUNT": dt = key.getRow().split("_")[0] if dt in mapping: mapping[dt] += int(value.get()) else: mapping[dt] = int(value.get()) arr = [None] * len(mapping.keys()) for field in mapping: caches['eventcount'].set(field, str(mapping[field]), 3600 * 48) except: pass
def mthd(self): import pysharkbite tableOperations = super().getTableOperations() if not tableOperations.exists(False): print("Creating table") if not tableOperations.create(False): print("Could not create table") else: print("Table already exists, so not creating it") auths = pysharkbite.Authorizations() """ Add authorizations """ """ mutation.put("cf","cq","cv",1569786960) """ writer = tableOperations.createWriter(auths, 10) mutation = pysharkbite.Mutation("sow2") mutation.put("cf", "cq", "", 1569786960, "value") mutation.put("cf2", "cq", "", 1569786960, "value") """ no value """ mutation.put("cf3", "cq", "", 1569786960, "value") writer.addMutation(mutation) writer.close() writer = tableOperations.createWriter(auths, 10) rng = range(0, 1000) for i in rng: row = ("row%i" % (i + 5)) mutation = pysharkbite.Mutation(row) mutation.put("cf", "cq", "", 1569786960, "value") writer.addMutation(mutation) writer.close() print("written") """ auths.addAuthorization("cv") """ scanner = tableOperations.createScanner(auths, 2) accumuloRange = pysharkbite.Range("sow", True, "sow3", False) scanner.addRange(accumuloRange) iterator = pysharkbite.PythonIterator("PythonIterator", 100) iterator = iterator.onNext( "lambda x : KeyValue( Key( x.getKey().getRow(), 'new cf', x.getKey().getColumnQualifier()), Value()) " ) scanner.addIterator(iterator) resultset = scanner.getResultSet() for keyvalue in resultset: key = keyvalue.getKey() assert ("sow2" == key.getRow()) value = keyvalue.getValue() if "cf" == key.getColumnFamily(): sys.exit(154) if "new cf" == key.getColumnFamily(): assert ("" == value.get()) scanner = tableOperations.createScanner(auths, 2) accumuloRange = pysharkbite.Range("sow", True, "sow3", False) scanner.addRange(accumuloRange) iterator = pysharkbite.PythonIterator("PythonIterator", 100) iterator = iterator.onNext( "lambda x : Key( x.getKey().getRow(), x.getKey().getColumnFamily(), 'new cq') " ) scanner.addIterator(iterator) resultset = scanner.getResultSet() for keyvalue in resultset: key = keyvalue.getKey() assert ("sow2" == key.getRow()) value = keyvalue.getValue() if "cq" == key.getColumnQualifier(): sys.exit(154) """ delete your table if user did not create temp """ tableOperations.remove()
def run_edge_query(query_id): model = apps.get_model(app_label='query', model_name='EdgeQuery') objs = model.objects.filter(query_id=query_id) for obj in objs: obj.running = True obj.save() import pysharkbite conf = pysharkbite.Configuration() conf.set("FILE_SYSTEM_ROOT", "/accumulo") model = apps.get_model(app_label='query', model_name='AccumuloCluster') accumulo_cluster = model.objects.first() if accumulo_cluster is None: return zk = pysharkbite.ZookeeperInstance(accumulo_cluster.instance, accumulo_cluster.zookeeper, 1000, conf) user = pysharkbite.AuthInfo("root", "secret", zk.getInstanceId()) connector = pysharkbite.AccumuloConnector(user, zk) auths = pysharkbite.Authorizations() if obj.auths: for auth in obj.auths.split(","): auths.addAuthorization(auth) sres_model = apps.get_model(app_label='query', model_name='ScanResult') res_model = apps.get_model(app_label='query', model_name='Result') sr = sres_model.objects.filter(query_id=obj.query_id).first() if not sr: print("No scan result, returning") return print("here") graphTableOps = connector.tableOps("graph") scanner = graphTableOps.createScanner(auths, 10) range = pysharkbite.Range(obj.query, True, obj.query + "\uffff" + "\uffff", False) ## for now the range should be this scanner.addRange(range) resultset = scanner.getResultSet() count = 0 try: for indexKeyValue in resultset: value = "0" ## row will be the to ## direction will be the cf to_value = "" direction = "one" try: protobuf = EdgeData_pb2.EdgeValue() protobuf.ParseFromString( indexKeyValue.getValue().get_bytes()) value = str(protobuf.count) + "/" + protobuf.uuid_string to_value = indexKeyValue.getKey().getRow().split( "\u0000")[1] direction = indexKeyValue.getKey().getColumnFamily().split( "/")[1] direction_split = direction.split("-") if len(direction_split ) != 2 or direction_split[0] == direction_split[1]: continue except Exception as e: print(e) continue except: continue scanresult = res_model.objects.create( scanResult=sr, value=value, row=to_value, cf=direction, cq=indexKeyValue.getKey().getColumnQualifier()) scanresult.save() count = count + 1 if count > 1000: break sr.is_finished = True sr.save() scanner.close() except Exception as e: print(e) except: print("An error occurred") pass ## user does not have PROV obj.running = False obj.finished = True obj.save()
def mthd(self): import pysharkbite #pysharkbite.LoggingConfiguration.enableTraceLogger() tableOperations = super().getTableOperations() if not tableOperations.exists(False): print ("Creating table") tableOperations.create(False) else: print ("Table already exists, so not creating it") auths = pysharkbite.Authorizations() auths.addAuthorization("blah1") auths.addAuthorization("blah2") securityOps = super().getConnector().securityOps() securityOps.grantAuthorizations(auths,"root") """ Add authorizations """ """ mutation.put("cf","cq","cv",1569786960) """ writer = tableOperations.createWriter(auths, 10) mutation = pysharkbite.Mutation("row2"); mutation.put("cf","cq","blah1",1569786960, "value") mutation.put("cf2","cq2","blah1",1569786960, "value2") """ no value """ mutation.put("cf3","cq3","blah2",1569786960, "") writer.addMutation( mutation ) writer.close() print("written") auths = pysharkbite.Authorizations() auths.addAuthorization("blah1") scanner = tableOperations.createScanner(auths, 2) startKey = pysharkbite.Key() endKey = pysharkbite.Key() startKey.setRow("row") endKey.setRow("row3") # test single range range = pysharkbite.Range("row2") scanner.addRange( range ) resultset = scanner.getResultSet() for keyvalue in resultset: key = keyvalue.getKey() assert( "row2" == key.getRow() ) value = keyvalue.getValue() if "cf" == key.getColumnFamily(): assert( "value" == value.get() ) if ("cf2" == key.getColumnFamily() ): assert( "value2" == value.get() ) if ("cf3" == key.getColumnFamily() ): print("Unexpected column cf3") sys.exit(1) range = pysharkbite.Range("row1",True,"row1.5",True) scanner.addRange( range ) resultset = scanner.getResultSet() for keyvalue in resultset: print("Unexpected result") sys.exit(1) # test single range range = pysharkbite.Range("row",False,"row3",True) scanner = tableOperations.createScanner(auths, 2) scanner.addRange( range ) resultset = scanner.getResultSet() count =0 for keyvalue in resultset: key = keyvalue.getKey() assert( "row2" == key.getRow() ) value = keyvalue.getValue() if "cf" == key.getColumnFamily(): assert( "value" == value.get() ) if ("cf2" == key.getColumnFamily() ): assert( "value2" == value.get() ) if ("cf3" == key.getColumnFamily() ): print("Unexpected column cf3") sys.exit(1) count=count+1 if count <= 0: print("Expected results") sys.exit(1) # test infinite range range = pysharkbite.Range("",False,"row3",True) scanner = tableOperations.createScanner(auths, 2) scanner.addRange( range ) resultset = scanner.getResultSet() count =0 for keyvalue in resultset: key = keyvalue.getKey() assert( "row2" == key.getRow() ) value = keyvalue.getValue() if "cf" == key.getColumnFamily(): assert( "value" == value.get() ) if ("cf2" == key.getColumnFamily() ): assert( "value2" == value.get() ) if ("cf3" == key.getColumnFamily() ): print("Unexpected column cf3") sys.exit(1) count=count+1 if count <= 0: print("Expected results") sys.exit(1) startKey = pysharkbite.Key("row3") range = pysharkbite.Range(None,False,startKey,True) scanner = tableOperations.createScanner(auths, 2) scanner.addRange( range ) resultset = scanner.getResultSet() count =0 for keyvalue in resultset: key = keyvalue.getKey() assert( "row2" == key.getRow() ) value = keyvalue.getValue() if "cf" == key.getColumnFamily(): assert( "value" == value.get() ) if ("cf2" == key.getColumnFamily() ): assert( "value2" == value.get() ) if ("cf3" == key.getColumnFamily() ): print("Unexpected column cf3") sys.exit(1) count=count+1 if count <= 0: print("Expected results") sys.exit(1) """ delete your table if user did not create temp """ tableOperations.remove()
time.sleep(2) """ auths.addAuthorization("cv") """ scanner = tableOperations.createScanner(auths, 2) startKey = pysharkbite.Key() endKey = pysharkbite.Key() startKey.setRow("row") endKey.setRow("row3") range = pysharkbite.Range(startKey,True,endKey,False) scanner.addRange( range ) resultset = scanner.getResultSet() for keyvalue in resultset: key = keyvalue.getKey() value = keyvalue.getValue() """ delete your table if user did not create temp """ if not args.table: tableOperations.remove() except RuntimeError as e:
auths = pysharkbite.Authorizations() """ Add authorizations """ """ mutation.put("cf","cq","cv",1569786960) """ scanner = tableOperations.createScanner(auths, 2) startKey = pysharkbite.Key() endKey = pysharkbite.Key() startKey.setRow("row") endKey.setRow("row3") range = pysharkbite.Range("a") scanner.addRange(range) iterator = pysharkbite.PythonIterator("PythonIterator", 100) iterator = iterator.onNext( "lambda x : sharkbite_iterator.Key( x.getRow(), 'new cf', x.getColumnQualifier()) " ) scanner.addIterator(iterator) resultset = scanner.getResultSet() for keyvalue in resultset: key = keyvalue.getKey() value = keyvalue.getValue() print(key.getRow() + ":" + key.getColumnFamily() + ":" +