def get(self, request, *args, **kwargs): shard = request.GET.get('shard') datatype = request.GET.get('datatype') uid = request.GET.get('uid') query = request.GET.get('query') authstring = request.GET.get('auths') url = "/search/?q=" + query auths = pysharkbite.Authorizations() if not authstring is None and len(authstring) > 0: auths.addAuthorization(authstring) user = pysharkbite.AuthInfo(AccumuloCluster.objects.first().user,AccumuloCluster.objects.first().password, ZkInstance().get().getInstanceId()) connector = pysharkbite.AccumuloConnector(user, ZkInstance().get()) tableOps = connector.tableOps(AccumuloCluster.objects.first().dataTable) scanner = tableOps.createScanner(auths,1) startKey = pysharkbite.Key(row=shard) endKey = pysharkbite.Key(row=shard) docid = datatype + "\x00" + uid; startKey.setColumnFamily(docid) endKey.setColumnFamily(docid + "\xff") rng = pysharkbite.Range(startKey,True,endKey,True) scanner.addRange(rng) writer = tableOps.createWriter(auths,10) deletes = pysharkbite.Mutation(shard) for keyValue in scanner.getResultSet(): key = keyValue.getKey() deletes.putDelete( key.getColumnFamily(), key.getColumnQualifier(), key.getColumnVisibility(), key.getTimestamp()) ## scan for the original document writer.addMutation(deletes) writer.close() for auth in authstring.split("|"): url = url + "&auths=" + auth return HttpResponseRedirect(url)
def writerfile(self): import pysharkbite rfile = pysharkbite.RFileOperations.openForWrite("blah.rf") for x in range(1000): key = pysharkbite.Key(row=str("row" + str(x))) value = pysharkbite.Value() kv = pysharkbite.KeyValue(key,value) rfile.append(kv) rfile.close()
mutation.put("cf","cq","",1569786960, "value") mutation.put("cf2","cq2","",1569786960, "value2") """ no value """ mutation.put("cf3","cq3","",1569786960, "") writer.addMutation( mutation ) writer.close() time.sleep(2) """ auths.addAuthorization("cv") """ scanner = tableOperations.createScanner(auths, 2) startKey = pysharkbite.Key() endKey = pysharkbite.Key() startKey.setRow("row") endKey.setRow("row3") range = pysharkbite.Range(startKey,True,endKey,False) scanner.addRange( range ) resultset = scanner.getResultSet() for keyvalue in resultset: key = keyvalue.getKey()
def mthd(self): import pysharkbite #pysharkbite.LoggingConfiguration.enableTraceLogger() tableOperations = super().getTableOperations() if not tableOperations.exists(False): print ("Creating table") tableOperations.create(False) else: print ("Table already exists, so not creating it") auths = pysharkbite.Authorizations() auths.addAuthorization("blah1") auths.addAuthorization("blah2") securityOps = super().getConnector().securityOps() securityOps.grantAuthorizations(auths,"root") """ Add authorizations """ """ mutation.put("cf","cq","cv",1569786960) """ writer = tableOperations.createWriter(auths, 10) mutation = pysharkbite.Mutation("row2"); mutation.put("cf","cq","blah1",1569786960, "value") mutation.put("cf2","cq2","blah1",1569786960, "value2") """ no value """ mutation.put("cf3","cq3","blah2",1569786960, "") writer.addMutation( mutation ) writer.close() print("written") auths = pysharkbite.Authorizations() auths.addAuthorization("blah1") scanner = tableOperations.createScanner(auths, 2) startKey = pysharkbite.Key() endKey = pysharkbite.Key() startKey.setRow("row") endKey.setRow("row3") # test single range range = pysharkbite.Range("row2") scanner.addRange( range ) resultset = scanner.getResultSet() for keyvalue in resultset: key = keyvalue.getKey() assert( "row2" == key.getRow() ) value = keyvalue.getValue() if "cf" == key.getColumnFamily(): assert( "value" == value.get() ) if ("cf2" == key.getColumnFamily() ): assert( "value2" == value.get() ) if ("cf3" == key.getColumnFamily() ): print("Unexpected column cf3") sys.exit(1) range = pysharkbite.Range("row1",True,"row1.5",True) scanner.addRange( range ) resultset = scanner.getResultSet() for keyvalue in resultset: print("Unexpected result") sys.exit(1) # test single range range = pysharkbite.Range("row",False,"row3",True) scanner = tableOperations.createScanner(auths, 2) scanner.addRange( range ) resultset = scanner.getResultSet() count =0 for keyvalue in resultset: key = keyvalue.getKey() assert( "row2" == key.getRow() ) value = keyvalue.getValue() if "cf" == key.getColumnFamily(): assert( "value" == value.get() ) if ("cf2" == key.getColumnFamily() ): assert( "value2" == value.get() ) if ("cf3" == key.getColumnFamily() ): print("Unexpected column cf3") sys.exit(1) count=count+1 if count <= 0: print("Expected results") sys.exit(1) # test infinite range range = pysharkbite.Range("",False,"row3",True) scanner = tableOperations.createScanner(auths, 2) scanner.addRange( range ) resultset = scanner.getResultSet() count =0 for keyvalue in resultset: key = keyvalue.getKey() assert( "row2" == key.getRow() ) value = keyvalue.getValue() if "cf" == key.getColumnFamily(): assert( "value" == value.get() ) if ("cf2" == key.getColumnFamily() ): assert( "value2" == value.get() ) if ("cf3" == key.getColumnFamily() ): print("Unexpected column cf3") sys.exit(1) count=count+1 if count <= 0: print("Expected results") sys.exit(1) startKey = pysharkbite.Key("row3") range = pysharkbite.Range(None,False,startKey,True) scanner = tableOperations.createScanner(auths, 2) scanner.addRange( range ) resultset = scanner.getResultSet() count =0 for keyvalue in resultset: key = keyvalue.getKey() assert( "row2" == key.getRow() ) value = keyvalue.getValue() if "cf" == key.getColumnFamily(): assert( "value" == value.get() ) if ("cf2" == key.getColumnFamily() ): assert( "value2" == value.get() ) if ("cf3" == key.getColumnFamily() ): print("Unexpected column cf3") sys.exit(1) count=count+1 if count <= 0: print("Expected results") sys.exit(1) """ delete your table if user did not create temp """ tableOperations.remove()
def mthd(self): import pysharkbite tableOperations = super().getTableOperations() if not tableOperations.exists(False): print("Creating table") if not tableOperations.create(False): print("Could not create table") else: print("Table already exists, so not creating it") auths = pysharkbite.Authorizations() """ Add authorizations """ """ mutation.put("cf","cq","cv",1569786960) """ writer = tableOperations.createWriter(auths, 10) mutation = pysharkbite.Mutation("sow2") mutation.put("cf", "cq", "", 1569786960, "value") mutation.put("cf2", "cq2", "", 1569786960, "value2") """ no value """ mutation.put("cf3", "cq3", "", 1569786960, "") writer.addMutation(mutation) writer.close() writer = tableOperations.createWriter(auths, 10) rng = range(0, 1000) for i in rng: row = ("row%i" % (i + 5)) mutation = pysharkbite.Mutation(row) mutation.put("cf", "cq", "", 1569786960, "value") writer.addMutation(mutation) writer.close() print("written") """ auths.addAuthorization("cv") """ scanner = tableOperations.createScanner(auths, 2) startKey = pysharkbite.Key() endKey = pysharkbite.Key() startKey.setRow("sow") endKey.setRow("sow3") accumuloRange = pysharkbite.Range(startKey, True, endKey, False) scanner.addRange(accumuloRange) resultset = scanner.getResultSet() for keyvalue in resultset: key = keyvalue.getKey() assert ("sow2" == key.getRow()) value = keyvalue.getValue() if "cf" == key.getColumnFamily(): assert ("value" == value.get()) if ("cf2" == key.getColumnFamily()): assert ("value2" == value.get()) if ("cf3" == key.getColumnFamily()): assert ("" == value.get()) scanner = tableOperations.createScanner(auths, 2) accumuloRange = pysharkbite.Range("row", True, "", False) scanner.addRange(accumuloRange) resultset = scanner.getResultSet() count = 0 for keyvalue in resultset: key = keyvalue.getKey() count = count + 1 print("count is ", count) assert (count == 1003) # all rows + sows """ delete your table if user did not create temp """ tableOperations.remove()
def getDocuments(cancellationtoken: CancellationToken, name: int, lookupInformation: LookupInformation, input: queue.SimpleQueue, outputQueue: queue.SimpleQueue): count = 0 while cancellationtoken.running(): docInfo = None try: try: if input.empty(): pass else: docInfo = input.get(timeout=1) except: pass # Handle empty queue here if not docInfo is None: tableOps = lookupInformation.getTableOps() scanner = tableOps.createScanner(lookupInformation.getAuths(), 5) startKey = pysharkbite.Key() endKey = pysharkbite.Key() startKey.setRow(docInfo.getShard()) docid = docInfo.getDataType() + "\x00" + docInfo.getDocId() startKey.setColumnFamily(docid) endKey.setRow(docInfo.getShard()) endKey.setColumnFamily(docid + "\xff") print("Searching for " + docInfo.getShard()) rng = pysharkbite.Range(startKey, True, endKey, True) scanner.addRange(rng) rangecount = 1 while rangecount < 10: try: docInfo = input.get(False) startKey = pysharkbite.Key() endKey = pysharkbite.Key() startKey.setRow(docInfo.getShard()) docid = docInfo.getDataType( ) + "\x00" + docInfo.getDocId() startKey.setColumnFamily(docid) endKey.setRow(docInfo.getShard()) endKey.setColumnFamily(docid + "\xff") rng = pysharkbite.Range(startKey, True, endKey, True) print("Searching for " + docInfo.getShard()) scanner.addRange(rng) rangecount = rangecount + 1 except: rangecount = 11 with open('jsoncombiner.py', 'r') as file: combinertxt = file.read() combiner = pysharkbite.PythonIterator( "PythonCombiner", combinertxt, 100) scanner.addIterator(combiner) try: count = count + scanDoc(scanner, outputQueue) except: pass else: time.sleep(0.5) except: e = sys.exc_info()[0] return True
def check(): model = apps.get_model(app_label='query', model_name='FileUpload') objs = model.objects.filter(status="NEW") for obj in objs: if obj.status == "NEW": import pysharkbite conf = pysharkbite.Configuration() conf.set("FILE_SYSTEM_ROOT", "/accumulo") model = apps.get_model(app_label='query', model_name='AccumuloCluster') accumulo_cluster = model.objects.first() print("Checking " + str(obj.uuid)) if accumulo_cluster is None: return print("Checking " + str(obj.uuid)) zk = pysharkbite.ZookeeperInstance(accumulo_cluster.instance, accumulo_cluster.zookeeper, 1000, conf) user = pysharkbite.AuthInfo("root", "secret", zk.getInstanceId()) connector = pysharkbite.AccumuloConnector(user, zk) indexTableOps = connector.tableOps("provenanceIndex") auths = pysharkbite.Authorizations() auths.addAuthorization("PROV") indexScanner = indexTableOps.createScanner(auths, 2) indexrange = pysharkbite.Range(str(obj.uuid)) indexScanner.addRange(indexrange) indexSet = indexScanner.getResultSet() rangelist = list() provops = connector.tableOps("provenance") scanner = provops.createScanner(auths, 10) for indexKeyValue in indexSet: value = indexKeyValue.getValue() protobuf = Uid_pb2.List() protobuf.ParseFromString(value.get().encode()) for uidvalue in protobuf.UID: shard = indexKeyValue.getKey().getColumnQualifier().split( "\u0000")[0] datatype = indexKeyValue.getKey().getColumnQualifier( ).split("\u0000")[1] startKey = pysharkbite.Key() stopKey = pysharkbite.Key() startKey.setRow(shard) stopKey.setRow(shard) startKey.setColumnFamily(datatype + "\x00" + uidvalue) stopKey.setColumnFamily(datatype + "\x00" + uidvalue + "\xff") rangelist.append( pysharkbite.Range(startKey, True, stopKey, False)) scanner = provops.createScanner(auths, 10) scanner.addRange( pysharkbite.Range(startKey, True, stopKey, False)) resultset = scanner.getResultSet() for keyvalue in resultset: key = keyvalue.getKey() value = keyvalue.getValue() eventid = key.getColumnFamily().split("\u0000")[1] fieldname = key.getColumnQualifier().split("\u0000")[0] fieldvalue = key.getColumnQualifier().split( "\u0000")[1] if (fieldname == "EVENTTYPE"): if fieldvalue == "DROP": obj.status = "COMPLETE" obj.save() break scanner.close() indexScanner.close()