def get(self, request, *args, **kwargs): user = pysharkbite.AuthInfo(AccumuloCluster.objects.first().user,AccumuloCluster.objects.first().password, ZkInstance().get().getInstanceId()) connector = pysharkbite.AccumuloConnector(user, ZkInstance().get()) table = AccumuloCluster.objects.first().dataTable authstring = request.GET.get('auths') if not authstring is None and len(authstring) > 0 and authstring=="PROV": table="provenance" table_operations = connector.tableOps(table) shard = request.GET.get('shard') datatype = request.GET.get('dt') uid = request.GET.get('id') q = request.GET.get('query') auths = pysharkbite.Authorizations() if not authstring is None and len(authstring) > 0: auths.addAuthorization(authstring) asyncQueue = queue.SimpleQueue() asyncQueue.put(Range(datatype,shard,uid)) shardLookupInformation=LookupInformation(table,auths,table_operations) docs = queue.SimpleQueue() getDoc(shardLookupInformation,asyncQueue,docs) wanted_items=list() while not docs.empty(): wanted_items.append(docs.get()) context = {'shard':shard, 'uid' : uid, 'datatype' : datatype,'query': q , 'auths': authstring,'results' : wanted_items} return render(request,'mutate_page.html',context)
def get(self, request, *args, **kwargs): shard = request.GET.get('shard') datatype = request.GET.get('datatype') uid = request.GET.get('uid') query = request.GET.get('query') authstring = request.GET.get('auths') url = "/search/?q=" + query auths = pysharkbite.Authorizations() if not authstring is None and len(authstring) > 0: auths.addAuthorization(authstring) user = pysharkbite.AuthInfo(AccumuloCluster.objects.first().user,AccumuloCluster.objects.first().password, ZkInstance().get().getInstanceId()) connector = pysharkbite.AccumuloConnector(user, ZkInstance().get()) tableOps = connector.tableOps(AccumuloCluster.objects.first().dataTable) scanner = tableOps.createScanner(auths,1) startKey = pysharkbite.Key(row=shard) endKey = pysharkbite.Key(row=shard) docid = datatype + "\x00" + uid; startKey.setColumnFamily(docid) endKey.setColumnFamily(docid + "\xff") rng = pysharkbite.Range(startKey,True,endKey,True) scanner.addRange(rng) writer = tableOps.createWriter(auths,10) deletes = pysharkbite.Mutation(shard) for keyValue in scanner.getResultSet(): key = keyValue.getKey() deletes.putDelete( key.getColumnFamily(), key.getColumnQualifier(), key.getColumnVisibility(), key.getTimestamp()) ## scan for the original document writer.addMutation(deletes) writer.close() for auth in authstring.split("|"): url = url + "&auths=" + auth return HttpResponseRedirect(url)
def populateMetadata(): import pysharkbite conf = pysharkbite.Configuration() conf.set("FILE_SYSTEM_ROOT", "/accumulo") model = apps.get_model(app_label='query', model_name='AccumuloCluster') accumulo_cluster = model.objects.first() if accumulo_cluster is None: return zk = pysharkbite.ZookeeperInstance(accumulo_cluster.instance, accumulo_cluster.zookeeper, 1000, conf) user = pysharkbite.AuthInfo("root", "secret", zk.getInstanceId()) connector = pysharkbite.AccumuloConnector(user, zk) indexTableOps = connector.tableOps("DatawaveMetadata") auths = pysharkbite.Authorizations() indexScanner = indexTableOps.createScanner(auths, 100) indexrange = pysharkbite.Range() indexScanner.addRange(indexrange) indexScanner.fetchColumn("f", "") combinertxt = "" ## load the combiner from the file system and send it to accumulo with open('countgatherer.py', 'r') as file: combinertxt = file.read() combiner = pysharkbite.PythonIterator("MetadataCounter", combinertxt, 200) indexScanner.addIterator(combiner) indexSet = indexScanner.getResultSet() import json counts = 0 mapping = {} for indexKeyValue in indexSet: value = indexKeyValue.getValue() key = indexKeyValue.getKey() if key.getColumnFamily() == "f": day = key.getColumnQualifier().split("\u0000")[1] dt = key.getColumnQualifier().split("\u0000")[0] if day in mapping: if key.getRow() in mapping[day]: try: mapping[day][key.getRow()] += int(value.get()) except: pass else: try: mapping[day][key.getRow()] = int(value.get()) except: pass else: mapping[day] = {} try: mapping[day][key.getRow()] = int(value.get()) except: pass caches['metadata'].set("field", json.dumps(mapping), 3600 * 48) return json.dumps(mapping)
def mthd(self): import pysharkbite tableOperations = super().getTableOperations() if not tableOperations.exists(False): print("Creating table") if not tableOperations.create(False): print("Could not create table") else: tableOperations.remove() time.sleep(2) tableOperations.create(False) print("Table already exists, so not creating it") tableOperations = super().newtableOperations() splits = {"row", "row5", "row9", "sow"} tableOperations.addSplits(splits) auths = pysharkbite.Authorizations() """ Add authorizations """ """ mutation.put("cf","cq","cv",1569786960) """ writer = tableOperations.createWriter(auths, 10) rng = range(0, 500) for i in rng: row = ("row%i" % (i + 5)) mutation = pysharkbite.Mutation(row) rng2 = range(0, 100) for j in rng: mutation.put("cf%i" % j, "cq", "", 1569786960, "value") writer.addMutation(mutation) print("wrote %i entries" % ((i + 1) * (j + 1))) writer.close() tableOperations.compact("a", "s", True) print("written") scanner = tableOperations.createScanner(auths, 2) time.sleep(1) accumuloRange = pysharkbite.Range("a", True, "sow", False) scanner.addRange(accumuloRange) resultset = scanner.getResultSet() count = 0 for keyvalue in resultset: key = keyvalue.getKey() count = count + 1 print("count is ", count) """ delete your table if user did not create temp """
def get_uploads(): model = apps.get_model(app_label='query', model_name='FileUpload') objs = model.objects.filter(status="NEW") haveNew = False for obj in objs: if obj.status == "NEW": caches['eventcount'].set("ingestcomplete", 95) haveNew = True if not haveNew: caches['eventcount'].set("ingestcomplete", 100) return import pysharkbite conf = pysharkbite.Configuration() conf.set("FILE_SYSTEM_ROOT", "/accumulo") model = apps.get_model(app_label='query', model_name='AccumuloCluster') accumulo_cluster = model.objects.first() if accumulo_cluster is None: return zk = pysharkbite.ZookeeperInstance(accumulo_cluster.instance, accumulo_cluster.zookeeper, 1000, conf) user = pysharkbite.AuthInfo("root", "secret", zk.getInstanceId()) connector = pysharkbite.AccumuloConnector(user, zk) auths = pysharkbite.Authorizations() auths.addAuthorization("PROV") indexTableOps = connector.tableOps("provenanceIndex") indexScanner = indexTableOps.createScanner(auths, 10) indexrange = pysharkbite.Range() indexScanner.addRange(indexrange) indexScanner.fetchColumn("CONTENTURI", "") indexScanner.fetchColumn("TRANSITURI", "") indexSet = indexScanner.getResultSet() count = 0 usercount = 0 try: for indexKeyValue in indexSet: if indexKeyValue.getKey().getColumnFamily() == "CONTENTURI": count = count + 1 else: usercount = usercount + 1 if count > 0: caches['eventcount'].set("ingestcount", count, 3600 * 48) if usercount > 0: caches['eventcount'].set("useruploads", usercount, 3600 * 48) indexScanner.close() except: pass ## user does not have PROV
def mthd(self): import pysharkbite #pysharkbite.LoggingConfiguration.enableTraceLogger() tableOperations = super().getTableOperations() if not tableOperations.exists(False): print ("Creating table") tableOperations.create(False) else: print ("Table already exists, so not creating it") auths = pysharkbite.Authorizations() auths.addAuthorization("blah1") auths.addAuthorization("blah2") securityOps = super().getConnector().securityOps() securityOps.grantAuthorizations(auths,"root") """ Add authorizations """ """ mutation.put("cf","cq","cv",1569786960) """ writer = tableOperations.createWriter(auths, 10) mutation = pysharkbite.Mutation("row2"); mutation.put("cf","cq","blah1",1569786960, "value") mutation.put("cf2","cq2","blah1",1569786960, "value2") """ no value """ mutation.put("cf3","cq3","blah2",1569786960, "") writer.addMutation( mutation ) writer.close() print("written") auths = pysharkbite.Authorizations() auths.addAuthorization("blah1") scanner = tableOperations.createScanner(auths, 2) startKey = pysharkbite.Key() endKey = pysharkbite.Key() startKey.setRow("row") endKey.setRow("row3") # test single range range = pysharkbite.Range("row2") scanner.addRange( range ) resultset = scanner.getResultSet() for keyvalue in resultset: key = keyvalue.getKey() assert( "row2" == key.getRow() ) value = keyvalue.getValue() if "cf" == key.getColumnFamily(): assert( "value" == value.get() ) if ("cf2" == key.getColumnFamily() ): assert( "value2" == value.get() ) if ("cf3" == key.getColumnFamily() ): print("Unexpected column cf3") sys.exit(1) range = pysharkbite.Range("row1",True,"row1.5",True) scanner.addRange( range ) resultset = scanner.getResultSet() for keyvalue in resultset: print("Unexpected result") sys.exit(1) # test single range range = pysharkbite.Range("row",False,"row3",True) scanner = tableOperations.createScanner(auths, 2) scanner.addRange( range ) resultset = scanner.getResultSet() count =0 for keyvalue in resultset: key = keyvalue.getKey() assert( "row2" == key.getRow() ) value = keyvalue.getValue() if "cf" == key.getColumnFamily(): assert( "value" == value.get() ) if ("cf2" == key.getColumnFamily() ): assert( "value2" == value.get() ) if ("cf3" == key.getColumnFamily() ): print("Unexpected column cf3") sys.exit(1) count=count+1 if count <= 0: print("Expected results") sys.exit(1) # test infinite range range = pysharkbite.Range("",False,"row3",True) scanner = tableOperations.createScanner(auths, 2) scanner.addRange( range ) resultset = scanner.getResultSet() count =0 for keyvalue in resultset: key = keyvalue.getKey() assert( "row2" == key.getRow() ) value = keyvalue.getValue() if "cf" == key.getColumnFamily(): assert( "value" == value.get() ) if ("cf2" == key.getColumnFamily() ): assert( "value2" == value.get() ) if ("cf3" == key.getColumnFamily() ): print("Unexpected column cf3") sys.exit(1) count=count+1 if count <= 0: print("Expected results") sys.exit(1) startKey = pysharkbite.Key("row3") range = pysharkbite.Range(None,False,startKey,True) scanner = tableOperations.createScanner(auths, 2) scanner.addRange( range ) resultset = scanner.getResultSet() count =0 for keyvalue in resultset: key = keyvalue.getKey() assert( "row2" == key.getRow() ) value = keyvalue.getValue() if "cf" == key.getColumnFamily(): assert( "value" == value.get() ) if ("cf2" == key.getColumnFamily() ): assert( "value2" == value.get() ) if ("cf3" == key.getColumnFamily() ): print("Unexpected column cf3") sys.exit(1) count=count+1 if count <= 0: print("Expected results") sys.exit(1) """ delete your table if user did not create temp """ tableOperations.remove()
def mthd(self): import pysharkbite tableOperations = super().getTableOperations() if not tableOperations.exists(False): print("Creating table") if not tableOperations.create(False): print("Could not create table") else: print("Table already exists, so not creating it") auths = pysharkbite.Authorizations() """ Add authorizations """ """ mutation.put("cf","cq","cv",1569786960) """ writer = tableOperations.createWriter(auths, 10) mutation = pysharkbite.Mutation("sow2") mutation.put("cf", "cq", "", 1569786960, "value") mutation.put("cf2", "cq2", "", 1569786960, "value2") """ no value """ mutation.put("cf3", "cq3", "", 1569786960, "") writer.addMutation(mutation) writer.close() writer = tableOperations.createWriter(auths, 10) rng = range(0, 1000) for i in rng: row = ("row%i" % (i + 5)) mutation = pysharkbite.Mutation(row) mutation.put("cf", "cq", "", 1569786960, "value") writer.addMutation(mutation) writer.close() print("written") """ auths.addAuthorization("cv") """ scanner = tableOperations.createScanner(auths, 2) startKey = pysharkbite.Key() endKey = pysharkbite.Key() startKey.setRow("sow") endKey.setRow("sow3") accumuloRange = pysharkbite.Range(startKey, True, endKey, False) scanner.addRange(accumuloRange) resultset = scanner.getResultSet() for keyvalue in resultset: key = keyvalue.getKey() assert ("sow2" == key.getRow()) value = keyvalue.getValue() if "cf" == key.getColumnFamily(): assert ("value" == value.get()) if ("cf2" == key.getColumnFamily()): assert ("value2" == value.get()) if ("cf3" == key.getColumnFamily()): assert ("" == value.get()) scanner = tableOperations.createScanner(auths, 2) accumuloRange = pysharkbite.Range("row", True, "", False) scanner.addRange(accumuloRange) resultset = scanner.getResultSet() count = 0 for keyvalue in resultset: key = keyvalue.getKey() count = count + 1 print("count is ", count) assert (count == 1003) # all rows + sows """ delete your table if user did not create temp """ tableOperations.remove()
def post(self, request, *args, **kwargs): query = request.POST.get('query') shard = request.POST.get('shard') authstring = request.POST.get('auths') datatype = request.POST.get('datatype') uid = request.POST.get('uid') originals = {} news = {} for key, value in request.POST.items(): if key == "query": query = value elif key.startswith("original"): split = key.split(".") originals[split[1]] = value elif key == "shard" or key == "datatype" or key == "uid" or key == "auths": pass elif key == "csrfmiddlewaretoken": pass else: news[key] = value user = pysharkbite.AuthInfo(AccumuloCluster.objects.first().user, AccumuloCluster.objects.first().password, ZkInstance().get().getInstanceId()) connector = pysharkbite.AccumuloConnector(user, ZkInstance().get()) auths = pysharkbite.Authorizations() #for auth in if not authstring is None and len(authstring) > 0: auths.addAuthorization(authstring) table = AccumuloCluster.objects.first().dataTable index_table = AccumuloCluster.objects.first().indexTable table_operations = connector.tableOps(table) index_table_ops = connector.tableOps(index_table) writer = table_operations.createWriter(auths, 10) indexWriter = index_table_ops.createWriter(auths, 5) mutation = pysharkbite.Mutation(shard) diff = 0 for key, value in news.items(): if news[key] != originals[key]: import datetime ts = int(datetime.datetime.now().timestamp()) * 1000 mutation.putDelete(datatype + "\x00" + uid, key + "\x00" + originals[key], authstring, ts) ts = int(datetime.datetime.now().timestamp()) * 1000 + 100 mutation.put(datatype + "\x00" + uid, key + "\x00" + news[key], authstring, ts) originalIndexMutation = pysharkbite.Mutation( originals[key].lower()) indexMutation = pysharkbite.Mutation(news[key].lower()) protobuf = Uid_pb2.List() protobuf.COUNT = 1 protobuf.IGNORE = False protobuf.UID.append(uid) indexMutation.put(key, shard + "\x00" + datatype, authstring, ts, protobuf.SerializeToString()) originalprotobuf = Uid_pb2.List() indexWriter.addMutation(indexMutation) originalprotobuf.COUNT = 1 originalprotobuf.IGNORE = False originalprotobuf.REMOVEDUID.append(uid) originalIndexMutation.put(key, shard + "\x00" + datatype, authstring, ts, originalprotobuf.SerializeToString()) indexWriter.addMutation(originalIndexMutation) diff = diff + 1 else: pass if diff > 0: writer.addMutation(mutation) indexWriter.close() writer.close() authy = "" url = "/search/?q=" + query for auth in authstring.split("|"): url = url + "&auths=" + auth return HttpResponseRedirect(url)
def get(self, request, *args, **kwargs): user = pysharkbite.AuthInfo(AccumuloCluster.objects.first().user, AccumuloCluster.objects.first().password, ZkInstance().get().getInstanceId()) connector = pysharkbite.AccumuloConnector(user, ZkInstance().get()) entry = request.GET.get('q') selectedauths = request.GET.getlist('auths') try: skip = int(request.GET.get('s')) except: skip = 0 field = request.GET.get('f') # try: # LuceneToJexlQueryParser = jnius.autoclass('datawave.query.language.parser.jexl.LuceneToJexlQueryParser') # luceneparser = LuceneToJexlQueryParser() # node = luceneparser.parse(entry) # jexl = node.getOriginalQuery() # except: # pass indexLookup = 1 table = AccumuloCluster.objects.first().dataTable index_table = AccumuloCluster.objects.first().indexTable isProv = False authlist = list() auths = pysharkbite.Authorizations() for auth in selectedauths: if len(auth) > 0: if auth == "PROV": isProv = True authlist.append(auth) auths.addAuthorization(auth) if isProv is True and len(authlist) == 1: table = "provenance" index_table = "provenanceIndex" table_operations = connector.tableOps(table) index_table_ops = connector.tableOps(index_table) #auths = pysharkbite.Authorizations() start = time.time() indexLookupInformation = LookupInformation(index_table, auths, index_table_ops) shardLookupInformation = LookupInformation(table, auths, table_operations) wanted_items = list() tree = parser.parse(entry) tree = resolver(tree) visitor = IndexLookup() iterator = visitor.visit(tree) if isinstance(iterator, RangeLookup): rng = iterator iterator = OrIterator() iterator.addRange(rng) docs = queue.SimpleQueue() lookup(indexLookupInformation, shardLookupInformation, iterator, docs) counts = 0 header = set() while not docs.empty(): jsondoc = docs.get() for key in jsondoc.keys(): if key != "ORIG_FILE" and key != "TERM_COUNT" and key != "RAW_FILE" and key != "shard" and key != "datatype" and key != "uid": header.add(key) wanted_items.append(jsondoc) counts = counts + 1 nxt = "" prv = "" userAuths = set() try: auths = UserAuths.objects.get(name=request.user) user_auths = auths.authorizations.all() if not user_auths is None: for authset in user_auths: userAuths.add(authset) except: pass s = "|" authy = s.join(authlist) context = { 'header': header, 'authstring': authy, 'selectedauths': selectedauths, 'results': wanted_items, 'time': (time.time() - start), 'prv': prv, 'nxt': nxt, 'field': field, 'admin': request.user.is_superuser, 'authenticated': True, 'userAuths': userAuths, 'query': entry } return render(request, 'search_results.html', context)
def mthd(self): import pysharkbite securityOps = super().getSecurityOperations() auths = pysharkbite.Authorizations() auths.addAuthorization("blah1") auths.addAuthorization("blah2") securityOps.grantAuthorizations(auths, "root") tableOperations = super().getTableOperations() tableOperations.create(False) ## validate that we DO see the permissions """ Add authorizations """ """ mutation.put("cf","cq","cv",1569786960) """ with tableOperations.createWriter(auths, 10) as writer: mutation = pysharkbite.Mutation("row2") mutation.put("cf", "cq", "blah1", 1569786960, "value") mutation.put("cf2", "cq2", "blah1", 1569786960, "value2") """ no value """ mutation.put("cf3", "cq3", "blah2", 1569786960, "") writer.addMutation(mutation) mutation = pysharkbite.Mutation("row4") mutation.put("cf", "cq", "blah1", 1569786960, "value") mutation.put("cf2", "cq2", "blah1", 1569786960, "value2") """ no value """ mutation.put("cf3", "cq3", "blah2", 1569786960, "") writer.addMutation(mutation) auths = pysharkbite.Authorizations() auths.addAuthorization("blah1") auths.addAuthorization("blah2") count = 0 with tableOperations.createScanner(auths, 2).withRange( pysharkbite.Range("row", True, "row3", False)) as scanner: resultset = scanner.getResultSet() for keyvalue in resultset: count = count + 1 key = keyvalue.getKey() assert ("row2" == key.getRow()) value = keyvalue.getValue() if "cf" == key.getColumnFamily(): assert ("value" == value.get()) if ("cf2" == key.getColumnFamily()): assert ("value2" == value.get()) with tableOperations.createScanner(auths, 2).withRange( pysharkbite.Range("row4", True, "row5", False)) as scanner: resultset = scanner.getResultSet() for keyvalue in resultset: count = count + 1 key = keyvalue.getKey() print("%r %r" % (keyvalue.getKey(), str(keyvalue.getValue()))) assert ("row4" == key.getRow()) value = keyvalue.getValue() if "cf" == key.getColumnFamily(): assert ("value" == value.get()) if ("cf2" == key.getColumnFamily()): assert ("value2" == value.get()) """ delete your table if user did not create temp """ tableOperations.remove() print("count is " + str(count)) assert (count == 6)
def run(*args): admin_user = "******" admin_password = "******" auth = Auth() auth.auth = "TST" auth.save() democonfig = IngestConfiguration() democonfig.name = "democonfig" democonfig.use_provenance = True democonfig.post_location = "http://nifi:8181/contentListener" democonfig.save() instance = "uno" zookeepers = "192.168.1.88:2181" acc_user = "******" acc_pass = "******" for arg in args: if arg.startswith("password"): split = arg.split("=") if len(split) == 2: admin_password = split[1] if arg.startswith("username"): split = arg.split("=") if len(split) == 2: admin_user = split[1] if arg.startswith("instance"): split = arg.split("=") if len(split) == 2: admin_user = split[1] if arg.startswith("zookeepers"): split = arg.split("=") if len(split) == 2: zookeepers = split[1] if arg.startswith("accuser"): split = arg.split("=") if len(split) == 2: acc_user = split[1] if arg.startswith("accpass"): split = arg.split("=") if len(split) == 2: acc_pass = split[1] acc = AccumuloCluster() acc.instance = instance acc.zookeeper = zookeepers acc.user = acc_user acc.password = acc_pass acc.save() import pysharkbite while True: try: conf = pysharkbite.Configuration() conf.set("FILE_SYSTEM_ROOT", "/accumulo") zoo_keeper = pysharkbite.ZookeeperInstance(instance, zookeepers, 1000, conf) user = pysharkbite.AuthInfo(acc_user, acc_pass, zoo_keeper.getInstanceId()) connector = pysharkbite.AccumuloConnector(user, zoo_keeper) security_ops = connector.securityOps() auths = pysharkbite.Authorizations() auths.addAuthorization("MTRCS") auths.addAuthorization("PROV") security_ops.grantAuthorizations(auths, acc_user) table_ops = connector.tableOps("shard") if not table_ops.exists(False): table_ops.create(False) table_ops = connector.tableOps("shardIndex") if not table_ops.exists(False): table_ops.create(False) table_ops = connector.tableOps("shardReverse") if not table_ops.exists(False): table_ops.create(False) table_ops = connector.tableOps("graph") if not table_ops.exists(False): table_ops.create(False) table_ops = connector.tableOps("provenance") if not table_ops.exists(False): table_ops.create(False) table_ops = connector.tableOps("provenanceIndex") if not table_ops.exists(False): table_ops.create(False) table_ops = connector.tableOps("provenanceReverseIndex") if not table_ops.exists(False): table_ops.create(False) break except RuntimeError as e: exc = str(e) if exc.find("Instance Id does not exist") > 0: pass else: break
def pouplateEventCountMetadata(): import pysharkbite import time conf = pysharkbite.Configuration() conf.set("FILE_SYSTEM_ROOT", "/accumulo") model = apps.get_model(app_label='query', model_name='AccumuloCluster') accumulo_cluster = model.objects.first() if accumulo_cluster is None: return zoo_keeper = pysharkbite.ZookeeperInstance(accumulo_cluster.instance, accumulo_cluster.zookeeper, 1000, conf) user = pysharkbite.AuthInfo("root", "secret", zoo_keeper.getInstanceId()) connector = pysharkbite.AccumuloConnector(user, zoo_keeper) queryRanges = list() #last 15 days for dateinrange in getDateRange(-15): shardbegin = dateinrange.strftime("%Y%m%d") if caches['eventcount'].get(shardbegin) is None or caches[ 'eventcount'].get(shardbegin) == 0: queryRanges.append(shardbegin) else: pass # don't add to range if len(queryRanges) > 0: ## all is cached user = pysharkbite.AuthInfo("root", "secret", zoo_keeper.getInstanceId()) connector = pysharkbite.AccumuloConnector(user, zoo_keeper) indexTableOps = connector.tableOps("DatawaveMetrics") auths = pysharkbite.Authorizations() auths.addAuthorization("MTRCS") indexScanner = indexTableOps.createScanner(auths, 100) start = time.time() for dt in queryRanges: indexrange = pysharkbite.Range(dt, True, dt + "\uffff", False) indexScanner.addRange(indexrange) indexScanner.fetchColumn("EVENT_COUNT", "") combinertxt = "" ## load the combiner from the file system and send it to accumulo with open('metricscombiner.py', 'r') as file: combinertxt = file.read() combiner = pysharkbite.PythonIterator("MetadataCounter", combinertxt, 200) indexScanner.addIterator(combiner) indexSet = indexScanner.getResultSet() counts = 0 mapping = {} try: for indexKeyValue in indexSet: value = indexKeyValue.getValue() key = indexKeyValue.getKey() if key.getColumnFamily() == "EVENT_COUNT": dt = key.getRow().split("_")[0] if dt in mapping: mapping[dt] += int(value.get()) else: mapping[dt] = int(value.get()) arr = [None] * len(mapping.keys()) for field in mapping: caches['eventcount'].set(field, str(mapping[field]), 3600 * 48) except: pass
def check(): model = apps.get_model(app_label='query', model_name='FileUpload') objs = model.objects.filter(status="NEW") for obj in objs: if obj.status == "NEW": import pysharkbite conf = pysharkbite.Configuration() conf.set("FILE_SYSTEM_ROOT", "/accumulo") model = apps.get_model(app_label='query', model_name='AccumuloCluster') accumulo_cluster = model.objects.first() print("Checking " + str(obj.uuid)) if accumulo_cluster is None: return print("Checking " + str(obj.uuid)) zk = pysharkbite.ZookeeperInstance(accumulo_cluster.instance, accumulo_cluster.zookeeper, 1000, conf) user = pysharkbite.AuthInfo("root", "secret", zk.getInstanceId()) connector = pysharkbite.AccumuloConnector(user, zk) indexTableOps = connector.tableOps("provenanceIndex") auths = pysharkbite.Authorizations() auths.addAuthorization("PROV") indexScanner = indexTableOps.createScanner(auths, 2) indexrange = pysharkbite.Range(str(obj.uuid)) indexScanner.addRange(indexrange) indexSet = indexScanner.getResultSet() rangelist = list() provops = connector.tableOps("provenance") scanner = provops.createScanner(auths, 10) for indexKeyValue in indexSet: value = indexKeyValue.getValue() protobuf = Uid_pb2.List() protobuf.ParseFromString(value.get().encode()) for uidvalue in protobuf.UID: shard = indexKeyValue.getKey().getColumnQualifier().split( "\u0000")[0] datatype = indexKeyValue.getKey().getColumnQualifier( ).split("\u0000")[1] startKey = pysharkbite.Key() stopKey = pysharkbite.Key() startKey.setRow(shard) stopKey.setRow(shard) startKey.setColumnFamily(datatype + "\x00" + uidvalue) stopKey.setColumnFamily(datatype + "\x00" + uidvalue + "\xff") rangelist.append( pysharkbite.Range(startKey, True, stopKey, False)) scanner = provops.createScanner(auths, 10) scanner.addRange( pysharkbite.Range(startKey, True, stopKey, False)) resultset = scanner.getResultSet() for keyvalue in resultset: key = keyvalue.getKey() value = keyvalue.getValue() eventid = key.getColumnFamily().split("\u0000")[1] fieldname = key.getColumnQualifier().split("\u0000")[0] fieldvalue = key.getColumnQualifier().split( "\u0000")[1] if (fieldname == "EVENTTYPE"): if fieldvalue == "DROP": obj.status = "COMPLETE" obj.save() break scanner.close() indexScanner.close()
def run_edge_query(query_id): model = apps.get_model(app_label='query', model_name='EdgeQuery') objs = model.objects.filter(query_id=query_id) for obj in objs: obj.running = True obj.save() import pysharkbite conf = pysharkbite.Configuration() conf.set("FILE_SYSTEM_ROOT", "/accumulo") model = apps.get_model(app_label='query', model_name='AccumuloCluster') accumulo_cluster = model.objects.first() if accumulo_cluster is None: return zk = pysharkbite.ZookeeperInstance(accumulo_cluster.instance, accumulo_cluster.zookeeper, 1000, conf) user = pysharkbite.AuthInfo("root", "secret", zk.getInstanceId()) connector = pysharkbite.AccumuloConnector(user, zk) auths = pysharkbite.Authorizations() if obj.auths: for auth in obj.auths.split(","): auths.addAuthorization(auth) sres_model = apps.get_model(app_label='query', model_name='ScanResult') res_model = apps.get_model(app_label='query', model_name='Result') sr = sres_model.objects.filter(query_id=obj.query_id).first() if not sr: print("No scan result, returning") return print("here") graphTableOps = connector.tableOps("graph") scanner = graphTableOps.createScanner(auths, 10) range = pysharkbite.Range(obj.query, True, obj.query + "\uffff" + "\uffff", False) ## for now the range should be this scanner.addRange(range) resultset = scanner.getResultSet() count = 0 try: for indexKeyValue in resultset: value = "0" ## row will be the to ## direction will be the cf to_value = "" direction = "one" try: protobuf = EdgeData_pb2.EdgeValue() protobuf.ParseFromString( indexKeyValue.getValue().get_bytes()) value = str(protobuf.count) + "/" + protobuf.uuid_string to_value = indexKeyValue.getKey().getRow().split( "\u0000")[1] direction = indexKeyValue.getKey().getColumnFamily().split( "/")[1] direction_split = direction.split("-") if len(direction_split ) != 2 or direction_split[0] == direction_split[1]: continue except Exception as e: print(e) continue except: continue scanresult = res_model.objects.create( scanResult=sr, value=value, row=to_value, cf=direction, cq=indexKeyValue.getKey().getColumnQualifier()) scanresult.save() count = count + 1 if count > 1000: break sr.is_finished = True sr.save() scanner.close() except Exception as e: print(e) except: print("An error occurred") pass ## user does not have PROV obj.running = False obj.finished = True obj.save()
def mthd(self): import pysharkbite tableOperations = super().getTableOperations() ## validate that table is removed try: if not tableOperations.exists(False): print("Creating table") if not tableOperations.create(False): print("Could not create table") else: print("Table already exists, so not creating it") auths = pysharkbite.Authorizations() """ Add authorizations """ """ mutation.put("cf","cq","cv",1569786960) """ tableOperations.remove() time.sleep(1) writer = tableOperations.createWriter(auths, 10) mutation = pysharkbite.Mutation("sow2") mutation.put("cf", "cq", "", 1569786960, "value") mutation.put("cf2", "cq2", "", 1569786960, "value2") """ no value """ mutation.put("cf3", "cq3", "", 1569786960, "") writer.addMutation(mutation) writer.close() writer = tableOperations.createWriter(auths, 10) rng = range(0, 1000) for i in rng: row = ("row%i" % (i + 5)) mutation = pysharkbite.Mutation(row) mutation.put("cf", "cq", "", 1569786960, "value") writer.addMutation(mutation) writer.close() print("Table not removed") sys.exit(1) except (RuntimeError, TypeError, NameError): print("Table successfully removed") # this is okay try: testzk = pysharkbite.ZookeeperInstance(None, None, 1000, None) print("Table not removed") sys.exit(1) except (RuntimeError, TypeError, NameError): print("Caught expected error") # this is okay try: writer = tableOperations.createWriter(None, 10) print("Expected error passing None") sys.exit(1) except (RuntimeError, TypeError, NameError): print("Caught expected error") # this is okay tableOperations.remove()
user = pysharkbite.AuthInfo(args.username, password, zk.getInstanceId()) try: connector = pysharkbite.AccumuloConnector(user, zk) tableOperations = connector.tableOps(table) if not tableOperations.exists(False): print ("Creating table " + table) tableOperations.create(False) else: print (table + " already exists, so not creating it") auths = pysharkbite.Authorizations() """ Add authorizations """ """ mutation.put("cf","cq","cv",1569786960) """ writer = tableOperations.createWriter(auths, 10) mutation = pysharkbite.Mutation("row2"); mutation.put("cf","cq","",1569786960, "value") mutation.put("cf2","cq2","",1569786960, "value2") """ no value """ mutation.put("cf3","cq3","",1569786960, "") writer.addMutation( mutation )
def mthd(self): import pysharkbite tableOperations = super().getTableOperations() if not tableOperations.exists(False): print("Creating table") if not tableOperations.create(False): print("Could not create table") else: print("Table already exists, so not creating it") auths = pysharkbite.Authorizations() """ Add authorizations """ """ mutation.put("cf","cq","cv",1569786960) """ writer = tableOperations.createWriter(auths, 10) mutation = pysharkbite.Mutation("sow2") mutation.put("cf", "cq", "", 1569786960, "value") mutation.put("cf2", "cq", "", 1569786960, "value") """ no value """ mutation.put("cf3", "cq", "", 1569786960, "value") writer.addMutation(mutation) writer.close() writer = tableOperations.createWriter(auths, 10) rng = range(0, 1000) for i in rng: row = ("row%i" % (i + 5)) mutation = pysharkbite.Mutation(row) mutation.put("cf", "cq", "", 1569786960, "value") writer.addMutation(mutation) writer.close() print("written") """ auths.addAuthorization("cv") """ scanner = tableOperations.createScanner(auths, 2) accumuloRange = pysharkbite.Range("sow", True, "sow3", False) scanner.addRange(accumuloRange) iterator = pysharkbite.PythonIterator("PythonIterator", 100) iterator = iterator.onNext( "lambda x : KeyValue( Key( x.getKey().getRow(), 'new cf', x.getKey().getColumnQualifier()), Value()) " ) scanner.addIterator(iterator) resultset = scanner.getResultSet() for keyvalue in resultset: key = keyvalue.getKey() assert ("sow2" == key.getRow()) value = keyvalue.getValue() if "cf" == key.getColumnFamily(): sys.exit(154) if "new cf" == key.getColumnFamily(): assert ("" == value.get()) scanner = tableOperations.createScanner(auths, 2) accumuloRange = pysharkbite.Range("sow", True, "sow3", False) scanner.addRange(accumuloRange) iterator = pysharkbite.PythonIterator("PythonIterator", 100) iterator = iterator.onNext( "lambda x : Key( x.getKey().getRow(), x.getKey().getColumnFamily(), 'new cq') " ) scanner.addIterator(iterator) resultset = scanner.getResultSet() for keyvalue in resultset: key = keyvalue.getKey() assert ("sow2" == key.getRow()) value = keyvalue.getValue() if "cq" == key.getColumnQualifier(): sys.exit(154) """ delete your table if user did not create temp """ tableOperations.remove()