def get(self, request, *args, **kwargs):
user = pysharkbite.AuthInfo(AccumuloCluster.objects.first().user,AccumuloCluster.objects.first().password, ZkInstance().get().getInstanceId())
connector = pysharkbite.AccumuloConnector(user, ZkInstance().get())
table = AccumuloCluster.objects.first().dataTable
authstring = request.GET.get('auths')
if not authstring is None and len(authstring) > 0 and authstring=="PROV":
table="provenance"
table_operations = connector.tableOps(table)
shard = request.GET.get('shard')
datatype = request.GET.get('dt')
uid = request.GET.get('id')
q = request.GET.get('query')
auths = pysharkbite.Authorizations()
if not authstring is None and len(authstring) > 0:
auths.addAuthorization(authstring)
asyncQueue = queue.SimpleQueue()
asyncQueue.put(Range(datatype,shard,uid))
shardLookupInformation=LookupInformation(table,auths,table_operations)
docs = queue.SimpleQueue()
getDoc(shardLookupInformation,asyncQueue,docs)
wanted_items=list()
while not docs.empty():
wanted_items.append(docs.get())
context = {'shard':shard, 'uid' : uid, 'datatype' : datatype,'query': q , 'auths': authstring,'results' : wanted_items}
return render(request,'mutate_page.html',context)
def get(self, request, *args, **kwargs):
shard = request.GET.get('shard')
datatype = request.GET.get('datatype')
uid = request.GET.get('uid')
query = request.GET.get('query')
authstring = request.GET.get('auths')
url = "/search/?q=" + query
auths = pysharkbite.Authorizations()
if not authstring is None and len(authstring) > 0:
auths.addAuthorization(authstring)
user = pysharkbite.AuthInfo(AccumuloCluster.objects.first().user,AccumuloCluster.objects.first().password, ZkInstance().get().getInstanceId())
connector = pysharkbite.AccumuloConnector(user, ZkInstance().get())
tableOps = connector.tableOps(AccumuloCluster.objects.first().dataTable)
scanner = tableOps.createScanner(auths,1)
startKey = pysharkbite.Key(row=shard)
endKey = pysharkbite.Key(row=shard)
docid = datatype + "\x00" + uid;
startKey.setColumnFamily(docid)
endKey.setColumnFamily(docid + "\xff")
rng = pysharkbite.Range(startKey,True,endKey,True)
scanner.addRange(rng)
writer = tableOps.createWriter(auths,10)
deletes = pysharkbite.Mutation(shard)
for keyValue in scanner.getResultSet():
key = keyValue.getKey()
deletes.putDelete( key.getColumnFamily(), key.getColumnQualifier(), key.getColumnVisibility(), key.getTimestamp())
## scan for the original document
writer.addMutation(deletes)
writer.close()
for auth in authstring.split("|"):
url = url + "&auths=" + auth
return HttpResponseRedirect(url)
def populateMetadata():
import pysharkbite
conf = pysharkbite.Configuration()
conf.set("FILE_SYSTEM_ROOT", "/accumulo")
model = apps.get_model(app_label='query', model_name='AccumuloCluster')
accumulo_cluster = model.objects.first()
if accumulo_cluster is None:
return
zk = pysharkbite.ZookeeperInstance(accumulo_cluster.instance,
accumulo_cluster.zookeeper, 1000, conf)
user = pysharkbite.AuthInfo("root", "secret", zk.getInstanceId())
connector = pysharkbite.AccumuloConnector(user, zk)
indexTableOps = connector.tableOps("DatawaveMetadata")
auths = pysharkbite.Authorizations()
indexScanner = indexTableOps.createScanner(auths, 100)
indexrange = pysharkbite.Range()
indexScanner.addRange(indexrange)
indexScanner.fetchColumn("f", "")
combinertxt = ""
## load the combiner from the file system and send it to accumulo
with open('countgatherer.py', 'r') as file:
combinertxt = file.read()
combiner = pysharkbite.PythonIterator("MetadataCounter", combinertxt, 200)
indexScanner.addIterator(combiner)
indexSet = indexScanner.getResultSet()
import json
counts = 0
mapping = {}
for indexKeyValue in indexSet:
value = indexKeyValue.getValue()
key = indexKeyValue.getKey()
if key.getColumnFamily() == "f":
day = key.getColumnQualifier().split("\u0000")[1]
dt = key.getColumnQualifier().split("\u0000")[0]
if day in mapping:
if key.getRow() in mapping[day]:
try:
mapping[day][key.getRow()] += int(value.get())
except:
pass
else:
try:
mapping[day][key.getRow()] = int(value.get())
except:
pass
else:
mapping[day] = {}
try:
mapping[day][key.getRow()] = int(value.get())
except:
pass
caches['metadata'].set("field", json.dumps(mapping), 3600 * 48)
return json.dumps(mapping)
def mthd(self):
import pysharkbite
tableOperations = super().getTableOperations()
if not tableOperations.exists(False):
print("Creating table")
if not tableOperations.create(False):
print("Could not create table")
else:
tableOperations.remove()
time.sleep(2)
tableOperations.create(False)
print("Table already exists, so not creating it")
tableOperations = super().newtableOperations()
splits = {"row", "row5", "row9", "sow"}
tableOperations.addSplits(splits)
auths = pysharkbite.Authorizations()
""" Add authorizations """
""" mutation.put("cf","cq","cv",1569786960) """
writer = tableOperations.createWriter(auths, 10)
rng = range(0, 500)
for i in rng:
row = ("row%i" % (i + 5))
mutation = pysharkbite.Mutation(row)
rng2 = range(0, 100)
for j in rng:
mutation.put("cf%i" % j, "cq", "", 1569786960, "value")
writer.addMutation(mutation)
print("wrote %i entries" % ((i + 1) * (j + 1)))
writer.close()
tableOperations.compact("a", "s", True)
print("written")
scanner = tableOperations.createScanner(auths, 2)
time.sleep(1)
accumuloRange = pysharkbite.Range("a", True, "sow", False)
scanner.addRange(accumuloRange)
resultset = scanner.getResultSet()
count = 0
for keyvalue in resultset:
key = keyvalue.getKey()
count = count + 1
print("count is ", count)
""" delete your table if user did not create temp """
def get_uploads():
model = apps.get_model(app_label='query', model_name='FileUpload')
objs = model.objects.filter(status="NEW")
haveNew = False
for obj in objs:
if obj.status == "NEW":
caches['eventcount'].set("ingestcomplete", 95)
haveNew = True
if not haveNew:
caches['eventcount'].set("ingestcomplete", 100)
return
import pysharkbite
conf = pysharkbite.Configuration()
conf.set("FILE_SYSTEM_ROOT", "/accumulo")
model = apps.get_model(app_label='query', model_name='AccumuloCluster')
accumulo_cluster = model.objects.first()
if accumulo_cluster is None:
return
zk = pysharkbite.ZookeeperInstance(accumulo_cluster.instance,
accumulo_cluster.zookeeper, 1000, conf)
user = pysharkbite.AuthInfo("root", "secret", zk.getInstanceId())
connector = pysharkbite.AccumuloConnector(user, zk)
auths = pysharkbite.Authorizations()
auths.addAuthorization("PROV")
indexTableOps = connector.tableOps("provenanceIndex")
indexScanner = indexTableOps.createScanner(auths, 10)
indexrange = pysharkbite.Range()
indexScanner.addRange(indexrange)
indexScanner.fetchColumn("CONTENTURI", "")
indexScanner.fetchColumn("TRANSITURI", "")
indexSet = indexScanner.getResultSet()
count = 0
usercount = 0
try:
for indexKeyValue in indexSet:
if indexKeyValue.getKey().getColumnFamily() == "CONTENTURI":
count = count + 1
else:
usercount = usercount + 1
if count > 0:
caches['eventcount'].set("ingestcount", count, 3600 * 48)
if usercount > 0:
caches['eventcount'].set("useruploads", usercount, 3600 * 48)
indexScanner.close()
except:
pass ## user does not have PROV
def mthd(self):
import pysharkbite
#pysharkbite.LoggingConfiguration.enableTraceLogger()
tableOperations = super().getTableOperations()
if not tableOperations.exists(False):
print ("Creating table")
tableOperations.create(False)
else:
print ("Table already exists, so not creating it")
auths = pysharkbite.Authorizations()
auths.addAuthorization("blah1")
auths.addAuthorization("blah2")
securityOps = super().getConnector().securityOps()
securityOps.grantAuthorizations(auths,"root")
""" Add authorizations """
""" mutation.put("cf","cq","cv",1569786960) """
writer = tableOperations.createWriter(auths, 10)
mutation = pysharkbite.Mutation("row2");
mutation.put("cf","cq","blah1",1569786960, "value")
mutation.put("cf2","cq2","blah1",1569786960, "value2")
""" no value """
mutation.put("cf3","cq3","blah2",1569786960, "")
writer.addMutation( mutation )
writer.close()
print("written")
auths = pysharkbite.Authorizations()
auths.addAuthorization("blah1")
scanner = tableOperations.createScanner(auths, 2)
startKey = pysharkbite.Key()
endKey = pysharkbite.Key()
startKey.setRow("row")
endKey.setRow("row3")
# test single range
range = pysharkbite.Range("row2")
scanner.addRange( range )
resultset = scanner.getResultSet()
for keyvalue in resultset:
key = keyvalue.getKey()
assert( "row2" == key.getRow() )
value = keyvalue.getValue()
if "cf" == key.getColumnFamily():
assert( "value" == value.get() )
if ("cf2" == key.getColumnFamily() ):
assert( "value2" == value.get() )
if ("cf3" == key.getColumnFamily() ):
print("Unexpected column cf3")
sys.exit(1)
range = pysharkbite.Range("row1",True,"row1.5",True)
scanner.addRange( range )
resultset = scanner.getResultSet()
for keyvalue in resultset:
print("Unexpected result")
sys.exit(1)
# test single range
range = pysharkbite.Range("row",False,"row3",True)
scanner = tableOperations.createScanner(auths, 2)
scanner.addRange( range )
resultset = scanner.getResultSet()
count =0
for keyvalue in resultset:
key = keyvalue.getKey()
assert( "row2" == key.getRow() )
value = keyvalue.getValue()
if "cf" == key.getColumnFamily():
assert( "value" == value.get() )
if ("cf2" == key.getColumnFamily() ):
assert( "value2" == value.get() )
if ("cf3" == key.getColumnFamily() ):
print("Unexpected column cf3")
sys.exit(1)
count=count+1
if count <= 0:
print("Expected results")
sys.exit(1)
# test infinite range
range = pysharkbite.Range("",False,"row3",True)
scanner = tableOperations.createScanner(auths, 2)
scanner.addRange( range )
resultset = scanner.getResultSet()
count =0
for keyvalue in resultset:
key = keyvalue.getKey()
assert( "row2" == key.getRow() )
value = keyvalue.getValue()
if "cf" == key.getColumnFamily():
assert( "value" == value.get() )
if ("cf2" == key.getColumnFamily() ):
assert( "value2" == value.get() )
if ("cf3" == key.getColumnFamily() ):
print("Unexpected column cf3")
sys.exit(1)
count=count+1
if count <= 0:
print("Expected results")
sys.exit(1)
startKey = pysharkbite.Key("row3")
range = pysharkbite.Range(None,False,startKey,True)
scanner = tableOperations.createScanner(auths, 2)
scanner.addRange( range )
resultset = scanner.getResultSet()
count =0
for keyvalue in resultset:
key = keyvalue.getKey()
assert( "row2" == key.getRow() )
value = keyvalue.getValue()
if "cf" == key.getColumnFamily():
assert( "value" == value.get() )
if ("cf2" == key.getColumnFamily() ):
assert( "value2" == value.get() )
if ("cf3" == key.getColumnFamily() ):
print("Unexpected column cf3")
sys.exit(1)
count=count+1
if count <= 0:
print("Expected results")
sys.exit(1)
""" delete your table if user did not create temp """
tableOperations.remove()
def mthd(self):
import pysharkbite
tableOperations = super().getTableOperations()
if not tableOperations.exists(False):
print("Creating table")
if not tableOperations.create(False):
print("Could not create table")
else:
print("Table already exists, so not creating it")
auths = pysharkbite.Authorizations()
""" Add authorizations """
""" mutation.put("cf","cq","cv",1569786960) """
writer = tableOperations.createWriter(auths, 10)
mutation = pysharkbite.Mutation("sow2")
mutation.put("cf", "cq", "", 1569786960, "value")
mutation.put("cf2", "cq2", "", 1569786960, "value2")
""" no value """
mutation.put("cf3", "cq3", "", 1569786960, "")
writer.addMutation(mutation)
writer.close()
writer = tableOperations.createWriter(auths, 10)
rng = range(0, 1000)
for i in rng:
row = ("row%i" % (i + 5))
mutation = pysharkbite.Mutation(row)
mutation.put("cf", "cq", "", 1569786960, "value")
writer.addMutation(mutation)
writer.close()
print("written")
""" auths.addAuthorization("cv") """
scanner = tableOperations.createScanner(auths, 2)
startKey = pysharkbite.Key()
endKey = pysharkbite.Key()
startKey.setRow("sow")
endKey.setRow("sow3")
accumuloRange = pysharkbite.Range(startKey, True, endKey, False)
scanner.addRange(accumuloRange)
resultset = scanner.getResultSet()
for keyvalue in resultset:
key = keyvalue.getKey()
assert ("sow2" == key.getRow())
value = keyvalue.getValue()
if "cf" == key.getColumnFamily():
assert ("value" == value.get())
if ("cf2" == key.getColumnFamily()):
assert ("value2" == value.get())
if ("cf3" == key.getColumnFamily()):
assert ("" == value.get())
scanner = tableOperations.createScanner(auths, 2)
accumuloRange = pysharkbite.Range("row", True, "", False)
scanner.addRange(accumuloRange)
resultset = scanner.getResultSet()
count = 0
for keyvalue in resultset:
key = keyvalue.getKey()
count = count + 1
print("count is ", count)
assert (count == 1003) # all rows + sows
""" delete your table if user did not create temp """
tableOperations.remove()
def post(self, request, *args, **kwargs):
query = request.POST.get('query')
shard = request.POST.get('shard')
authstring = request.POST.get('auths')
datatype = request.POST.get('datatype')
uid = request.POST.get('uid')
originals = {}
news = {}
for key, value in request.POST.items():
if key == "query":
query = value
elif key.startswith("original"):
split = key.split(".")
originals[split[1]] = value
elif key == "shard" or key == "datatype" or key == "uid" or key == "auths":
pass
elif key == "csrfmiddlewaretoken":
pass
else:
news[key] = value
user = pysharkbite.AuthInfo(AccumuloCluster.objects.first().user,
AccumuloCluster.objects.first().password,
ZkInstance().get().getInstanceId())
connector = pysharkbite.AccumuloConnector(user, ZkInstance().get())
auths = pysharkbite.Authorizations()
#for auth in
if not authstring is None and len(authstring) > 0:
auths.addAuthorization(authstring)
table = AccumuloCluster.objects.first().dataTable
index_table = AccumuloCluster.objects.first().indexTable
table_operations = connector.tableOps(table)
index_table_ops = connector.tableOps(index_table)
writer = table_operations.createWriter(auths, 10)
indexWriter = index_table_ops.createWriter(auths, 5)
mutation = pysharkbite.Mutation(shard)
diff = 0
for key, value in news.items():
if news[key] != originals[key]:
import datetime
ts = int(datetime.datetime.now().timestamp()) * 1000
mutation.putDelete(datatype + "\x00" + uid,
key + "\x00" + originals[key], authstring,
ts)
ts = int(datetime.datetime.now().timestamp()) * 1000 + 100
mutation.put(datatype + "\x00" + uid, key + "\x00" + news[key],
authstring, ts)
originalIndexMutation = pysharkbite.Mutation(
originals[key].lower())
indexMutation = pysharkbite.Mutation(news[key].lower())
protobuf = Uid_pb2.List()
protobuf.COUNT = 1
protobuf.IGNORE = False
protobuf.UID.append(uid)
indexMutation.put(key, shard + "\x00" + datatype, authstring,
ts, protobuf.SerializeToString())
originalprotobuf = Uid_pb2.List()
indexWriter.addMutation(indexMutation)
originalprotobuf.COUNT = 1
originalprotobuf.IGNORE = False
originalprotobuf.REMOVEDUID.append(uid)
originalIndexMutation.put(key, shard + "\x00" + datatype,
authstring, ts,
originalprotobuf.SerializeToString())
indexWriter.addMutation(originalIndexMutation)
diff = diff + 1
else:
pass
if diff > 0:
writer.addMutation(mutation)
indexWriter.close()
writer.close()
authy = ""
url = "/search/?q=" + query
for auth in authstring.split("|"):
url = url + "&auths=" + auth
return HttpResponseRedirect(url)
def get(self, request, *args, **kwargs):
user = pysharkbite.AuthInfo(AccumuloCluster.objects.first().user,
AccumuloCluster.objects.first().password,
ZkInstance().get().getInstanceId())
connector = pysharkbite.AccumuloConnector(user, ZkInstance().get())
entry = request.GET.get('q')
selectedauths = request.GET.getlist('auths')
try:
skip = int(request.GET.get('s'))
except:
skip = 0
field = request.GET.get('f')
# try:
# LuceneToJexlQueryParser = jnius.autoclass('datawave.query.language.parser.jexl.LuceneToJexlQueryParser')
# luceneparser = LuceneToJexlQueryParser()
# node = luceneparser.parse(entry)
# jexl = node.getOriginalQuery()
# except:
# pass
indexLookup = 1
table = AccumuloCluster.objects.first().dataTable
index_table = AccumuloCluster.objects.first().indexTable
isProv = False
authlist = list()
auths = pysharkbite.Authorizations()
for auth in selectedauths:
if len(auth) > 0:
if auth == "PROV":
isProv = True
authlist.append(auth)
auths.addAuthorization(auth)
if isProv is True and len(authlist) == 1:
table = "provenance"
index_table = "provenanceIndex"
table_operations = connector.tableOps(table)
index_table_ops = connector.tableOps(index_table)
#auths = pysharkbite.Authorizations()
start = time.time()
indexLookupInformation = LookupInformation(index_table, auths,
index_table_ops)
shardLookupInformation = LookupInformation(table, auths,
table_operations)
wanted_items = list()
tree = parser.parse(entry)
tree = resolver(tree)
visitor = IndexLookup()
iterator = visitor.visit(tree)
if isinstance(iterator, RangeLookup):
rng = iterator
iterator = OrIterator()
iterator.addRange(rng)
docs = queue.SimpleQueue()
lookup(indexLookupInformation, shardLookupInformation, iterator, docs)
counts = 0
header = set()
while not docs.empty():
jsondoc = docs.get()
for key in jsondoc.keys():
if key != "ORIG_FILE" and key != "TERM_COUNT" and key != "RAW_FILE" and key != "shard" and key != "datatype" and key != "uid":
header.add(key)
wanted_items.append(jsondoc)
counts = counts + 1
nxt = ""
prv = ""
userAuths = set()
try:
auths = UserAuths.objects.get(name=request.user)
user_auths = auths.authorizations.all()
if not user_auths is None:
for authset in user_auths:
userAuths.add(authset)
except:
pass
s = "|"
authy = s.join(authlist)
context = {
'header': header,
'authstring': authy,
'selectedauths': selectedauths,
'results': wanted_items,
'time': (time.time() - start),
'prv': prv,
'nxt': nxt,
'field': field,
'admin': request.user.is_superuser,
'authenticated': True,
'userAuths': userAuths,
'query': entry
}
return render(request, 'search_results.html', context)
def mthd(self):
import pysharkbite
securityOps = super().getSecurityOperations()
auths = pysharkbite.Authorizations()
auths.addAuthorization("blah1")
auths.addAuthorization("blah2")
securityOps.grantAuthorizations(auths, "root")
tableOperations = super().getTableOperations()
tableOperations.create(False)
## validate that we DO see the permissions
""" Add authorizations """
""" mutation.put("cf","cq","cv",1569786960) """
with tableOperations.createWriter(auths, 10) as writer:
mutation = pysharkbite.Mutation("row2")
mutation.put("cf", "cq", "blah1", 1569786960, "value")
mutation.put("cf2", "cq2", "blah1", 1569786960, "value2")
""" no value """
mutation.put("cf3", "cq3", "blah2", 1569786960, "")
writer.addMutation(mutation)
mutation = pysharkbite.Mutation("row4")
mutation.put("cf", "cq", "blah1", 1569786960, "value")
mutation.put("cf2", "cq2", "blah1", 1569786960, "value2")
""" no value """
mutation.put("cf3", "cq3", "blah2", 1569786960, "")
writer.addMutation(mutation)
auths = pysharkbite.Authorizations()
auths.addAuthorization("blah1")
auths.addAuthorization("blah2")
count = 0
with tableOperations.createScanner(auths, 2).withRange(
pysharkbite.Range("row", True, "row3", False)) as scanner:
resultset = scanner.getResultSet()
for keyvalue in resultset:
count = count + 1
key = keyvalue.getKey()
assert ("row2" == key.getRow())
value = keyvalue.getValue()
if "cf" == key.getColumnFamily():
assert ("value" == value.get())
if ("cf2" == key.getColumnFamily()):
assert ("value2" == value.get())
with tableOperations.createScanner(auths, 2).withRange(
pysharkbite.Range("row4", True, "row5", False)) as scanner:
resultset = scanner.getResultSet()
for keyvalue in resultset:
count = count + 1
key = keyvalue.getKey()
print("%r %r" % (keyvalue.getKey(), str(keyvalue.getValue())))
assert ("row4" == key.getRow())
value = keyvalue.getValue()
if "cf" == key.getColumnFamily():
assert ("value" == value.get())
if ("cf2" == key.getColumnFamily()):
assert ("value2" == value.get())
""" delete your table if user did not create temp """
tableOperations.remove()
print("count is " + str(count))
assert (count == 6)
def run(*args):
admin_user = "******"
admin_password = "******"
auth = Auth()
auth.auth = "TST"
auth.save()
democonfig = IngestConfiguration()
democonfig.name = "democonfig"
democonfig.use_provenance = True
democonfig.post_location = "http://nifi:8181/contentListener"
democonfig.save()
instance = "uno"
zookeepers = "192.168.1.88:2181"
acc_user = "******"
acc_pass = "******"
for arg in args:
if arg.startswith("password"):
split = arg.split("=")
if len(split) == 2:
admin_password = split[1]
if arg.startswith("username"):
split = arg.split("=")
if len(split) == 2:
admin_user = split[1]
if arg.startswith("instance"):
split = arg.split("=")
if len(split) == 2:
admin_user = split[1]
if arg.startswith("zookeepers"):
split = arg.split("=")
if len(split) == 2:
zookeepers = split[1]
if arg.startswith("accuser"):
split = arg.split("=")
if len(split) == 2:
acc_user = split[1]
if arg.startswith("accpass"):
split = arg.split("=")
if len(split) == 2:
acc_pass = split[1]
acc = AccumuloCluster()
acc.instance = instance
acc.zookeeper = zookeepers
acc.user = acc_user
acc.password = acc_pass
acc.save()
import pysharkbite
while True:
try:
conf = pysharkbite.Configuration()
conf.set("FILE_SYSTEM_ROOT", "/accumulo")
zoo_keeper = pysharkbite.ZookeeperInstance(instance, zookeepers,
1000, conf)
user = pysharkbite.AuthInfo(acc_user, acc_pass,
zoo_keeper.getInstanceId())
connector = pysharkbite.AccumuloConnector(user, zoo_keeper)
security_ops = connector.securityOps()
auths = pysharkbite.Authorizations()
auths.addAuthorization("MTRCS")
auths.addAuthorization("PROV")
security_ops.grantAuthorizations(auths, acc_user)
table_ops = connector.tableOps("shard")
if not table_ops.exists(False):
table_ops.create(False)
table_ops = connector.tableOps("shardIndex")
if not table_ops.exists(False):
table_ops.create(False)
table_ops = connector.tableOps("shardReverse")
if not table_ops.exists(False):
table_ops.create(False)
table_ops = connector.tableOps("graph")
if not table_ops.exists(False):
table_ops.create(False)
table_ops = connector.tableOps("provenance")
if not table_ops.exists(False):
table_ops.create(False)
table_ops = connector.tableOps("provenanceIndex")
if not table_ops.exists(False):
table_ops.create(False)
table_ops = connector.tableOps("provenanceReverseIndex")
if not table_ops.exists(False):
table_ops.create(False)
break
except RuntimeError as e:
exc = str(e)
if exc.find("Instance Id does not exist") > 0:
pass
else:
break
def pouplateEventCountMetadata():
import pysharkbite
import time
conf = pysharkbite.Configuration()
conf.set("FILE_SYSTEM_ROOT", "/accumulo")
model = apps.get_model(app_label='query', model_name='AccumuloCluster')
accumulo_cluster = model.objects.first()
if accumulo_cluster is None:
return
zoo_keeper = pysharkbite.ZookeeperInstance(accumulo_cluster.instance,
accumulo_cluster.zookeeper,
1000, conf)
user = pysharkbite.AuthInfo("root", "secret", zoo_keeper.getInstanceId())
connector = pysharkbite.AccumuloConnector(user, zoo_keeper)
queryRanges = list()
#last 15 days
for dateinrange in getDateRange(-15):
shardbegin = dateinrange.strftime("%Y%m%d")
if caches['eventcount'].get(shardbegin) is None or caches[
'eventcount'].get(shardbegin) == 0:
queryRanges.append(shardbegin)
else:
pass # don't add to range
if len(queryRanges) > 0:
## all is cached
user = pysharkbite.AuthInfo("root", "secret",
zoo_keeper.getInstanceId())
connector = pysharkbite.AccumuloConnector(user, zoo_keeper)
indexTableOps = connector.tableOps("DatawaveMetrics")
auths = pysharkbite.Authorizations()
auths.addAuthorization("MTRCS")
indexScanner = indexTableOps.createScanner(auths, 100)
start = time.time()
for dt in queryRanges:
indexrange = pysharkbite.Range(dt, True, dt + "\uffff", False)
indexScanner.addRange(indexrange)
indexScanner.fetchColumn("EVENT_COUNT", "")
combinertxt = ""
## load the combiner from the file system and send it to accumulo
with open('metricscombiner.py', 'r') as file:
combinertxt = file.read()
combiner = pysharkbite.PythonIterator("MetadataCounter", combinertxt,
200)
indexScanner.addIterator(combiner)
indexSet = indexScanner.getResultSet()
counts = 0
mapping = {}
try:
for indexKeyValue in indexSet:
value = indexKeyValue.getValue()
key = indexKeyValue.getKey()
if key.getColumnFamily() == "EVENT_COUNT":
dt = key.getRow().split("_")[0]
if dt in mapping:
mapping[dt] += int(value.get())
else:
mapping[dt] = int(value.get())
arr = [None] * len(mapping.keys())
for field in mapping:
caches['eventcount'].set(field, str(mapping[field]), 3600 * 48)
except:
pass
def check():
model = apps.get_model(app_label='query', model_name='FileUpload')
objs = model.objects.filter(status="NEW")
for obj in objs:
if obj.status == "NEW":
import pysharkbite
conf = pysharkbite.Configuration()
conf.set("FILE_SYSTEM_ROOT", "/accumulo")
model = apps.get_model(app_label='query',
model_name='AccumuloCluster')
accumulo_cluster = model.objects.first()
print("Checking " + str(obj.uuid))
if accumulo_cluster is None:
return
print("Checking " + str(obj.uuid))
zk = pysharkbite.ZookeeperInstance(accumulo_cluster.instance,
accumulo_cluster.zookeeper,
1000, conf)
user = pysharkbite.AuthInfo("root", "secret", zk.getInstanceId())
connector = pysharkbite.AccumuloConnector(user, zk)
indexTableOps = connector.tableOps("provenanceIndex")
auths = pysharkbite.Authorizations()
auths.addAuthorization("PROV")
indexScanner = indexTableOps.createScanner(auths, 2)
indexrange = pysharkbite.Range(str(obj.uuid))
indexScanner.addRange(indexrange)
indexSet = indexScanner.getResultSet()
rangelist = list()
provops = connector.tableOps("provenance")
scanner = provops.createScanner(auths, 10)
for indexKeyValue in indexSet:
value = indexKeyValue.getValue()
protobuf = Uid_pb2.List()
protobuf.ParseFromString(value.get().encode())
for uidvalue in protobuf.UID:
shard = indexKeyValue.getKey().getColumnQualifier().split(
"\u0000")[0]
datatype = indexKeyValue.getKey().getColumnQualifier(
).split("\u0000")[1]
startKey = pysharkbite.Key()
stopKey = pysharkbite.Key()
startKey.setRow(shard)
stopKey.setRow(shard)
startKey.setColumnFamily(datatype + "\x00" + uidvalue)
stopKey.setColumnFamily(datatype + "\x00" + uidvalue +
"\xff")
rangelist.append(
pysharkbite.Range(startKey, True, stopKey, False))
scanner = provops.createScanner(auths, 10)
scanner.addRange(
pysharkbite.Range(startKey, True, stopKey, False))
resultset = scanner.getResultSet()
for keyvalue in resultset:
key = keyvalue.getKey()
value = keyvalue.getValue()
eventid = key.getColumnFamily().split("\u0000")[1]
fieldname = key.getColumnQualifier().split("\u0000")[0]
fieldvalue = key.getColumnQualifier().split(
"\u0000")[1]
if (fieldname == "EVENTTYPE"):
if fieldvalue == "DROP":
obj.status = "COMPLETE"
obj.save()
break
scanner.close()
indexScanner.close()
def run_edge_query(query_id):
model = apps.get_model(app_label='query', model_name='EdgeQuery')
objs = model.objects.filter(query_id=query_id)
for obj in objs:
obj.running = True
obj.save()
import pysharkbite
conf = pysharkbite.Configuration()
conf.set("FILE_SYSTEM_ROOT", "/accumulo")
model = apps.get_model(app_label='query', model_name='AccumuloCluster')
accumulo_cluster = model.objects.first()
if accumulo_cluster is None:
return
zk = pysharkbite.ZookeeperInstance(accumulo_cluster.instance,
accumulo_cluster.zookeeper, 1000,
conf)
user = pysharkbite.AuthInfo("root", "secret", zk.getInstanceId())
connector = pysharkbite.AccumuloConnector(user, zk)
auths = pysharkbite.Authorizations()
if obj.auths:
for auth in obj.auths.split(","):
auths.addAuthorization(auth)
sres_model = apps.get_model(app_label='query', model_name='ScanResult')
res_model = apps.get_model(app_label='query', model_name='Result')
sr = sres_model.objects.filter(query_id=obj.query_id).first()
if not sr:
print("No scan result, returning")
return
print("here")
graphTableOps = connector.tableOps("graph")
scanner = graphTableOps.createScanner(auths, 10)
range = pysharkbite.Range(obj.query, True,
obj.query + "\uffff" + "\uffff",
False) ## for now the range should be this
scanner.addRange(range)
resultset = scanner.getResultSet()
count = 0
try:
for indexKeyValue in resultset:
value = "0"
## row will be the to
## direction will be the cf
to_value = ""
direction = "one"
try:
protobuf = EdgeData_pb2.EdgeValue()
protobuf.ParseFromString(
indexKeyValue.getValue().get_bytes())
value = str(protobuf.count) + "/" + protobuf.uuid_string
to_value = indexKeyValue.getKey().getRow().split(
"\u0000")[1]
direction = indexKeyValue.getKey().getColumnFamily().split(
"/")[1]
direction_split = direction.split("-")
if len(direction_split
) != 2 or direction_split[0] == direction_split[1]:
continue
except Exception as e:
print(e)
continue
except:
continue
scanresult = res_model.objects.create(
scanResult=sr,
value=value,
row=to_value,
cf=direction,
cq=indexKeyValue.getKey().getColumnQualifier())
scanresult.save()
count = count + 1
if count > 1000:
break
sr.is_finished = True
sr.save()
scanner.close()
except Exception as e:
print(e)
except:
print("An error occurred")
pass ## user does not have PROV
obj.running = False
obj.finished = True
obj.save()
def mthd(self):
import pysharkbite
tableOperations = super().getTableOperations()
## validate that table is removed
try:
if not tableOperations.exists(False):
print("Creating table")
if not tableOperations.create(False):
print("Could not create table")
else:
print("Table already exists, so not creating it")
auths = pysharkbite.Authorizations()
""" Add authorizations """
""" mutation.put("cf","cq","cv",1569786960) """
tableOperations.remove()
time.sleep(1)
writer = tableOperations.createWriter(auths, 10)
mutation = pysharkbite.Mutation("sow2")
mutation.put("cf", "cq", "", 1569786960, "value")
mutation.put("cf2", "cq2", "", 1569786960, "value2")
""" no value """
mutation.put("cf3", "cq3", "", 1569786960, "")
writer.addMutation(mutation)
writer.close()
writer = tableOperations.createWriter(auths, 10)
rng = range(0, 1000)
for i in rng:
row = ("row%i" % (i + 5))
mutation = pysharkbite.Mutation(row)
mutation.put("cf", "cq", "", 1569786960, "value")
writer.addMutation(mutation)
writer.close()
print("Table not removed")
sys.exit(1)
except (RuntimeError, TypeError, NameError):
print("Table successfully removed")
# this is okay
try:
testzk = pysharkbite.ZookeeperInstance(None, None, 1000, None)
print("Table not removed")
sys.exit(1)
except (RuntimeError, TypeError, NameError):
print("Caught expected error")
# this is okay
try:
writer = tableOperations.createWriter(None, 10)
print("Expected error passing None")
sys.exit(1)
except (RuntimeError, TypeError, NameError):
print("Caught expected error")
# this is okay
tableOperations.remove()
user = pysharkbite.AuthInfo(args.username, password, zk.getInstanceId())
try:
connector = pysharkbite.AccumuloConnector(user, zk)
tableOperations = connector.tableOps(table)
if not tableOperations.exists(False):
print ("Creating table " + table)
tableOperations.create(False)
else:
print (table + " already exists, so not creating it")
auths = pysharkbite.Authorizations()
""" Add authorizations """
""" mutation.put("cf","cq","cv",1569786960) """
writer = tableOperations.createWriter(auths, 10)
mutation = pysharkbite.Mutation("row2");
mutation.put("cf","cq","",1569786960, "value")
mutation.put("cf2","cq2","",1569786960, "value2")
""" no value """
mutation.put("cf3","cq3","",1569786960, "")
writer.addMutation( mutation )
def mthd(self):
import pysharkbite
tableOperations = super().getTableOperations()
if not tableOperations.exists(False):
print("Creating table")
if not tableOperations.create(False):
print("Could not create table")
else:
print("Table already exists, so not creating it")
auths = pysharkbite.Authorizations()
""" Add authorizations """
""" mutation.put("cf","cq","cv",1569786960) """
writer = tableOperations.createWriter(auths, 10)
mutation = pysharkbite.Mutation("sow2")
mutation.put("cf", "cq", "", 1569786960, "value")
mutation.put("cf2", "cq", "", 1569786960, "value")
""" no value """
mutation.put("cf3", "cq", "", 1569786960, "value")
writer.addMutation(mutation)
writer.close()
writer = tableOperations.createWriter(auths, 10)
rng = range(0, 1000)
for i in rng:
row = ("row%i" % (i + 5))
mutation = pysharkbite.Mutation(row)
mutation.put("cf", "cq", "", 1569786960, "value")
writer.addMutation(mutation)
writer.close()
print("written")
""" auths.addAuthorization("cv") """
scanner = tableOperations.createScanner(auths, 2)
accumuloRange = pysharkbite.Range("sow", True, "sow3", False)
scanner.addRange(accumuloRange)
iterator = pysharkbite.PythonIterator("PythonIterator", 100)
iterator = iterator.onNext(
"lambda x : KeyValue( Key( x.getKey().getRow(), 'new cf', x.getKey().getColumnQualifier()), Value()) "
)
scanner.addIterator(iterator)
resultset = scanner.getResultSet()
for keyvalue in resultset:
key = keyvalue.getKey()
assert ("sow2" == key.getRow())
value = keyvalue.getValue()
if "cf" == key.getColumnFamily():
sys.exit(154)
if "new cf" == key.getColumnFamily():
assert ("" == value.get())
scanner = tableOperations.createScanner(auths, 2)
accumuloRange = pysharkbite.Range("sow", True, "sow3", False)
scanner.addRange(accumuloRange)
iterator = pysharkbite.PythonIterator("PythonIterator", 100)
iterator = iterator.onNext(
"lambda x : Key( x.getKey().getRow(), x.getKey().getColumnFamily(), 'new cq') "
)
scanner.addIterator(iterator)
resultset = scanner.getResultSet()
for keyvalue in resultset:
key = keyvalue.getKey()
assert ("sow2" == key.getRow())
value = keyvalue.getValue()
if "cq" == key.getColumnQualifier():
sys.exit(154)
""" delete your table if user did not create temp """
tableOperations.remove()
