def lookupRange(lookupInformation: LookupInformation, rng: RangeLookup,
output) -> None:
index_table_ops = lookupInformation.getTableOps()
indexScanner = index_table_ops.createScanner(lookupInformation.getAuths(),
1)
fv = rng.getValue()
if fv.endswith("*"):
base = fv.replace('*', '')
indexrange = pysharkbite.Range(base, True, base + "\uffff", False)
else:
indexrange = pysharkbite.Range(rng.getValue().lower())
indexScanner.addRange(indexrange)
if not rng.getField() is None:
indexScanner.fetchColumn(rng.getField().upper(), "")
indexSet = indexScanner.getResultSet()
try:
for indexKeyValue in indexSet:
value = indexKeyValue.getValue()
protobuf = Uid_pb2.List()
protobuf.ParseFromString(value.get().encode())
for uidvalue in protobuf.UID:
shard = indexKeyValue.getKey().getColumnQualifier().split(
"\u0000")[0]
datatype = indexKeyValue.getKey().getColumnQualifier().split(
"\u0000")[1]
output.put(Range(datatype, shard, uidvalue))
except:
pass
indexScanner.close()
def __iter__(self):
if self._indexLookupInformation is None:
return self
indexTableOps = lookupInformation.getTableOps()
self._rngs = [None] * len(rangeQueue)
self._scnrs = [None] * len(rangeQueue)
self._iters = [None] * len(rangeQueue)
count=0
for rng in rangeQueue:
if isinstance(rng,LookupIterator):
self._iters[count]=rng
self._scnrs=None
else:
scnrs[count] = indexTableOps.createScanner(lookupInformation.getAuths(),1)
indexrange = pysharkbite.Range(rng.getValue())
scnrs[count].addRange(indexrange)
if not rng.getField() is None:
scnrs[count].fetchColumn(rng.getField().upper(),"")
self._iters=ForwardIterator(scnrs[count].getResultSet())
count=count+1
self._iter=combineScanners(self._iters)
return self
def get(self, request, *args, **kwargs):
shard = request.GET.get('shard')
datatype = request.GET.get('datatype')
uid = request.GET.get('uid')
query = request.GET.get('query')
authstring = request.GET.get('auths')
url = "/search/?q=" + query
auths = pysharkbite.Authorizations()
if not authstring is None and len(authstring) > 0:
auths.addAuthorization(authstring)
user = pysharkbite.AuthInfo(AccumuloCluster.objects.first().user,AccumuloCluster.objects.first().password, ZkInstance().get().getInstanceId())
connector = pysharkbite.AccumuloConnector(user, ZkInstance().get())
tableOps = connector.tableOps(AccumuloCluster.objects.first().dataTable)
scanner = tableOps.createScanner(auths,1)
startKey = pysharkbite.Key(row=shard)
endKey = pysharkbite.Key(row=shard)
docid = datatype + "\x00" + uid;
startKey.setColumnFamily(docid)
endKey.setColumnFamily(docid + "\xff")
rng = pysharkbite.Range(startKey,True,endKey,True)
scanner.addRange(rng)
writer = tableOps.createWriter(auths,10)
deletes = pysharkbite.Mutation(shard)
for keyValue in scanner.getResultSet():
key = keyValue.getKey()
deletes.putDelete( key.getColumnFamily(), key.getColumnQualifier(), key.getColumnVisibility(), key.getTimestamp())
## scan for the original document
writer.addMutation(deletes)
writer.close()
for auth in authstring.split("|"):
url = url + "&auths=" + auth
return HttpResponseRedirect(url)
def populateMetadata():
import pysharkbite
conf = pysharkbite.Configuration()
conf.set("FILE_SYSTEM_ROOT", "/accumulo")
model = apps.get_model(app_label='query', model_name='AccumuloCluster')
accumulo_cluster = model.objects.first()
if accumulo_cluster is None:
return
zk = pysharkbite.ZookeeperInstance(accumulo_cluster.instance,
accumulo_cluster.zookeeper, 1000, conf)
user = pysharkbite.AuthInfo("root", "secret", zk.getInstanceId())
connector = pysharkbite.AccumuloConnector(user, zk)
indexTableOps = connector.tableOps("DatawaveMetadata")
auths = pysharkbite.Authorizations()
indexScanner = indexTableOps.createScanner(auths, 100)
indexrange = pysharkbite.Range()
indexScanner.addRange(indexrange)
indexScanner.fetchColumn("f", "")
combinertxt = ""
## load the combiner from the file system and send it to accumulo
with open('countgatherer.py', 'r') as file:
combinertxt = file.read()
combiner = pysharkbite.PythonIterator("MetadataCounter", combinertxt, 200)
indexScanner.addIterator(combiner)
indexSet = indexScanner.getResultSet()
import json
counts = 0
mapping = {}
for indexKeyValue in indexSet:
value = indexKeyValue.getValue()
key = indexKeyValue.getKey()
if key.getColumnFamily() == "f":
day = key.getColumnQualifier().split("\u0000")[1]
dt = key.getColumnQualifier().split("\u0000")[0]
if day in mapping:
if key.getRow() in mapping[day]:
try:
mapping[day][key.getRow()] += int(value.get())
except:
pass
else:
try:
mapping[day][key.getRow()] = int(value.get())
except:
pass
else:
mapping[day] = {}
try:
mapping[day][key.getRow()] = int(value.get())
except:
pass
caches['metadata'].set("field", json.dumps(mapping), 3600 * 48)
return json.dumps(mapping)
def mthd(self):
import pysharkbite
tableOperations = super().getTableOperations()
if not tableOperations.exists(False):
print("Creating table")
if not tableOperations.create(False):
print("Could not create table")
else:
tableOperations.remove()
time.sleep(2)
tableOperations.create(False)
print("Table already exists, so not creating it")
tableOperations = super().newtableOperations()
splits = {"row", "row5", "row9", "sow"}
tableOperations.addSplits(splits)
auths = pysharkbite.Authorizations()
""" Add authorizations """
""" mutation.put("cf","cq","cv",1569786960) """
writer = tableOperations.createWriter(auths, 10)
rng = range(0, 500)
for i in rng:
row = ("row%i" % (i + 5))
mutation = pysharkbite.Mutation(row)
rng2 = range(0, 100)
for j in rng:
mutation.put("cf%i" % j, "cq", "", 1569786960, "value")
writer.addMutation(mutation)
print("wrote %i entries" % ((i + 1) * (j + 1)))
writer.close()
tableOperations.compact("a", "s", True)
print("written")
scanner = tableOperations.createScanner(auths, 2)
time.sleep(1)
accumuloRange = pysharkbite.Range("a", True, "sow", False)
scanner.addRange(accumuloRange)
resultset = scanner.getResultSet()
count = 0
for keyvalue in resultset:
key = keyvalue.getKey()
count = count + 1
print("count is ", count)
""" delete your table if user did not create temp """
def lookupRanges(lookupInformation: LookupInformation, ranges: list,
output) -> None:
index_table_ops = lookupInformation.getTableOps()
rngs = [None] * len(ranges)
scnrs = [None] * len(ranges)
itrs = [None] * len(ranges)
count = 0
for rng in ranges:
if isinstance(rng, LookupIterator):
itrs[count] = ForwardIterator(rng)
scnrs[count] = None
count = count + 1
else:
scnrs[count] = index_table_ops.createScanner(
lookupInformation.getAuths(), 1)
fv = rng.getValue()
if fv.endswith("*"):
base = fv.replace('*', '')
indexrange = pysharkbite.Range(base, True, base + "\uffff",
False)
else:
indexrange = pysharkbite.Range(fv.lower())
scnrs[count].addRange(indexrange)
if not rng.getField() is None:
scnrs[count].fetchColumn(rng.getField().upper(), "")
itrs[count] = scnrs[count].getResultSet()
count = count + 1
try:
for indexKeyValue in intersect_sets(itrs):
output.put(indexKeyValue)
except StopIteration:
pass
except:
traceback.print_exc()
raise
for scnr in scnrs:
if not scnr is None:
scnr.close()
def get_uploads():
model = apps.get_model(app_label='query', model_name='FileUpload')
objs = model.objects.filter(status="NEW")
haveNew = False
for obj in objs:
if obj.status == "NEW":
caches['eventcount'].set("ingestcomplete", 95)
haveNew = True
if not haveNew:
caches['eventcount'].set("ingestcomplete", 100)
return
import pysharkbite
conf = pysharkbite.Configuration()
conf.set("FILE_SYSTEM_ROOT", "/accumulo")
model = apps.get_model(app_label='query', model_name='AccumuloCluster')
accumulo_cluster = model.objects.first()
if accumulo_cluster is None:
return
zk = pysharkbite.ZookeeperInstance(accumulo_cluster.instance,
accumulo_cluster.zookeeper, 1000, conf)
user = pysharkbite.AuthInfo("root", "secret", zk.getInstanceId())
connector = pysharkbite.AccumuloConnector(user, zk)
auths = pysharkbite.Authorizations()
auths.addAuthorization("PROV")
indexTableOps = connector.tableOps("provenanceIndex")
indexScanner = indexTableOps.createScanner(auths, 10)
indexrange = pysharkbite.Range()
indexScanner.addRange(indexrange)
indexScanner.fetchColumn("CONTENTURI", "")
indexScanner.fetchColumn("TRANSITURI", "")
indexSet = indexScanner.getResultSet()
count = 0
usercount = 0
try:
for indexKeyValue in indexSet:
if indexKeyValue.getKey().getColumnFamily() == "CONTENTURI":
count = count + 1
else:
usercount = usercount + 1
if count > 0:
caches['eventcount'].set("ingestcount", count, 3600 * 48)
if usercount > 0:
caches['eventcount'].set("useruploads", usercount, 3600 * 48)
indexScanner.close()
except:
pass ## user does not have PROV
def mthd(self):
import pysharkbite
pysharkbite.LoggingConfiguration.enableTraceLogger()
self.writerfile()
readrfile = pysharkbite.RFileOperations.sequentialRead("blah.rf")
accrange = pysharkbite.Range()
seekable = pysharkbite.Seekable(accrange)
readrfile.seek( seekable)
i = 0
while readrfile.hasNext():
assert( str("row" + str(i)) == readrfile.getTopKey().getRow() )
i=i+1
readrfile.next()
assert(i==1000)
def check():
model = apps.get_model(app_label='query', model_name='FileUpload')
objs = model.objects.filter(status="NEW")
for obj in objs:
if obj.status == "NEW":
import pysharkbite
conf = pysharkbite.Configuration()
conf.set("FILE_SYSTEM_ROOT", "/accumulo")
model = apps.get_model(app_label='query',
model_name='AccumuloCluster')
accumulo_cluster = model.objects.first()
print("Checking " + str(obj.uuid))
if accumulo_cluster is None:
return
print("Checking " + str(obj.uuid))
zk = pysharkbite.ZookeeperInstance(accumulo_cluster.instance,
accumulo_cluster.zookeeper,
1000, conf)
user = pysharkbite.AuthInfo("root", "secret", zk.getInstanceId())
connector = pysharkbite.AccumuloConnector(user, zk)
indexTableOps = connector.tableOps("provenanceIndex")
auths = pysharkbite.Authorizations()
auths.addAuthorization("PROV")
indexScanner = indexTableOps.createScanner(auths, 2)
indexrange = pysharkbite.Range(str(obj.uuid))
indexScanner.addRange(indexrange)
indexSet = indexScanner.getResultSet()
rangelist = list()
provops = connector.tableOps("provenance")
scanner = provops.createScanner(auths, 10)
for indexKeyValue in indexSet:
value = indexKeyValue.getValue()
protobuf = Uid_pb2.List()
protobuf.ParseFromString(value.get().encode())
for uidvalue in protobuf.UID:
shard = indexKeyValue.getKey().getColumnQualifier().split(
"\u0000")[0]
datatype = indexKeyValue.getKey().getColumnQualifier(
).split("\u0000")[1]
startKey = pysharkbite.Key()
stopKey = pysharkbite.Key()
startKey.setRow(shard)
stopKey.setRow(shard)
startKey.setColumnFamily(datatype + "\x00" + uidvalue)
stopKey.setColumnFamily(datatype + "\x00" + uidvalue +
"\xff")
rangelist.append(
pysharkbite.Range(startKey, True, stopKey, False))
scanner = provops.createScanner(auths, 10)
scanner.addRange(
pysharkbite.Range(startKey, True, stopKey, False))
resultset = scanner.getResultSet()
for keyvalue in resultset:
key = keyvalue.getKey()
value = keyvalue.getValue()
eventid = key.getColumnFamily().split("\u0000")[1]
fieldname = key.getColumnQualifier().split("\u0000")[0]
fieldvalue = key.getColumnQualifier().split(
"\u0000")[1]
if (fieldname == "EVENTTYPE"):
if fieldvalue == "DROP":
obj.status = "COMPLETE"
obj.save()
break
scanner.close()
indexScanner.close()
def mthd(self):
import pysharkbite
securityOps = super().getSecurityOperations()
auths = pysharkbite.Authorizations()
auths.addAuthorization("blah1")
auths.addAuthorization("blah2")
securityOps.grantAuthorizations(auths, "root")
tableOperations = super().getTableOperations()
tableOperations.create(False)
## validate that we DO see the permissions
""" Add authorizations """
""" mutation.put("cf","cq","cv",1569786960) """
with tableOperations.createWriter(auths, 10) as writer:
mutation = pysharkbite.Mutation("row2")
mutation.put("cf", "cq", "blah1", 1569786960, "value")
mutation.put("cf2", "cq2", "blah1", 1569786960, "value2")
""" no value """
mutation.put("cf3", "cq3", "blah2", 1569786960, "")
writer.addMutation(mutation)
mutation = pysharkbite.Mutation("row4")
mutation.put("cf", "cq", "blah1", 1569786960, "value")
mutation.put("cf2", "cq2", "blah1", 1569786960, "value2")
""" no value """
mutation.put("cf3", "cq3", "blah2", 1569786960, "")
writer.addMutation(mutation)
auths = pysharkbite.Authorizations()
auths.addAuthorization("blah1")
auths.addAuthorization("blah2")
count = 0
with tableOperations.createScanner(auths, 2).withRange(
pysharkbite.Range("row", True, "row3", False)) as scanner:
resultset = scanner.getResultSet()
for keyvalue in resultset:
count = count + 1
key = keyvalue.getKey()
assert ("row2" == key.getRow())
value = keyvalue.getValue()
if "cf" == key.getColumnFamily():
assert ("value" == value.get())
if ("cf2" == key.getColumnFamily()):
assert ("value2" == value.get())
with tableOperations.createScanner(auths, 2).withRange(
pysharkbite.Range("row4", True, "row5", False)) as scanner:
resultset = scanner.getResultSet()
for keyvalue in resultset:
count = count + 1
key = keyvalue.getKey()
print("%r %r" % (keyvalue.getKey(), str(keyvalue.getValue())))
assert ("row4" == key.getRow())
value = keyvalue.getValue()
if "cf" == key.getColumnFamily():
assert ("value" == value.get())
if ("cf2" == key.getColumnFamily()):
assert ("value2" == value.get())
""" delete your table if user did not create temp """
tableOperations.remove()
print("count is " + str(count))
assert (count == 6)
def mthd(self):
import pysharkbite
tableOperations = super().getTableOperations()
if not tableOperations.exists(False):
print("Creating table")
if not tableOperations.create(False):
print("Could not create table")
else:
print("Table already exists, so not creating it")
auths = pysharkbite.Authorizations()
""" Add authorizations """
""" mutation.put("cf","cq","cv",1569786960) """
writer = tableOperations.createWriter(auths, 10)
mutation = pysharkbite.Mutation("sow2")
mutation.put("cf", "cq", "", 1569786960, "value")
mutation.put("cf2", "cq2", "", 1569786960, "value2")
""" no value """
mutation.put("cf3", "cq3", "", 1569786960, "")
writer.addMutation(mutation)
writer.close()
writer = tableOperations.createWriter(auths, 10)
rng = range(0, 1000)
for i in rng:
row = ("row%i" % (i + 5))
mutation = pysharkbite.Mutation(row)
mutation.put("cf", "cq", "", 1569786960, "value")
writer.addMutation(mutation)
writer.close()
print("written")
""" auths.addAuthorization("cv") """
scanner = tableOperations.createScanner(auths, 2)
startKey = pysharkbite.Key()
endKey = pysharkbite.Key()
startKey.setRow("sow")
endKey.setRow("sow3")
accumuloRange = pysharkbite.Range(startKey, True, endKey, False)
scanner.addRange(accumuloRange)
resultset = scanner.getResultSet()
for keyvalue in resultset:
key = keyvalue.getKey()
assert ("sow2" == key.getRow())
value = keyvalue.getValue()
if "cf" == key.getColumnFamily():
assert ("value" == value.get())
if ("cf2" == key.getColumnFamily()):
assert ("value2" == value.get())
if ("cf3" == key.getColumnFamily()):
assert ("" == value.get())
scanner = tableOperations.createScanner(auths, 2)
accumuloRange = pysharkbite.Range("row", True, "", False)
scanner.addRange(accumuloRange)
resultset = scanner.getResultSet()
count = 0
for keyvalue in resultset:
key = keyvalue.getKey()
count = count + 1
print("count is ", count)
assert (count == 1003) # all rows + sows
""" delete your table if user did not create temp """
tableOperations.remove()
args = parser.parse_args()
file = args.file
import pysharkbite
try:
rfile = pysharkbite.RFileOperations.open(file)
# list of column families
cf = list()
## range
rng = pysharkbite.Range()
## seekable is an object that replaces the function call
## of seek(range,cf,boolean). This exists because the extension point
## in the C++ Api allows varying seekable objects that change how seeks occur
seek = pysharkbite.Seekable(rng, cf, False)
rfile.seek(seek)
## print the row
while rfile.hasNext():
kv = rfile.getTop()
print(kv.getKey())
rfile.next()
except RuntimeError as e:
def getDocuments(cancellationtoken: CancellationToken, name: int,
lookupInformation: LookupInformation,
input: queue.SimpleQueue, outputQueue: queue.SimpleQueue):
count = 0
while cancellationtoken.running():
docInfo = None
try:
try:
if input.empty():
pass
else:
docInfo = input.get(timeout=1)
except:
pass
# Handle empty queue here
if not docInfo is None:
tableOps = lookupInformation.getTableOps()
scanner = tableOps.createScanner(lookupInformation.getAuths(),
5)
startKey = pysharkbite.Key()
endKey = pysharkbite.Key()
startKey.setRow(docInfo.getShard())
docid = docInfo.getDataType() + "\x00" + docInfo.getDocId()
startKey.setColumnFamily(docid)
endKey.setRow(docInfo.getShard())
endKey.setColumnFamily(docid + "\xff")
print("Searching for " + docInfo.getShard())
rng = pysharkbite.Range(startKey, True, endKey, True)
scanner.addRange(rng)
rangecount = 1
while rangecount < 10:
try:
docInfo = input.get(False)
startKey = pysharkbite.Key()
endKey = pysharkbite.Key()
startKey.setRow(docInfo.getShard())
docid = docInfo.getDataType(
) + "\x00" + docInfo.getDocId()
startKey.setColumnFamily(docid)
endKey.setRow(docInfo.getShard())
endKey.setColumnFamily(docid + "\xff")
rng = pysharkbite.Range(startKey, True, endKey, True)
print("Searching for " + docInfo.getShard())
scanner.addRange(rng)
rangecount = rangecount + 1
except:
rangecount = 11
with open('jsoncombiner.py', 'r') as file:
combinertxt = file.read()
combiner = pysharkbite.PythonIterator(
"PythonCombiner", combinertxt, 100)
scanner.addIterator(combiner)
try:
count = count + scanDoc(scanner, outputQueue)
except:
pass
else:
time.sleep(0.5)
except:
e = sys.exc_info()[0]
return True
def pouplateEventCountMetadata():
import pysharkbite
import time
conf = pysharkbite.Configuration()
conf.set("FILE_SYSTEM_ROOT", "/accumulo")
model = apps.get_model(app_label='query', model_name='AccumuloCluster')
accumulo_cluster = model.objects.first()
if accumulo_cluster is None:
return
zoo_keeper = pysharkbite.ZookeeperInstance(accumulo_cluster.instance,
accumulo_cluster.zookeeper,
1000, conf)
user = pysharkbite.AuthInfo("root", "secret", zoo_keeper.getInstanceId())
connector = pysharkbite.AccumuloConnector(user, zoo_keeper)
queryRanges = list()
#last 15 days
for dateinrange in getDateRange(-15):
shardbegin = dateinrange.strftime("%Y%m%d")
if caches['eventcount'].get(shardbegin) is None or caches[
'eventcount'].get(shardbegin) == 0:
queryRanges.append(shardbegin)
else:
pass # don't add to range
if len(queryRanges) > 0:
## all is cached
user = pysharkbite.AuthInfo("root", "secret",
zoo_keeper.getInstanceId())
connector = pysharkbite.AccumuloConnector(user, zoo_keeper)
indexTableOps = connector.tableOps("DatawaveMetrics")
auths = pysharkbite.Authorizations()
auths.addAuthorization("MTRCS")
indexScanner = indexTableOps.createScanner(auths, 100)
start = time.time()
for dt in queryRanges:
indexrange = pysharkbite.Range(dt, True, dt + "\uffff", False)
indexScanner.addRange(indexrange)
indexScanner.fetchColumn("EVENT_COUNT", "")
combinertxt = ""
## load the combiner from the file system and send it to accumulo
with open('metricscombiner.py', 'r') as file:
combinertxt = file.read()
combiner = pysharkbite.PythonIterator("MetadataCounter", combinertxt,
200)
indexScanner.addIterator(combiner)
indexSet = indexScanner.getResultSet()
counts = 0
mapping = {}
try:
for indexKeyValue in indexSet:
value = indexKeyValue.getValue()
key = indexKeyValue.getKey()
if key.getColumnFamily() == "EVENT_COUNT":
dt = key.getRow().split("_")[0]
if dt in mapping:
mapping[dt] += int(value.get())
else:
mapping[dt] = int(value.get())
arr = [None] * len(mapping.keys())
for field in mapping:
caches['eventcount'].set(field, str(mapping[field]), 3600 * 48)
except:
pass
def mthd(self):
import pysharkbite
tableOperations = super().getTableOperations()
if not tableOperations.exists(False):
print("Creating table")
if not tableOperations.create(False):
print("Could not create table")
else:
print("Table already exists, so not creating it")
auths = pysharkbite.Authorizations()
""" Add authorizations """
""" mutation.put("cf","cq","cv",1569786960) """
writer = tableOperations.createWriter(auths, 10)
mutation = pysharkbite.Mutation("sow2")
mutation.put("cf", "cq", "", 1569786960, "value")
mutation.put("cf2", "cq", "", 1569786960, "value")
""" no value """
mutation.put("cf3", "cq", "", 1569786960, "value")
writer.addMutation(mutation)
writer.close()
writer = tableOperations.createWriter(auths, 10)
rng = range(0, 1000)
for i in rng:
row = ("row%i" % (i + 5))
mutation = pysharkbite.Mutation(row)
mutation.put("cf", "cq", "", 1569786960, "value")
writer.addMutation(mutation)
writer.close()
print("written")
""" auths.addAuthorization("cv") """
scanner = tableOperations.createScanner(auths, 2)
accumuloRange = pysharkbite.Range("sow", True, "sow3", False)
scanner.addRange(accumuloRange)
iterator = pysharkbite.PythonIterator("PythonIterator", 100)
iterator = iterator.onNext(
"lambda x : KeyValue( Key( x.getKey().getRow(), 'new cf', x.getKey().getColumnQualifier()), Value()) "
)
scanner.addIterator(iterator)
resultset = scanner.getResultSet()
for keyvalue in resultset:
key = keyvalue.getKey()
assert ("sow2" == key.getRow())
value = keyvalue.getValue()
if "cf" == key.getColumnFamily():
sys.exit(154)
if "new cf" == key.getColumnFamily():
assert ("" == value.get())
scanner = tableOperations.createScanner(auths, 2)
accumuloRange = pysharkbite.Range("sow", True, "sow3", False)
scanner.addRange(accumuloRange)
iterator = pysharkbite.PythonIterator("PythonIterator", 100)
iterator = iterator.onNext(
"lambda x : Key( x.getKey().getRow(), x.getKey().getColumnFamily(), 'new cq') "
)
scanner.addIterator(iterator)
resultset = scanner.getResultSet()
for keyvalue in resultset:
key = keyvalue.getKey()
assert ("sow2" == key.getRow())
value = keyvalue.getValue()
if "cq" == key.getColumnQualifier():
sys.exit(154)
""" delete your table if user did not create temp """
tableOperations.remove()
def run_edge_query(query_id):
model = apps.get_model(app_label='query', model_name='EdgeQuery')
objs = model.objects.filter(query_id=query_id)
for obj in objs:
obj.running = True
obj.save()
import pysharkbite
conf = pysharkbite.Configuration()
conf.set("FILE_SYSTEM_ROOT", "/accumulo")
model = apps.get_model(app_label='query', model_name='AccumuloCluster')
accumulo_cluster = model.objects.first()
if accumulo_cluster is None:
return
zk = pysharkbite.ZookeeperInstance(accumulo_cluster.instance,
accumulo_cluster.zookeeper, 1000,
conf)
user = pysharkbite.AuthInfo("root", "secret", zk.getInstanceId())
connector = pysharkbite.AccumuloConnector(user, zk)
auths = pysharkbite.Authorizations()
if obj.auths:
for auth in obj.auths.split(","):
auths.addAuthorization(auth)
sres_model = apps.get_model(app_label='query', model_name='ScanResult')
res_model = apps.get_model(app_label='query', model_name='Result')
sr = sres_model.objects.filter(query_id=obj.query_id).first()
if not sr:
print("No scan result, returning")
return
print("here")
graphTableOps = connector.tableOps("graph")
scanner = graphTableOps.createScanner(auths, 10)
range = pysharkbite.Range(obj.query, True,
obj.query + "\uffff" + "\uffff",
False) ## for now the range should be this
scanner.addRange(range)
resultset = scanner.getResultSet()
count = 0
try:
for indexKeyValue in resultset:
value = "0"
## row will be the to
## direction will be the cf
to_value = ""
direction = "one"
try:
protobuf = EdgeData_pb2.EdgeValue()
protobuf.ParseFromString(
indexKeyValue.getValue().get_bytes())
value = str(protobuf.count) + "/" + protobuf.uuid_string
to_value = indexKeyValue.getKey().getRow().split(
"\u0000")[1]
direction = indexKeyValue.getKey().getColumnFamily().split(
"/")[1]
direction_split = direction.split("-")
if len(direction_split
) != 2 or direction_split[0] == direction_split[1]:
continue
except Exception as e:
print(e)
continue
except:
continue
scanresult = res_model.objects.create(
scanResult=sr,
value=value,
row=to_value,
cf=direction,
cq=indexKeyValue.getKey().getColumnQualifier())
scanresult.save()
count = count + 1
if count > 1000:
break
sr.is_finished = True
sr.save()
scanner.close()
except Exception as e:
print(e)
except:
print("An error occurred")
pass ## user does not have PROV
obj.running = False
obj.finished = True
obj.save()
def mthd(self):
import pysharkbite
#pysharkbite.LoggingConfiguration.enableTraceLogger()
tableOperations = super().getTableOperations()
if not tableOperations.exists(False):
print ("Creating table")
tableOperations.create(False)
else:
print ("Table already exists, so not creating it")
auths = pysharkbite.Authorizations()
auths.addAuthorization("blah1")
auths.addAuthorization("blah2")
securityOps = super().getConnector().securityOps()
securityOps.grantAuthorizations(auths,"root")
""" Add authorizations """
""" mutation.put("cf","cq","cv",1569786960) """
writer = tableOperations.createWriter(auths, 10)
mutation = pysharkbite.Mutation("row2");
mutation.put("cf","cq","blah1",1569786960, "value")
mutation.put("cf2","cq2","blah1",1569786960, "value2")
""" no value """
mutation.put("cf3","cq3","blah2",1569786960, "")
writer.addMutation( mutation )
writer.close()
print("written")
auths = pysharkbite.Authorizations()
auths.addAuthorization("blah1")
scanner = tableOperations.createScanner(auths, 2)
startKey = pysharkbite.Key()
endKey = pysharkbite.Key()
startKey.setRow("row")
endKey.setRow("row3")
# test single range
range = pysharkbite.Range("row2")
scanner.addRange( range )
resultset = scanner.getResultSet()
for keyvalue in resultset:
key = keyvalue.getKey()
assert( "row2" == key.getRow() )
value = keyvalue.getValue()
if "cf" == key.getColumnFamily():
assert( "value" == value.get() )
if ("cf2" == key.getColumnFamily() ):
assert( "value2" == value.get() )
if ("cf3" == key.getColumnFamily() ):
print("Unexpected column cf3")
sys.exit(1)
range = pysharkbite.Range("row1",True,"row1.5",True)
scanner.addRange( range )
resultset = scanner.getResultSet()
for keyvalue in resultset:
print("Unexpected result")
sys.exit(1)
# test single range
range = pysharkbite.Range("row",False,"row3",True)
scanner = tableOperations.createScanner(auths, 2)
scanner.addRange( range )
resultset = scanner.getResultSet()
count =0
for keyvalue in resultset:
key = keyvalue.getKey()
assert( "row2" == key.getRow() )
value = keyvalue.getValue()
if "cf" == key.getColumnFamily():
assert( "value" == value.get() )
if ("cf2" == key.getColumnFamily() ):
assert( "value2" == value.get() )
if ("cf3" == key.getColumnFamily() ):
print("Unexpected column cf3")
sys.exit(1)
count=count+1
if count <= 0:
print("Expected results")
sys.exit(1)
# test infinite range
range = pysharkbite.Range("",False,"row3",True)
scanner = tableOperations.createScanner(auths, 2)
scanner.addRange( range )
resultset = scanner.getResultSet()
count =0
for keyvalue in resultset:
key = keyvalue.getKey()
assert( "row2" == key.getRow() )
value = keyvalue.getValue()
if "cf" == key.getColumnFamily():
assert( "value" == value.get() )
if ("cf2" == key.getColumnFamily() ):
assert( "value2" == value.get() )
if ("cf3" == key.getColumnFamily() ):
print("Unexpected column cf3")
sys.exit(1)
count=count+1
if count <= 0:
print("Expected results")
sys.exit(1)
startKey = pysharkbite.Key("row3")
range = pysharkbite.Range(None,False,startKey,True)
scanner = tableOperations.createScanner(auths, 2)
scanner.addRange( range )
resultset = scanner.getResultSet()
count =0
for keyvalue in resultset:
key = keyvalue.getKey()
assert( "row2" == key.getRow() )
value = keyvalue.getValue()
if "cf" == key.getColumnFamily():
assert( "value" == value.get() )
if ("cf2" == key.getColumnFamily() ):
assert( "value2" == value.get() )
if ("cf3" == key.getColumnFamily() ):
print("Unexpected column cf3")
sys.exit(1)
count=count+1
if count <= 0:
print("Expected results")
sys.exit(1)
""" delete your table if user did not create temp """
tableOperations.remove()
time.sleep(2)
""" auths.addAuthorization("cv") """
scanner = tableOperations.createScanner(auths, 2)
startKey = pysharkbite.Key()
endKey = pysharkbite.Key()
startKey.setRow("row")
endKey.setRow("row3")
range = pysharkbite.Range(startKey,True,endKey,False)
scanner.addRange( range )
resultset = scanner.getResultSet()
for keyvalue in resultset:
key = keyvalue.getKey()
value = keyvalue.getValue()
""" delete your table if user did not create temp """
if not args.table:
tableOperations.remove()
except RuntimeError as e:
auths = pysharkbite.Authorizations()
""" Add authorizations """
""" mutation.put("cf","cq","cv",1569786960) """
scanner = tableOperations.createScanner(auths, 2)
startKey = pysharkbite.Key()
endKey = pysharkbite.Key()
startKey.setRow("row")
endKey.setRow("row3")
range = pysharkbite.Range("a")
scanner.addRange(range)
iterator = pysharkbite.PythonIterator("PythonIterator", 100)
iterator = iterator.onNext(
"lambda x : sharkbite_iterator.Key( x.getRow(), 'new cf', x.getColumnQualifier()) "
)
scanner.addIterator(iterator)
resultset = scanner.getResultSet()
for keyvalue in resultset:
key = keyvalue.getKey()
value = keyvalue.getValue()
print(key.getRow() + ":" + key.getColumnFamily() + ":" +
