def get(self, request, *args, **kwargs):
shard = request.GET.get('shard')
datatype = request.GET.get('datatype')
uid = request.GET.get('uid')
query = request.GET.get('query')
authstring = request.GET.get('auths')
url = "/search/?q=" + query
auths = pysharkbite.Authorizations()
if not authstring is None and len(authstring) > 0:
auths.addAuthorization(authstring)
user = pysharkbite.AuthInfo(AccumuloCluster.objects.first().user,AccumuloCluster.objects.first().password, ZkInstance().get().getInstanceId())
connector = pysharkbite.AccumuloConnector(user, ZkInstance().get())
tableOps = connector.tableOps(AccumuloCluster.objects.first().dataTable)
scanner = tableOps.createScanner(auths,1)
startKey = pysharkbite.Key(row=shard)
endKey = pysharkbite.Key(row=shard)
docid = datatype + "\x00" + uid;
startKey.setColumnFamily(docid)
endKey.setColumnFamily(docid + "\xff")
rng = pysharkbite.Range(startKey,True,endKey,True)
scanner.addRange(rng)
writer = tableOps.createWriter(auths,10)
deletes = pysharkbite.Mutation(shard)
for keyValue in scanner.getResultSet():
key = keyValue.getKey()
deletes.putDelete( key.getColumnFamily(), key.getColumnQualifier(), key.getColumnVisibility(), key.getTimestamp())
## scan for the original document
writer.addMutation(deletes)
writer.close()
for auth in authstring.split("|"):
url = url + "&auths=" + auth
return HttpResponseRedirect(url)
def writerfile(self):
import pysharkbite
rfile = pysharkbite.RFileOperations.openForWrite("blah.rf")
for x in range(1000):
key = pysharkbite.Key(row=str("row" + str(x)))
value = pysharkbite.Value()
kv = pysharkbite.KeyValue(key,value)
rfile.append(kv)
rfile.close()
mutation.put("cf","cq","",1569786960, "value")
mutation.put("cf2","cq2","",1569786960, "value2")
""" no value """
mutation.put("cf3","cq3","",1569786960, "")
writer.addMutation( mutation )
writer.close()
time.sleep(2)
""" auths.addAuthorization("cv") """
scanner = tableOperations.createScanner(auths, 2)
startKey = pysharkbite.Key()
endKey = pysharkbite.Key()
startKey.setRow("row")
endKey.setRow("row3")
range = pysharkbite.Range(startKey,True,endKey,False)
scanner.addRange( range )
resultset = scanner.getResultSet()
for keyvalue in resultset:
key = keyvalue.getKey()
def mthd(self):
import pysharkbite
#pysharkbite.LoggingConfiguration.enableTraceLogger()
tableOperations = super().getTableOperations()
if not tableOperations.exists(False):
print ("Creating table")
tableOperations.create(False)
else:
print ("Table already exists, so not creating it")
auths = pysharkbite.Authorizations()
auths.addAuthorization("blah1")
auths.addAuthorization("blah2")
securityOps = super().getConnector().securityOps()
securityOps.grantAuthorizations(auths,"root")
""" Add authorizations """
""" mutation.put("cf","cq","cv",1569786960) """
writer = tableOperations.createWriter(auths, 10)
mutation = pysharkbite.Mutation("row2");
mutation.put("cf","cq","blah1",1569786960, "value")
mutation.put("cf2","cq2","blah1",1569786960, "value2")
""" no value """
mutation.put("cf3","cq3","blah2",1569786960, "")
writer.addMutation( mutation )
writer.close()
print("written")
auths = pysharkbite.Authorizations()
auths.addAuthorization("blah1")
scanner = tableOperations.createScanner(auths, 2)
startKey = pysharkbite.Key()
endKey = pysharkbite.Key()
startKey.setRow("row")
endKey.setRow("row3")
# test single range
range = pysharkbite.Range("row2")
scanner.addRange( range )
resultset = scanner.getResultSet()
for keyvalue in resultset:
key = keyvalue.getKey()
assert( "row2" == key.getRow() )
value = keyvalue.getValue()
if "cf" == key.getColumnFamily():
assert( "value" == value.get() )
if ("cf2" == key.getColumnFamily() ):
assert( "value2" == value.get() )
if ("cf3" == key.getColumnFamily() ):
print("Unexpected column cf3")
sys.exit(1)
range = pysharkbite.Range("row1",True,"row1.5",True)
scanner.addRange( range )
resultset = scanner.getResultSet()
for keyvalue in resultset:
print("Unexpected result")
sys.exit(1)
# test single range
range = pysharkbite.Range("row",False,"row3",True)
scanner = tableOperations.createScanner(auths, 2)
scanner.addRange( range )
resultset = scanner.getResultSet()
count =0
for keyvalue in resultset:
key = keyvalue.getKey()
assert( "row2" == key.getRow() )
value = keyvalue.getValue()
if "cf" == key.getColumnFamily():
assert( "value" == value.get() )
if ("cf2" == key.getColumnFamily() ):
assert( "value2" == value.get() )
if ("cf3" == key.getColumnFamily() ):
print("Unexpected column cf3")
sys.exit(1)
count=count+1
if count <= 0:
print("Expected results")
sys.exit(1)
# test infinite range
range = pysharkbite.Range("",False,"row3",True)
scanner = tableOperations.createScanner(auths, 2)
scanner.addRange( range )
resultset = scanner.getResultSet()
count =0
for keyvalue in resultset:
key = keyvalue.getKey()
assert( "row2" == key.getRow() )
value = keyvalue.getValue()
if "cf" == key.getColumnFamily():
assert( "value" == value.get() )
if ("cf2" == key.getColumnFamily() ):
assert( "value2" == value.get() )
if ("cf3" == key.getColumnFamily() ):
print("Unexpected column cf3")
sys.exit(1)
count=count+1
if count <= 0:
print("Expected results")
sys.exit(1)
startKey = pysharkbite.Key("row3")
range = pysharkbite.Range(None,False,startKey,True)
scanner = tableOperations.createScanner(auths, 2)
scanner.addRange( range )
resultset = scanner.getResultSet()
count =0
for keyvalue in resultset:
key = keyvalue.getKey()
assert( "row2" == key.getRow() )
value = keyvalue.getValue()
if "cf" == key.getColumnFamily():
assert( "value" == value.get() )
if ("cf2" == key.getColumnFamily() ):
assert( "value2" == value.get() )
if ("cf3" == key.getColumnFamily() ):
print("Unexpected column cf3")
sys.exit(1)
count=count+1
if count <= 0:
print("Expected results")
sys.exit(1)
""" delete your table if user did not create temp """
tableOperations.remove()
def mthd(self):
import pysharkbite
tableOperations = super().getTableOperations()
if not tableOperations.exists(False):
print("Creating table")
if not tableOperations.create(False):
print("Could not create table")
else:
print("Table already exists, so not creating it")
auths = pysharkbite.Authorizations()
""" Add authorizations """
""" mutation.put("cf","cq","cv",1569786960) """
writer = tableOperations.createWriter(auths, 10)
mutation = pysharkbite.Mutation("sow2")
mutation.put("cf", "cq", "", 1569786960, "value")
mutation.put("cf2", "cq2", "", 1569786960, "value2")
""" no value """
mutation.put("cf3", "cq3", "", 1569786960, "")
writer.addMutation(mutation)
writer.close()
writer = tableOperations.createWriter(auths, 10)
rng = range(0, 1000)
for i in rng:
row = ("row%i" % (i + 5))
mutation = pysharkbite.Mutation(row)
mutation.put("cf", "cq", "", 1569786960, "value")
writer.addMutation(mutation)
writer.close()
print("written")
""" auths.addAuthorization("cv") """
scanner = tableOperations.createScanner(auths, 2)
startKey = pysharkbite.Key()
endKey = pysharkbite.Key()
startKey.setRow("sow")
endKey.setRow("sow3")
accumuloRange = pysharkbite.Range(startKey, True, endKey, False)
scanner.addRange(accumuloRange)
resultset = scanner.getResultSet()
for keyvalue in resultset:
key = keyvalue.getKey()
assert ("sow2" == key.getRow())
value = keyvalue.getValue()
if "cf" == key.getColumnFamily():
assert ("value" == value.get())
if ("cf2" == key.getColumnFamily()):
assert ("value2" == value.get())
if ("cf3" == key.getColumnFamily()):
assert ("" == value.get())
scanner = tableOperations.createScanner(auths, 2)
accumuloRange = pysharkbite.Range("row", True, "", False)
scanner.addRange(accumuloRange)
resultset = scanner.getResultSet()
count = 0
for keyvalue in resultset:
key = keyvalue.getKey()
count = count + 1
print("count is ", count)
assert (count == 1003) # all rows + sows
""" delete your table if user did not create temp """
tableOperations.remove()
def getDocuments(cancellationtoken: CancellationToken, name: int,
lookupInformation: LookupInformation,
input: queue.SimpleQueue, outputQueue: queue.SimpleQueue):
count = 0
while cancellationtoken.running():
docInfo = None
try:
try:
if input.empty():
pass
else:
docInfo = input.get(timeout=1)
except:
pass
# Handle empty queue here
if not docInfo is None:
tableOps = lookupInformation.getTableOps()
scanner = tableOps.createScanner(lookupInformation.getAuths(),
5)
startKey = pysharkbite.Key()
endKey = pysharkbite.Key()
startKey.setRow(docInfo.getShard())
docid = docInfo.getDataType() + "\x00" + docInfo.getDocId()
startKey.setColumnFamily(docid)
endKey.setRow(docInfo.getShard())
endKey.setColumnFamily(docid + "\xff")
print("Searching for " + docInfo.getShard())
rng = pysharkbite.Range(startKey, True, endKey, True)
scanner.addRange(rng)
rangecount = 1
while rangecount < 10:
try:
docInfo = input.get(False)
startKey = pysharkbite.Key()
endKey = pysharkbite.Key()
startKey.setRow(docInfo.getShard())
docid = docInfo.getDataType(
) + "\x00" + docInfo.getDocId()
startKey.setColumnFamily(docid)
endKey.setRow(docInfo.getShard())
endKey.setColumnFamily(docid + "\xff")
rng = pysharkbite.Range(startKey, True, endKey, True)
print("Searching for " + docInfo.getShard())
scanner.addRange(rng)
rangecount = rangecount + 1
except:
rangecount = 11
with open('jsoncombiner.py', 'r') as file:
combinertxt = file.read()
combiner = pysharkbite.PythonIterator(
"PythonCombiner", combinertxt, 100)
scanner.addIterator(combiner)
try:
count = count + scanDoc(scanner, outputQueue)
except:
pass
else:
time.sleep(0.5)
except:
e = sys.exc_info()[0]
return True
def check():
model = apps.get_model(app_label='query', model_name='FileUpload')
objs = model.objects.filter(status="NEW")
for obj in objs:
if obj.status == "NEW":
import pysharkbite
conf = pysharkbite.Configuration()
conf.set("FILE_SYSTEM_ROOT", "/accumulo")
model = apps.get_model(app_label='query',
model_name='AccumuloCluster')
accumulo_cluster = model.objects.first()
print("Checking " + str(obj.uuid))
if accumulo_cluster is None:
return
print("Checking " + str(obj.uuid))
zk = pysharkbite.ZookeeperInstance(accumulo_cluster.instance,
accumulo_cluster.zookeeper,
1000, conf)
user = pysharkbite.AuthInfo("root", "secret", zk.getInstanceId())
connector = pysharkbite.AccumuloConnector(user, zk)
indexTableOps = connector.tableOps("provenanceIndex")
auths = pysharkbite.Authorizations()
auths.addAuthorization("PROV")
indexScanner = indexTableOps.createScanner(auths, 2)
indexrange = pysharkbite.Range(str(obj.uuid))
indexScanner.addRange(indexrange)
indexSet = indexScanner.getResultSet()
rangelist = list()
provops = connector.tableOps("provenance")
scanner = provops.createScanner(auths, 10)
for indexKeyValue in indexSet:
value = indexKeyValue.getValue()
protobuf = Uid_pb2.List()
protobuf.ParseFromString(value.get().encode())
for uidvalue in protobuf.UID:
shard = indexKeyValue.getKey().getColumnQualifier().split(
"\u0000")[0]
datatype = indexKeyValue.getKey().getColumnQualifier(
).split("\u0000")[1]
startKey = pysharkbite.Key()
stopKey = pysharkbite.Key()
startKey.setRow(shard)
stopKey.setRow(shard)
startKey.setColumnFamily(datatype + "\x00" + uidvalue)
stopKey.setColumnFamily(datatype + "\x00" + uidvalue +
"\xff")
rangelist.append(
pysharkbite.Range(startKey, True, stopKey, False))
scanner = provops.createScanner(auths, 10)
scanner.addRange(
pysharkbite.Range(startKey, True, stopKey, False))
resultset = scanner.getResultSet()
for keyvalue in resultset:
key = keyvalue.getKey()
value = keyvalue.getValue()
eventid = key.getColumnFamily().split("\u0000")[1]
fieldname = key.getColumnQualifier().split("\u0000")[0]
fieldvalue = key.getColumnQualifier().split(
"\u0000")[1]
if (fieldname == "EVENTTYPE"):
if fieldvalue == "DROP":
obj.status = "COMPLETE"
obj.save()
break
scanner.close()
indexScanner.close()
