def __init__(self):
self.index = indexer.indexer()
self.database = db_communication.db_com()
self.base_dir = ""
self.store_me = dict()
self.documentCount = 0 # Count the documents you crawled
self.chunk = chunksys.chunksys()
self.chunkstore = list()
def __init__(self):
self.index = indexer.indexer()
self.database = db_communication.db_com()
self.base_dir = ""
self.store_me = dict()
self.documentCount = 0 # Count the documents you crawled
self.chunk = chunksys.chunksys()
self.chunkstore = list()
def storagePersistor(self, chunks):
# TODO: Create db_connection multiset for local use
try:
if len(chunks) > 3:
db = db_communication.db_com()
for item in chunks:
db.localMultiset(item)
return 0
except Exception as msg:
# TODO: Exceptionhandler
print str(msg)
return -1
def storagePersistor(self, chunks):
# TODO: Create db_connection multiset for local use
try:
if len(chunks) > 3:
db = db_communication.db_com()
for item in chunks:
db.localMultiset(item)
return 0
except Exception as msg:
# TODO: Exceptionhandler
print str(msg)
return -1
def transfer(self, chunkstorage):
try:
db = db_communication.db_com()
# push self.chunks to cassandra
for item in chunkstorage:
res = db.multi_set_words(item)
if res == -1:
print "Error, could not push to cassandra."
if res == -1:
print "[FAILED] - Chunks were not pushed to cassandra."
return -1
return 0
except Exception as msg:
print str(msg)
def transfer(self, chunkstorage):
try:
db = db_communication.db_com()
# push self.chunks to cassandra
for item in chunkstorage:
res = db.multi_set_words(item)
if res == -1:
print "Error, could not push to cassandra."
if res == -1:
print "[FAILED] - Chunks were not pushed to cassandra."
return -1
return 0
except Exception as msg:
print str(msg)
