def __init__(self): self.index = indexer.indexer() self.database = db_communication.db_com() self.base_dir = "" self.store_me = dict() self.documentCount = 0 # Count the documents you crawled self.chunk = chunksys.chunksys() self.chunkstore = list()
def storagePersistor(self, chunks): # TODO: Create db_connection multiset for local use try: if len(chunks) > 3: db = db_communication.db_com() for item in chunks: db.localMultiset(item) return 0 except Exception as msg: # TODO: Exceptionhandler print str(msg) return -1
def transfer(self, chunkstorage): try: db = db_communication.db_com() # push self.chunks to cassandra for item in chunkstorage: res = db.multi_set_words(item) if res == -1: print "Error, could not push to cassandra." if res == -1: print "[FAILED] - Chunks were not pushed to cassandra." return -1 return 0 except Exception as msg: print str(msg)