def create(self): try: if isfile(self.filename): d_print("File '%s' is already exist. Trying to remove..." % self.filename) remove(self.filename) except: self.log.error('create:fail') d_print("Cannot remove DB file '%s'. Please, close all connections to DB and try again." % self.filename) return False try: c = connect(self.filename) #c.execute('PRAGMA page_size=32768;') c.execute('create table state(offset integer)') c.execute('create table state2(offset integer)') # TODO(ibragim) remove id field c.execute(''' create table pfrTransactions( id integer primary key autoincrement, orgId_hi integer, orgId_low integer, dcId_hi integer, dcId_low integer, transactionTime integer, transactionType integer, upfrCode text, accountingYear integer, providerIdHash integer, correctionType integer, documents text, signatures text )''') c.execute('create index pfrTransactions_index on pfrTransactions(orgId_hi, orgId_low)') c.commit() c.close() self.log.info('create:ok') return True except OperationalError, e: log.error('create:fail:%s' % e) return False
def run(kanso_filenames, events): d_print("Update server is started.") log.info("start") # DANGEROUS: Could crash updater updater = IndexServerUpdater(config.IDX_FILENAME) ks = kstream(config.KANSO_FILENAME) ks2 = kstream(config.KANSO_FILENAME2) offset = updater.offset offset2 = updater.offset2 snapshot_manager = Snapshot(".", config.IDX_FILENAME, config.IDX_SNAPSHOT_DIR) while not events.stop.isSet(): start = datetime.now() events.endupdate.clear() try: # Important note: # --------------- # We guess that records (transactions in our case) stored in KANSO does not cross # KANSO chunk border and it is because of KANSO atomic writes. for b in ks.read(offset): trydb(updater.begin, "db-begin:%s") try: for inner_offset, txn in read_structs(b): trydb(updater.insert_record, "db-insert:%s", (offset + inner_offset, txn)) if events.stop.isSet(): break except error, e: log.warning("read_structs:%s" % e) trydb(updater.commit, "db-commit:%s") offset = updater.offset for b in ks2.read(offset2): trydb(updater.begin, "db-begin:%s") try: for inner_offset, txn in read_structs(b): trydb(updater.insert_record2, "db-insert:%s", (offset2 + inner_offset, txn)) if events.stop.isSet(): break except error, e: log.warning("read_structs:%s" % e) trydb(updater.commit, "db-commit:%s") offset2 = updater.offset2
def search(self, query): trials = 0 while True: try: resData = self.__get_results(query) break except OperationalError as e: d_print("Get result error") log.warning('search:%s' % e) trials += 1 if trials >= config.IDX_TRIALS: d_print("Has no more trials.") log.error('search:fail') raise e d_print("Will try again after delay...") sleep(config.IDX_DB_BUSY_DELAY) for row in resData: r = '%s\t%s\t%d\t%d\t"%s"\t%d\t%d\t%d\t\n\n%s\n%s\n\n' % \ ((UUID(int=combineInt128(row[0], row[1])), UUID(int=combineInt128(row[2], row[3]))) + row[4:]) yield r
for b in ks2.read(offset2): trydb(updater.begin, "db-begin:%s") try: for inner_offset, txn in read_structs(b): trydb(updater.insert_record2, "db-insert:%s", (offset2 + inner_offset, txn)) if events.stop.isSet(): break except error, e: log.warning("read_structs:%s" % e) trydb(updater.commit, "db-commit:%s") offset2 = updater.offset2 except s_error, e: log.warning("nokanso:%s" % e) except: # TODO:log indexserver shutdown unexpectedly (email?) d_print(exc_info()) log.critical("unexpected:%s" % str(exc_info()[1])) events.endupdate.set() updater.stop() raise if events.stop.isSet(): break offset = updater.offset offset2 = updater.offset2 # The fact is snapshot_manager changes index on start and stop (purges and commit), # So, because of this fact we have to create snapshot during update phase and it is # an issue: we have to do not start search until snapshot completes. if snapshot_manager.isTime(): start = datetime.now() try:
def do_STOP(self): d_print('Shutting down search server...') log.info('stop') t = Thread(target=shutdown_server, args=(self.server,)) t.daemon = True t.start()
def run(port, events=None): s = SearchServer(('', port), SearchRequestHandler, events) d_print('Search server started') log.info('run') s.serve_forever()
def shutdown_server(server): server.shutdown() if server.events and server.events.stop: server.events.stop.set() d_print('Search server is down.') log.info('shutdown')