def create(self):
try:
if isfile(self.filename):
d_print("File '%s' is already exist. Trying to remove..." % self.filename)
remove(self.filename)
except:
self.log.error('create:fail')
d_print("Cannot remove DB file '%s'. Please, close all connections to DB and try again." % self.filename)
return False
try:
c = connect(self.filename)
#c.execute('PRAGMA page_size=32768;')
c.execute('create table state(offset integer)')
c.execute('create table state2(offset integer)')
# TODO(ibragim) remove id field
c.execute('''
create table pfrTransactions(
id integer primary key autoincrement,
orgId_hi integer, orgId_low integer, dcId_hi integer, dcId_low integer,
transactionTime integer, transactionType integer,
upfrCode text, accountingYear integer, providerIdHash integer, correctionType integer,
documents text, signatures text
)''')
c.execute('create index pfrTransactions_index on pfrTransactions(orgId_hi, orgId_low)')
c.commit()
c.close()
self.log.info('create:ok')
return True
except OperationalError, e:
log.error('create:fail:%s' % e)
return False
def run(kanso_filenames, events):
d_print("Update server is started.")
log.info("start")
# DANGEROUS: Could crash updater
updater = IndexServerUpdater(config.IDX_FILENAME)
ks = kstream(config.KANSO_FILENAME)
ks2 = kstream(config.KANSO_FILENAME2)
offset = updater.offset
offset2 = updater.offset2
snapshot_manager = Snapshot(".", config.IDX_FILENAME, config.IDX_SNAPSHOT_DIR)
while not events.stop.isSet():
start = datetime.now()
events.endupdate.clear()
try:
# Important note:
# ---------------
# We guess that records (transactions in our case) stored in KANSO does not cross
# KANSO chunk border and it is because of KANSO atomic writes.
for b in ks.read(offset):
trydb(updater.begin, "db-begin:%s")
try:
for inner_offset, txn in read_structs(b):
trydb(updater.insert_record, "db-insert:%s", (offset + inner_offset, txn))
if events.stop.isSet():
break
except error, e:
log.warning("read_structs:%s" % e)
trydb(updater.commit, "db-commit:%s")
offset = updater.offset
for b in ks2.read(offset2):
trydb(updater.begin, "db-begin:%s")
try:
for inner_offset, txn in read_structs(b):
trydb(updater.insert_record2, "db-insert:%s", (offset2 + inner_offset, txn))
if events.stop.isSet():
break
except error, e:
log.warning("read_structs:%s" % e)
trydb(updater.commit, "db-commit:%s")
offset2 = updater.offset2
def search(self, query):
trials = 0
while True:
try:
resData = self.__get_results(query)
break
except OperationalError as e:
d_print("Get result error")
log.warning('search:%s' % e)
trials += 1
if trials >= config.IDX_TRIALS:
d_print("Has no more trials.")
log.error('search:fail')
raise e
d_print("Will try again after delay...")
sleep(config.IDX_DB_BUSY_DELAY)
for row in resData:
r = '%s\t%s\t%d\t%d\t"%s"\t%d\t%d\t%d\t\n\n%s\n%s\n\n' % \
((UUID(int=combineInt128(row[0], row[1])), UUID(int=combineInt128(row[2], row[3]))) + row[4:])
yield r
for b in ks2.read(offset2):
trydb(updater.begin, "db-begin:%s")
try:
for inner_offset, txn in read_structs(b):
trydb(updater.insert_record2, "db-insert:%s", (offset2 + inner_offset, txn))
if events.stop.isSet():
break
except error, e:
log.warning("read_structs:%s" % e)
trydb(updater.commit, "db-commit:%s")
offset2 = updater.offset2
except s_error, e:
log.warning("nokanso:%s" % e)
except:
# TODO:log indexserver shutdown unexpectedly (email?)
d_print(exc_info())
log.critical("unexpected:%s" % str(exc_info()[1]))
events.endupdate.set()
updater.stop()
raise
if events.stop.isSet():
break
offset = updater.offset
offset2 = updater.offset2
# The fact is snapshot_manager changes index on start and stop (purges and commit),
# So, because of this fact we have to create snapshot during update phase and it is
# an issue: we have to do not start search until snapshot completes.
if snapshot_manager.isTime():
start = datetime.now()
try:
def do_STOP(self):
d_print('Shutting down search server...')
log.info('stop')
t = Thread(target=shutdown_server, args=(self.server,))
t.daemon = True
t.start()
def run(port, events=None):
s = SearchServer(('', port), SearchRequestHandler, events)
d_print('Search server started')
log.info('run')
s.serve_forever()
def shutdown_server(server):
server.shutdown()
if server.events and server.events.stop:
server.events.stop.set()
d_print('Search server is down.')
log.info('shutdown')
