def init_engine(self):
if self.config.get('orgs_db', None):
source = OrgsSource(self.config)
OrgsIndex(self.engine, source, self.config)
if self.config.get('tender_api_url', None):
source = TenderSource(self.config)
TenderIndex(self.engine, source, self.config)
if self.config.get('plan_api_url', None):
source = PlanSource(self.config)
PlanIndex(self.engine, source, self.config)
if self.config.get('ocds_dir', None):
source = OcdsSource(self.config)
OcdsIndex(self.engine, source, self.config)
if self.config.get('auction_api_url', None):
source = AuctionSource(self.config)
AuctionIndex(self.engine, source, self.config)
if self.config.get('auction2_api_url', None):
source = AuctionSource2(self.config)
AuctionIndex2(self.engine, source, self.config)
if self.config.get('asset_api_url', None):
source = AssetSource(self.config)
AssetIndex(self.engine, source, self.config)
if self.config.get('lot_api_url', None):
source = DgfLotSource(self.config)
DgfLotIndex(self.engine, source, self.config)
def main():
if len(sys.argv) < 2 or '-h' in sys.argv:
print("Usage: update_orgs etc/search.ini [custom_index_names]")
sys.exit(1)
parser = ConfigParser()
parser.read(sys.argv[1])
config = dict(parser.items('search_engine'))
uo_config = dict(parser.items('update_orgs'))
if len(sys.argv) > 2:
config['index_names'] = sys.argv[2]
logging.config.fileConfig(sys.argv[1])
logger.info("Starting openprocurement.search.update_orgs v%s", __version__)
logger.info("Copyright (c) 2015-2016 Volodymyr Flonts <*****@*****.**>")
# try get exclusive lock to prevent second start
lock_filename = uo_config.get('pidfile') or 'update_orgs.pid'
lock_file = open(lock_filename, "w")
fcntl.lockf(lock_file, fcntl.LOCK_EX + fcntl.LOCK_NB)
lock_file.write(str(os.getpid()) + "\n")
lock_file.flush()
signal.signal(signal.SIGTERM, sigterm_handler)
# signal.signal(signal.SIGINT, sigterm_handler)
try:
global engine
engine = IndexOrgsEngine(config, uo_config)
source = OrgsSource(config)
index = OrgsIndex(engine, source, config)
# manualy reset and prevent second reset on first process_source
source.reset()
index.last_current_index = index.current_index
if config.get('tender_api_url', None):
source = TenderSource(config)
engine.process_source(source)
if config.get('ocds_dir', None):
source = OcdsSource(config)
engine.process_source(source)
if config.get('plan_api_url', None):
source = PlanSource(config)
engine.process_source(source)
if config.get('auction_api_url', None):
source = AuctionSource(config)
engine.process_source(source)
engine.flush_orgs_map()
except Exception as e:
logger.exception("Exception: %s", str(e))
finally:
lock_file.close()
os.remove(lock_filename)
logger.info("Shutdown")
return 0
def main():
if len(sys.argv) < 2:
print("Usage: index_worker etc/search.ini [custom_index_names_file]")
sys.exit(1)
if '--version' in sys.argv:
print(__version__)
sys.exit(0)
parser = ConfigParser()
parser.read(sys.argv[1])
config = dict(parser.items('search_engine'))
config = decode_bool_values(config)
# disable slave mode if used custom_index_names
if len(sys.argv) > 2:
config['index_names'] = sys.argv[2]
config['slave_mode'] = ''
config['start_wait'] = 0
logging.config.fileConfig(sys.argv[1])
logger.info("Starting ProZorro openprocurement.search.index_worker v%s",
__version__)
logger.info("Copyright (c) 2015-2018 Volodymyr Flonts <*****@*****.**>")
try:
chage_process_user_group(config, logger)
except Exception as e:
logger.error("Can't change process user: %s", str(e))
# try get exclusive lock to prevent second start
lock_filename = config.get('index_names', 'index_worker') + '.lock'
lock_file = open(lock_filename, "w")
try:
fcntl.lockf(lock_file, fcntl.LOCK_EX + fcntl.LOCK_NB)
lock_file.write(str(os.getpid()) + "\n")
lock_file.flush()
except:
logger.error("Can't get lock %s maybe already started", lock_filename)
lock_file.close()
return 1
signal.signal(signal.SIGTERM, sigterm_handler)
# signal.signal(signal.SIGINT, sigterm_handler)
try:
global engine
engine = IndexEngine(config)
if config.get('orgs_db', None):
source = OrgsSource(config, True)
OrgsIndex(engine, source, config)
if config.get('tender_api_url', None):
source = TenderSource(config, True)
TenderIndex(engine, source, config)
if config.get('ocds_dir', None):
source = OcdsSource(config)
OcdsIndex(engine, source, config)
if config.get('plan_api_url', None):
source = PlanSource(config, True)
PlanIndex(engine, source, config)
if config.get('auction_api_url', None):
source = AuctionSource(config, True)
AuctionIndex(engine, source, config)
if config.get('auction2_api_url', None):
source = AuctionSource2(config, True)
AuctionIndex2(engine, source, config)
if config.get('asset_api_url', None):
source = AssetSource(config, True)
AssetIndex(engine, source, config)
if config.get('lot_api_url', None):
source = DgfLotSource(config, True)
DgfLotIndex(engine, source, config)
engine.run()
except Exception as e:
logger.exception("Unhandled Exception: %s", str(e))
finally:
lock_file.close()
os.remove(lock_filename)
if engine and hasattr(engine, 'stop_childs'):
engine.stop_childs()
logger.info("Shutdown")
return 0
def test_plans(self):
# self.engine_config['plan_skip_until'] = ''
self.engine_config['plan_fast_client'] = False
source = PlanSource(self.engine_config)
index = PlanIndex(self.engine, source, self.engine_config)
offset = datetime.now() - timedelta(minutes=self.offset)
offset = offset.isoformat()
if self.full_test:
test_limit = 5000
skip_limit = 100
preload = 500000
limit = 1000
else:
test_limit = 5
skip_limit = 0
preload = 100
limit = 100
source.client_user_agent += " test_search"
source.reset()
source.client.params.update({'descending': 1, 'limit': limit})
source.config['plan_preload'] = preload
source.skip_until = None
logger.info("Client %s/api/%s mode=%s", source.config['plan_api_url'],
source.config['plan_api_version'],
source.client.params.get('mode', ''))
logger.info("Offset %s (%s minutes)", offset, self.offset)
logger.info("Search %s:%s", self.search_config['host'],
self.search_config['port'])
test_count = 0
skip_count = skip_limit
while test_count < test_limit:
meta = None
for meta in source.items():
if meta.dateModified > offset:
continue
if skip_count < skip_limit:
skip_count += 1
continue
try:
tender = source.get(meta)
if index.test_noindex(tender):
continue
if tender.data.dateModified > offset:
continue
self.do_search('plans?pid=%(planID)s', tender.data)
except Exception as e:
if self.ignore:
logger.error("%s (ignored)", e)
self.errors += 1
else:
raise
test_count += 1
skip_count = 0
if test_count >= test_limit:
break
if not meta:
break
if test_count < 5:
raise RuntimeError("Not enough queries")