def run(self):
LOG.info(
'rest server run at %s:%d' %
(self.conf.rest_server_address(), self.conf.rest_server_port()))
self.app.run(self.conf.rest_server_address(),
self.conf.rest_server_port(), True)
def schedule_users(self):
user_list = self.db.user_list()
if not user_list or len(user_list) == 0:
LOG.info("not user in database.")
return
for user in user_list:
self.add_user(user.name)
def add_user(self, username):
if len(self.se_mgr.stats_get()) == 0:
LOG.info("not search engine.")
return
if self.se_mgr.user_exist(username):
return
se_key = self.filter.select(self.se_mgr.stats_get())
if not se_key:
LOG.error("filter do not select useful search engine for user:%s" %
(username))
return
self.se_mgr.add_user(se_key, username)
def start(self):
LOG.info("search engine running...")
while True:
self.user_list_reload()
if len(self.user_list) == 0:
time.sleep(SearchEngineBase.USER_LIST_EMPTY_RESCHED_INTERVAL)
continue
for user in self.user_list:
self.search_user(user)
time.sleep(self.conf.search_interval())
time.sleep(self.conf.search_interval())
def result_list_direct(self, username, start_time, end_time):
user = self.session.query(User.id).filter_by(name=username).first()
if not user:
LOG.info('user %s not find' % (username))
return []
if not end_time:
end_time = datetime.datetime.now()
results = self.session.query(
Result.url, Result.source_url, Result.datetime,
Result.keyword).filter(Result.user_id == user.id).filter(
Result.datetime >= start_time).filter(
Result.datetime < end_time).all()
return results
def search_user(self, user):
print user
LOG.info("username:%s" % (user['username']))
LOG.info('keyword:%s' % (user['keyword']))
LOG.info('negative_word:%s' % (user['negative_word']))
for keyword in user['keyword']:
self._search_keyword(user, keyword)
time.sleep(self.conf.search_interval())
def _search_one_negative_word(self, user, page, word, url):
if self.algorithm.match(word, page):
LOG.info("find match: user: %s url:%s word:%s" % (user, url, word))
self.db.add_result(user['username'], url, self.conf.url(), word)