def _fetch_group_articles(self, group, start, end):
LOG.debug((group, start, end))
with self.nntp_connection as nntp:
nntp.group(group)
resp, articles = nntp.xover(str(start), str(end))
LOG.debug(len(articles))
if articles:
Article.add_from_nntplib(group, articles)
def get_articles(request):
query = request.GET.get('query', '')
tq = parse_tq(request.GET.get('tq', ''))
tqx = parse_tqx(request.GET.get('tqx', ''))
select = Article.select()
select = select.limit(tq.get('limit', 1))
select = select.offset(tq.get('offset', 0))
if query:
select = select.where(Article.subject % ('*%s*' % query))
subjects = [ a.subject for a in select ]
LOG.debug(lcs(subjects))
LOG.debug(commonprefix(subjects))
dt = gviz.DataTable({
'posted': ('datetime', 'Posted'),
'poster': ('string', 'Poster'),
'subject': ('string', 'Subject'),
'message_id': ('string', 'ID')
})
dt.LoadData( a._data for a in select )
dt_order = ['subject', 'posted', 'poster', 'message_id']
gviz_json = dt.ToJSonResponse(req_id=tqx.get('reqId', 0),
columns_order=dt_order)
return itty.Response(gviz_json, content_type='application/json')
def get_state(request):
data = {
'jobs': IDXR.task_queue.qsize(),
'articles': Article.select().count(),
'groups': Group.select().count()
}
return itty.Response(json.dumps(data), content_type='application/json')
def __init__(self, config_file='defaults.cfg'):
self.config = ConfigParser.SafeConfigParser()
self.config.readfp(open(config_file))
cache.database.init(self.config.get('indexer', 'cache_file'))
Group.create_table(True)
Article.create_table(True)
max_nntp = self.config.getint('indexer', 'max_connections')
self.nntp_semaphore = threading.BoundedSemaphore(max_nntp)
max_task = 5
self.task_queue = Queue()
self.task_runners = []
for i in xrange(max_task):
runner = threading.Thread(target=self.task_runner)
runner.daemon = True
self.task_runners.append(runner)
runner.start()
