def _fetch_group_articles(self, group, start, end): LOG.debug((group, start, end)) with self.nntp_connection as nntp: nntp.group(group) resp, articles = nntp.xover(str(start), str(end)) LOG.debug(len(articles)) if articles: Article.add_from_nntplib(group, articles)
def get_articles(request): query = request.GET.get('query', '') tq = parse_tq(request.GET.get('tq', '')) tqx = parse_tqx(request.GET.get('tqx', '')) select = Article.select() select = select.limit(tq.get('limit', 1)) select = select.offset(tq.get('offset', 0)) if query: select = select.where(Article.subject % ('*%s*' % query)) subjects = [a.subject for a in select] LOG.debug(lcs(subjects)) LOG.debug(commonprefix(subjects)) dt = gviz.DataTable({ 'posted': ('datetime', 'Posted'), 'poster': ('string', 'Poster'), 'subject': ('string', 'Subject'), 'message_id': ('string', 'ID') }) dt.LoadData(a._data for a in select) dt_order = ['subject', 'posted', 'poster', 'message_id'] gviz_json = dt.ToJSonResponse(req_id=tqx.get('reqId', 0), columns_order=dt_order) return itty.Response(gviz_json, content_type='application/json')
def get_state(request): data = { 'jobs': IDXR.task_queue.qsize(), 'articles': Article.select().count(), 'groups': Group.select().count() } return itty.Response(json.dumps(data), content_type='application/json')
def get_articles(request): query = request.GET.get('query', '') tq = parse_tq(request.GET.get('tq', '')) tqx = parse_tqx(request.GET.get('tqx', '')) select = Article.select() select = select.limit(tq.get('limit', 1)) select = select.offset(tq.get('offset', 0)) if query: select = select.where(Article.subject % ('*%s*' % query)) subjects = [ a.subject for a in select ] LOG.debug(lcs(subjects)) LOG.debug(commonprefix(subjects)) dt = gviz.DataTable({ 'posted': ('datetime', 'Posted'), 'poster': ('string', 'Poster'), 'subject': ('string', 'Subject'), 'message_id': ('string', 'ID') }) dt.LoadData( a._data for a in select ) dt_order = ['subject', 'posted', 'poster', 'message_id'] gviz_json = dt.ToJSonResponse(req_id=tqx.get('reqId', 0), columns_order=dt_order) return itty.Response(gviz_json, content_type='application/json')
def __init__(self, config_file='defaults.cfg'): self.config = ConfigParser.SafeConfigParser() self.config.readfp(open(config_file)) cache.database.init(self.config.get('indexer', 'cache_file')) Group.create_table(True) Article.create_table(True) max_nntp = self.config.getint('indexer', 'max_connections') self.nntp_semaphore = threading.BoundedSemaphore(max_nntp) max_task = 5 self.task_queue = Queue() self.task_runners = [] for i in xrange(max_task): runner = threading.Thread(target=self.task_runner) runner.daemon = True self.task_runners.append(runner) runner.start()