def static(filename): bottle.response.content_type = mimetypes.guess_type(filename)[0] filename = os.path.normpath(os.path.join(conf('http.static_path'), 'html/', filename)) if not filename.startswith(os.path.join(conf('http.static_path'), 'html/')): bottle.abort(404) if not os.path.exists(filename): bottle.abort(404) return file(filename).read()
def get_nzb(group, filename): path = os.path.join(conf('scraper.nzb_path'), group, filename) path = os.path.normpath(path) if not path.startswith(conf('scraper.nzb_path')): bottle.abort(404) bottle.response.content_type = 'application/x-nzb' if os.path.exists(path): return file(path, 'rb').read() if os.path.exists(path + '.gz'): return gzip.open(path + '.gz', 'rb').read() bottle.abort(404)
def get_nzbdir(self, group): nzbdir = '%s/%s/' % (conf('scraper.nzb_path'), group) try: makedirs(nzbdir) except: pass return nzbdir
def get_tmpdir(self, group): tmpdir = '%s/%s/' % (conf('scraper.tmp_path'), group) try: makedirs(tmpdir) except: pass return tmpdir
def get_subjects(self): lastid = self.db.hget('lastid', self.group) if not lastid: lastid = conf('scraper.firstid') if not lastid: log.warning('lastid not found in db and firstid missing from config file, starting at post id 0') lastid = '0' self.send('XHDR Subject %s-' % lastid) line = self.readline() if not line.startswith('221'): log.error(line) log.info('Started downloading headers for ' + self.group) while True: line = self.readline() #log.debug('< %s' % line) if line == '.': break yield line #postid, subject = line.split(' ', 1) #yield (postid, subject) log.info('Finished downloading headers for ' + self.group) return
def search(): return file(os.path.join(conf('http.static_path'), 'html/search.html')).read()
def recent(): return file(os.path.join(conf('http.static_path'), 'html/recent.html')).read()
def get_groups(): return json.dumps(conf('scraper.groups'), indent=2)
def search_files(group, fmt): t = db.pipeline() q = bottle.request.params['q'] query = q.replace('-', ' ').replace('_', ' ') keywords = query.split(' ') keywords = ['%s/zindex/%s' % (group, x) for x in keywords if x] for k in keywords: t.exists(k) for i, exists in enumerate(t.execute()): if not exists: keywords.remove(keywords[i]) t.reset() tmpkey = 'zinter-%.02f+%i' % (time(), randint(1, 9999)) if keywords: db.zinterstore(tmpkey, keywords, 'MAX') matches = db.zrange(tmpkey, 0, 500, desc=True) if matches: results = [json.loads(x) for x in db.hmget('%s/metadata' % group, matches) if x] else: results = [] db.delete(tmpkey) else: results = [] if fmt == 'rss': bottle.response.content_type = 'application/rss+xml' template = templates.get_template('search.rss') return template.render(query=q, group=group, now=time(), results=results, baseurl=conf('http.baseurl')) else: bottle.response.content_type = 'text/javascript' return json.dumps(results, indent=2)
def recent_files(group, fmt): results = db.zrange('%s/recent' % group, 0, 100, desc=True) results = [json.loads(x) for x in results] if fmt == 'rss': bottle.response.content_type = 'application/rss+xml' template = templates.get_template('search.rss') return template.render(query='Recent posts', group=group, now=time(), results=results, baseurl=conf('http.baseurl')) else: bottle.response.content_type = 'text/javascript' return json.dumps(results, indent=2)
from time import strftime, gmtime, time from urllib import quote_plus from random import randint import mimetypes import os.path import gzip import simplejson as json import bottle import jinja2 import redis bottle.debug(True) db = redis.Redis(conf('redis.server'), db=conf('redis.db')) templates = jinja2.Environment(loader=jinja2.FileSystemLoader(os.path.join(conf('http.static_path'), 'templates/'))) def rfc822(ts): return strftime('%a, %d %b %Y %H:%M:%S GMT', gmtime(ts)) templates.filters['rfc822'] = rfc822 templates.filters['quote_plus'] = quote_plus @bottle.get('/api/1/:group/recent.:fmt#(json|rss)#') def recent_files(group, fmt): results = db.zrange('%s/recent' % group, 0, 100, desc=True) results = [json.loads(x) for x in results] if fmt == 'rss': bottle.response.content_type = 'application/rss+xml' template = templates.get_template('search.rss')
def __init__(self, host, username, password, database, port=119): NNTPConnection.__init__(self, host, port, reconnect_timeout=conf('scraper.reconnect_timeout')) self.username = username self.password = password self.db = database