def index_logs(self):
irclogs = web_ui.IrcLogsView(self.env)
framework = Framework('builtin://%s/%s.idx' %
(quote(irclogs.search_db_path),
quote(irclogs.prefix)))
framework.add_source('file://%s' % quote(irclogs.path))
framework.update()
framework.close()
def index_logs(self):
irclogs = web_ui.IrcLogsView(self.env)
framework = Framework(
'builtin://%s/%s.idx' %
(quote(irclogs.search_db_path), quote(irclogs.prefix)))
framework.add_source('file://%s' % quote(irclogs.path))
framework.update()
framework.close()
def get_search_results(self, req, terms, filters):
def _cut_line(text, size):
"""Cut the line down appending ... if the line is larger """
if len(text) > size:
return text[:size] + ' ...'
return text
if not 'irclogs' in filters:
return
import time
from datetime import datetime
from pyndexter import Framework, READONLY
from pyndexter.util import quote
framework = Framework(self.indexer, stemmer='porter://', mode=READONLY)
framework.add_source('file://%s?include=*.log' % quote(self.path))
try:
for hit in framework.search(' '.join(terms)):
path = hit.uri.path
anchor = self.find_anchor(hit.current.content, terms)
year, month, day = path[-12:-8], path[-8:-6], path[-6:-4]
timestamp = time.mktime(datetime(int(year), int(month), int(day)).timetuple())
yield (req.href('/irclogs/%s/%s/%s' % (year, month, day)) + anchor,
'%s logs for %s-%s-%s' % (self.prefix, year, month, day),
timestamp,
'irclog',
unicode(hit.excerpt(terms)))
except Exception, e:
self.log.debug('pyndexter has a big fat bug. Give alect crap about it.', exc_info=e)
return
def get_search_results(self, req, terms, filters):
def _cut_line(text, size):
if len(text) > size:
return text[:size] + '...'
return text
if not 'irclogs' in filters:
return
irclogs = web_ui.IrcLogsView(self.env)
framework = Framework(
'builtin://%s/%s.idx' %
(quote(irclogs.search_db_path), quote(irclogs.prefix)),
mode=READWRITE)
framework.add_source('file://%s' % quote(irclogs.path))
tz = req.tz
utc = pytz.utc
unicode_terms = []
for term in terms:
unicode_terms.append(unicode(term))
for hit in framework.search(' '.join(terms)):
path = hit.uri.path
server_dt = self.find_anchor(irclogs, hit.current.content, terms)
if server_dt is None:
continue
utc_dt = utc.normalize(server_dt.astimezone(utc))
user_dt = tz.normalize(server_dt.astimezone(tz))
user_time = user_dt.strftime("%H:%M:%S")
anchor = utc_dt.strftime("#UTC%Y-%m-%dT%H:%M:%S")
year, month, day, hour, minute, second = \
path[-14:-10], path[-9:-7], path[-6:-4], user_time[0:2], \
user_time[3:5], user_time[6:8]
timestamp = user_dt.replace(tzinfo=localtz)
yield req.href('/irclogs/%s/%s/%s' % (year, month, day)) + anchor, \
'IRC: %s logs for %s-%s-%s' % \
(irclogs.prefix, year, month, day), \
timestamp, 'irclog', \
hit.excerpt(unicode_terms)
framework.close()
def update_irc_search():
args = sys.argv
index_path = args[1]
log_path = args[2]
files = os.listdir(args[2])
for file in files:
try:
if os.path.isdir("%s/%s.idx" % (index_path, file)):
output = shutil.rmtree("%s/%s.idx" % (index_path, file))
framework = Framework('builtin://%s/%s.idx' %
(quote(index_path), quote(file)), mode=READWRITE)
framework.add_source('file://%s/%s' % (quote(log_path), quote(file)))
framework.update()
framework.close()
except Exception, e:
code, message = e
print 'Error %s: %s' % (code, message)
def get_search_results(self, req, terms, filters):
def _cut_line(text, size):
if len(text) > size:
return text[:size] + '...'
return text
if not 'irclogs' in filters:
return
irclogs = web_ui.IrcLogsView(self.env)
framework = Framework('builtin://%s/%s.idx' %
(quote(irclogs.search_db_path),
quote(irclogs.prefix)), mode=READWRITE)
framework.add_source('file://%s' % quote(irclogs.path))
tz = req.tz
utc = pytz.utc
unicode_terms = []
for term in terms:
unicode_terms.append(unicode(term))
for hit in framework.search(' '.join(terms)):
path = hit.uri.path
server_dt = self.find_anchor(irclogs, hit.current.content, terms)
if server_dt is None:
continue
utc_dt = utc.normalize(server_dt.astimezone(utc))
user_dt = tz.normalize(server_dt.astimezone(tz))
user_time = user_dt.strftime("%H:%M:%S")
anchor = utc_dt.strftime("#UTC%Y-%m-%dT%H:%M:%S")
year, month, day, hour, minute, second = \
path[-14:-10], path[-9:-7], path[-6:-4], user_time[0:2], \
user_time[3:5], user_time[6:8]
timestamp = user_dt.replace(tzinfo=localtz)
yield req.href('/irclogs/%s/%s/%s' % (year, month, day)) + anchor, \
'IRC: %s logs for %s-%s-%s' % \
(irclogs.prefix, year, month, day), \
timestamp, 'irclog', \
hit.excerpt(unicode_terms)
framework.close()
def update_irc_search():
args = sys.argv
index_path = args[1]
log_path = args[2]
files = os.listdir(args[2])
for file in files:
try:
if os.path.isdir("%s/%s.idx" % (index_path, file)):
output = shutil.rmtree("%s/%s.idx" % (index_path, file))
framework = Framework('builtin://%s/%s.idx' %
(quote(index_path), quote(file)),
mode=READWRITE)
framework.add_source('file://%s/%s' %
(quote(log_path), quote(file)))
framework.update()
framework.close()
except Exception, e:
code, message = e
print 'Error %s: %s' % (code, message)
from pyndexter import *
from pyndexter.util import TimingFilter, quote
VERSION = '0.2'
def index_logs():
"""Run the indexing"""
framework = Framework(
'builtin:///var/pacopablo/trac/casa_de_pacopablo/indexes/irclogs.idx?cache=true',
stemmer='porter://')
framework.add_source(
'file://%s?include=*.log' %
quote('/home/pacopablo/irc/trac/logs/ChannelLogger/freenode/#trac/'))
framework.update(filter=TimingFilter(progressive=True))
framework.close()
def doArgs():
""" Look if you can't guess what this function does, just give up now. """
global VERSION
version = "%%prog %s" % VERSION
usage = "usage: %prog [options] [site]"
description = "%prog is used to index the irclogs for searching."
parser = OptionParser(usage=usage,
# All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution.
#
# Author: John Hampton <*****@*****.**>
from pyndexter import *
from pyndexter.util import TimingFilter, quote
VERSION='0.2'
def index_logs():
"""Run the indexing"""
framework = Framework('builtin:///var/pacopablo/trac/casa_de_pacopablo/indexes/irclogs.idx?cache=true', stemmer='porter://')
framework.add_source('file://%s?include=*.log' % quote('/home/pacopablo/irc/trac/logs/ChannelLogger/freenode/#trac/'))
framework.update(filter=TimingFilter(progressive=True))
framework.close()
def doArgs():
""" Look if you can't guess what this function does, just give up now. """
global VERSION
version = "%%prog %s" % VERSION
usage ="usage: %prog [options] [site]"
description="%prog is used to index the irclogs for searching."