def read(self): """ read complete event-log from disk """ print "Reading event-log. Depending on the size of event-log this may take a long time and" print "consume lots of memory (if you don't need statistics, truncate event-log to speed this" print "up and use less memory)." data = [] try: lineno = 0 f = file( self.fname, 'r') # read in text mode, so we can iterate over text lines for line in f: lineno += 1 line = line.replace('\r', '').replace('\n', '') if not line.strip(): # skip empty lines continue fields = line.split('\t') try: timestamp, action, kvpairs = fields[:3] timestamp = int(timestamp) kvdict = wikiutil.parseQueryString(kvpairs) data.append((timestamp, action, kvdict)) except ValueError, err: # corrupt event log line, log error and skip it print "Error: invalid event log (%s) line %d, err: %s, SKIPPING THIS LINE!" % ( self.fname, lineno, str(err)) f.close()
def read(self): """ read complete event-log from disk """ data = [] try: lineno = 0 f = file(self.fname, "r") for line in f: lineno += 1 line = line.replace("\r", "").replace("\n", "") if not line.strip(): # skip empty lines continue fields = line.split("\t") try: timestamp, action, kvpairs = fields[:3] timestamp = int(timestamp) kvdict = wikiutil.parseQueryString(kvpairs) data.append((timestamp, action, kvdict)) except ValueError, err: # corrupt event log line, log error and skip it print "Error: invalid event log (%s) line %d, err: %s, SKIPPING THIS LINE!" % ( self.fname, lineno, str(err), ) f.close()
def parser(self, line): try: time_usecs, eventtype, kvpairs = line.rstrip().split('\t') except ValueError: # badly formatted line in file, skip it return None return long(time_usecs), eventtype, wikiutil.parseQueryString(kvpairs)
def parser(self, line): """ parse a event-log line into its components """ try: time_usecs, eventtype, kvpairs = line.rstrip().split('\t') except ValueError: # badly formatted line in file, skip it return None return long(time_usecs), eventtype, wikiutil.parseQueryString(kvpairs)
def parser(self, line): """ parse a event-log line into its components """ try: time_usecs, eventtype, kvpairs = line.rstrip().split("\t") except ValueError: # badly formatted line in file, skip it return None return long(time_usecs), eventtype, wikiutil.parseQueryString(kvpairs)
def read(self): """ read complete event-log from disk """ data = [] try: lineno = 0 f = file(self.fname, 'r') for line in f: lineno += 1 line = line.replace('\r', '').replace('\n', '') if not line.strip(): # skip empty lines continue fields = line.split('\t') try: timestamp, action, kvpairs = fields[:3] timestamp = int(timestamp) kvdict = wikiutil.parseQueryString(kvpairs) data.append((timestamp, action, kvdict)) except ValueError, err: # corrupt event log line, log error and skip it print "Error: invalid event log (%s) line %d, err: %s, SKIPPING THIS LINE!" % (self.fname, lineno, str(err)) f.close()
def read(self): """ read complete event-log from disk """ print "Reading event-log. Depending on the size of event-log this may take a long time and" print "consume lots of memory (if you don't need statistics, truncate event-log to speed this" print "up and use less memory)." data = [] try: lineno = 0 f = file(self.fname, 'r') # read in text mode, so we can iterate over text lines for line in f: lineno += 1 line = line.replace('\r', '').replace('\n', '') if not line.strip(): # skip empty lines continue fields = line.split('\t') try: timestamp, action, kvpairs = fields[:3] timestamp = int(timestamp) kvdict = wikiutil.parseQueryString(kvpairs) data.append((timestamp, action, kvdict)) except ValueError, err: # corrupt event log line, log error and skip it print "Error: invalid event log (%s) line %d, err: %s, SKIPPING THIS LINE!" % (self.fname, lineno, str(err)) f.close()
def execute(pagename, request, fieldname='value', titlesearch=0, statistic=0): _ = request.getText titlesearch = checkTitleSearch(request) if titlesearch < 0: check_surge_protect(request, kick=True) # get rid of spammer return if 'metasearch' in request.values: form = MultiDict(request.values) form['action'] = 'MetaSearch' val = form.get('value', '') form['q'] = val request.values = CombinedMultiDict([MultiDict(form)]) return ms_execute(pagename, request) advancedsearch = isAdvancedSearch(request) form = request.values # context is relevant only for full search if titlesearch: context = 0 elif advancedsearch: context = 180 # XXX: hardcoded context count for advancedsearch else: context = int(form.get('context', 0)) # Get other form parameters needle = form.get(fieldname, '') case = int(form.get('case', 0)) regex = int(form.get('regex', 0)) # no interface currently hitsFrom = int(form.get('from', 0)) highlight_titles = int(form.get('highlight_titles', 1)) highlight_pages = int(form.get('highlight_pages', 1)) mtime = None msg = '' historysearch = 0 # if advanced search is enabled we construct our own search query if advancedsearch: and_terms = form.get('and_terms', '').strip() or_terms = form.get('or_terms', '').strip() not_terms = form.get('not_terms', '').strip() #xor_terms = form.get('xor_terms', '').strip() categories = form.getlist('categories') or [''] timeframe = form.get('time', '').strip() language = form.getlist('language') or [''] mimetype = form.getlist('mimetype') or [0] excludeunderlay = form.get('excludeunderlay', 0) nosystemitems = form.get('nosystemitems', 0) historysearch = form.get('historysearch', 0) mtime = form.get('mtime', '') if mtime: mtime_parsed = None # get mtime from known date/time formats for fmt in (request.user.datetime_fmt, request.cfg.datetime_fmt, request.user.date_fmt, request.cfg.date_fmt): try: mtime_parsed = time.strptime(mtime, fmt) except ValueError: continue else: break if mtime_parsed: mtime = time.mktime(mtime_parsed) else: # didn't work, let's try parsedatetime cal = Calendar() mtime_parsed, parsed_what = cal.parse(mtime) # XXX it is unclear if usage of localtime here and in parsedatetime module is correct. # time.localtime is the SERVER's local time and of no relevance to the user (being # somewhere in the world) # mktime is reverse function for localtime, so this maybe fixes it again!? if parsed_what > 0 and mtime_parsed <= time.localtime(): mtime = time.mktime(mtime_parsed) else: mtime_parsed = None # we don't use invalid stuff # show info if mtime_parsed: # XXX mtime_msg is not shown in some cases mtime_msg = _("(!) Only pages changed since '''%s''' are being displayed!", wiki=True) % request.user.getFormattedDateTime(mtime) else: mtime_msg = _('/!\\ The modification date you entered was not ' 'recognized and is therefore not considered for the ' 'search results!', wiki=True) else: mtime_msg = None word_re = re.compile(r'(\"[\w\s]+"|\w+)', re.UNICODE) needle = '' if categories[0]: needle += 'category:%s ' % ','.join(categories) if language[0]: needle += 'language:%s ' % ','.join(language) if mimetype[0]: needle += 'mimetype:%s ' % ','.join(mimetype) if excludeunderlay: needle += '-domain:underlay ' if nosystemitems: needle += '-domain:system ' if and_terms: needle += '(%s) ' % and_terms if not_terms: needle += '(%s) ' % ' '.join(['-%s' % t for t in word_re.findall(not_terms)]) if or_terms: needle += '(%s) ' % ' or '.join(word_re.findall(or_terms)) # check for sensible search term stripped = needle.strip() if len(stripped) == 0: request.theme.add_msg(_('Please use a more selective search term instead ' 'of {{{"%s"}}}', wiki=True) % wikiutil.escape(needle), "error") Page(request, pagename).send_page() return needle = stripped # Setup for type of search if titlesearch: title = _('Title Search: "%s"') sort = 'page_name' else: if advancedsearch: title = _('Advanced Search: "%s"') else: title = _('Full Text Search: "%s"') sort = 'weight' # search the pages from MoinMoin.search import searchPages, QueryParser, QueryError try: query = QueryParser(case=case, regex=regex, titlesearch=titlesearch).parse_query(needle) except QueryError: # catch errors in the search query request.theme.add_msg(_('Your search query {{{"%s"}}} is invalid. Please refer to ' 'HelpOnSearching for more information.', wiki=True, percent=True) % wikiutil.escape(needle), "error") Page(request, pagename).send_page() return results = searchPages(request, query, sort, mtime, historysearch) # directly show a single hit for title searches # this is the "quick jump" functionality if you don't remember # the pagename exactly, but just some parts of it if titlesearch and len(results.hits) == 1: page = results.hits[0] if not page.attachment: # we did not find an attachment page = Page(request, page.page_name) querydict = {} if highlight_pages: highlight = query.highlight_re() if highlight: querydict.update({'highlight': highlight}) url = page.url(request, querystr=querydict) request.http_redirect(url) return if not results.hits: # no hits? f = request.formatter querydict = wikiutil.parseQueryString(request.query_string).to_dict() querydict.update({'titlesearch': 0}) request.theme.add_msg(_('Your search query {{{"%s"}}} didn\'t return any results. ' 'Please change some terms and refer to HelpOnSearching for ' 'more information.%s', wiki=True, percent=True) % (wikiutil.escape(needle), titlesearch and ''.join([ '<br>', _('(!) Consider performing a', wiki=True), ' ', f.url(1, href=request.page.url(request, querydict, escape=0)), _('full-text search with your search terms'), f.url(0), '.', ]) or ''), "error") Page(request, pagename).send_page() return # This action generates data using the user language request.setContentLanguage(request.lang) request.theme.send_title(title % needle, pagename=pagename) # Start content (important for RTL support) request.write(request.formatter.startContent("content")) # Hints f = request.formatter hints = [] if titlesearch: querydict = wikiutil.parseQueryString(request.query_string).to_dict() querydict.update({'titlesearch': 0}) hints.append(''.join([ _("(!) You're performing a title search that might not include" ' all related results of your search query in this wiki. <<BR>>', wiki=True), ' ', f.url(1, href=request.page.url(request, querydict, escape=0)), f.text(_('Click here to perform a full-text search with your ' 'search terms!')), f.url(0), ])) if advancedsearch and mtime_msg: hints.append(mtime_msg) if hints: request.write(searchHints(f, hints)) # Search stats request.write(results.stats(request, request.formatter, hitsFrom)) # Then search results info = not titlesearch if context: output = results.pageListWithContext(request, request.formatter, info=info, context=context, hitsFrom=hitsFrom, hitsInfo=1, highlight_titles=highlight_titles, highlight_pages=highlight_pages) else: output = results.pageList(request, request.formatter, info=info, hitsFrom=hitsFrom, hitsInfo=1, highlight_titles=highlight_titles, highlight_pages=highlight_pages) request.write(output) request.write(request.formatter.endContent()) request.theme.send_footer(pagename) request.theme.send_closing_html()
def formatPageLinks(self, hitsFrom, hitsPerPage, hitsNum): """ Format previous and next page links in page @param hitsFrom: current position in the hits @param hitsPerPage: number of hits per page @param hitsNum: number of hits @rtype: unicode @return: links to previous and next pages (if exist) """ _ = self.request.getText f = self.formatter querydict = dict(wikiutil.parseQueryString(self.request.query_string)) def page_url(n): querydict.update({'from': n * hitsPerPage}) return self.request.page.url(self.request, querydict, escape=0) pages = hitsNum // hitsPerPage remainder = hitsNum % hitsPerPage if remainder: pages += 1 cur_page = hitsFrom // hitsPerPage textlinks = [] # previous page available if cur_page > 0: textlinks.append(''.join([ f.url(1, href=page_url(cur_page-1)), f.text(_('Previous')), f.url(0)])) else: textlinks.append('') # list of pages to be shown page_range = range(*( cur_page - 5 < 0 and (0, pages > 10 and 10 or pages) or (cur_page - 5, cur_page + 6 > pages and pages or cur_page + 6))) textlinks.extend([''.join([ i != cur_page and f.url(1, href=page_url(i)) or '', f.text(str(i+1)), i != cur_page and f.url(0) or '', ]) for i in page_range]) # next page available if cur_page < pages - 1: textlinks.append(''.join([ f.url(1, href=page_url(cur_page+1)), f.text(_('Next')), f.url(0)])) else: textlinks.append('') return ''.join([ f.table(1, attrs={'tableclass': 'searchpages'}), f.table_row(1), f.table_cell(1), # textlinks (f.table_cell(0) + f.table_cell(1)).join(textlinks), f.table_cell(0), f.table_row(0), f.table(0), ])
def execute(pagename, request, fieldname='value', titlesearch=0, statistic=0): _ = request.getText titlesearch = checkTitleSearch(request) if titlesearch < 0: check_surge_protect(request, kick=True) # get rid of spammer return advancedsearch = isAdvancedSearch(request) form = request.values # context is relevant only for full search if titlesearch: context = 0 elif advancedsearch: context = 180 # XXX: hardcoded context count for advancedsearch else: context = int(form.get('context', 0)) # Get other form parameters needle = form.get(fieldname, '') case = int(form.get('case', 0)) regex = int(form.get('regex', 0)) # no interface currently hitsFrom = int(form.get('from', 0)) mtime = None msg = '' historysearch = 0 # if advanced search is enabled we construct our own search query if advancedsearch: and_terms = form.get('and_terms', '').strip() or_terms = form.get('or_terms', '').strip() not_terms = form.get('not_terms', '').strip() #xor_terms = form.get('xor_terms', '').strip() categories = form.getlist('categories') or [''] timeframe = form.get('time', '').strip() language = form.getlist('language') or [''] mimetype = form.getlist('mimetype') or [0] excludeunderlay = form.get('excludeunderlay', 0) nosystemitems = form.get('nosystemitems', 0) historysearch = form.get('historysearch', 0) mtime = form.get('mtime', '') if mtime: mtime_parsed = None # get mtime from known date/time formats for fmt in (request.user.datetime_fmt, request.cfg.datetime_fmt, request.user.date_fmt, request.cfg.date_fmt): try: mtime_parsed = time.strptime(mtime, fmt) except ValueError: continue else: break if mtime_parsed: mtime = time.mktime(mtime_parsed) else: # didn't work, let's try parsedatetime cal = Calendar() mtime_parsed, parsed_what = cal.parse(mtime) # XXX it is unclear if usage of localtime here and in parsedatetime module is correct. # time.localtime is the SERVER's local time and of no relevance to the user (being # somewhere in the world) # mktime is reverse function for localtime, so this maybe fixes it again!? if parsed_what > 0 and mtime_parsed <= time.localtime(): mtime = time.mktime(mtime_parsed) else: mtime_parsed = None # we don't use invalid stuff # show info if mtime_parsed: # XXX mtime_msg is not shown in some cases mtime_msg = _( "(!) Only pages changed since '''%s''' are being displayed!", wiki=True) % request.user.getFormattedDateTime(mtime) else: mtime_msg = _( '/!\\ The modification date you entered was not ' 'recognized and is therefore not considered for the ' 'search results!', wiki=True) else: mtime_msg = None word_re = re.compile(r'(\"[\w\s]+"|\w+)') needle = '' if categories[0]: needle += 'category:%s ' % ','.join(categories) if language[0]: needle += 'language:%s ' % ','.join(language) if mimetype[0]: needle += 'mimetype:%s ' % ','.join(mimetype) if excludeunderlay: needle += '-domain:underlay ' if nosystemitems: needle += '-domain:system ' if and_terms: needle += '(%s) ' % and_terms if not_terms: needle += '(%s) ' % ' '.join( ['-%s' % t for t in word_re.findall(not_terms)]) if or_terms: needle += '(%s) ' % ' or '.join(word_re.findall(or_terms)) # check for sensible search term stripped = needle.strip() if len(stripped) == 0: request.theme.add_msg( _( 'Please use a more selective search term instead ' 'of {{{"%s"}}}', wiki=True) % wikiutil.escape(needle), "error") Page(request, pagename).send_page() return needle = stripped # Setup for type of search if titlesearch: title = _('Title Search: "%s"') sort = 'page_name' else: if advancedsearch: title = _('Advanced Search: "%s"') else: title = _('Full Text Search: "%s"') sort = 'weight' # search the pages from MoinMoin.search import searchPages, QueryParser, QueryError try: query = QueryParser(case=case, regex=regex, titlesearch=titlesearch).parse_query(needle) except QueryError: # catch errors in the search query request.theme.add_msg( _( 'Your search query {{{"%s"}}} is invalid. Please refer to ' 'HelpOnSearching for more information.', wiki=True, percent=True) % wikiutil.escape(needle), "error") Page(request, pagename).send_page() return results = searchPages(request, query, sort, mtime, historysearch) # directly show a single hit for title searches # this is the "quick jump" functionality if you don't remember # the pagename exactly, but just some parts of it if titlesearch and len(results.hits) == 1: page = results.hits[0] if not page.attachment: # we did not find an attachment page = Page(request, page.page_name) highlight = query.highlight_re() if highlight: querydict = {'highlight': highlight} else: querydict = {} url = page.url(request, querystr=querydict) request.http_redirect(url) return if not results.hits: # no hits? f = request.formatter querydict = dict(wikiutil.parseQueryString(request.query_string)) querydict.update({'titlesearch': 0}) request.theme.add_msg( _( 'Your search query {{{"%s"}}} didn\'t return any results. ' 'Please change some terms and refer to HelpOnSearching for ' 'more information.%s', wiki=True, percent=True) % (wikiutil.escape(needle), titlesearch and ''.join([ '<br>', _('(!) Consider performing a', wiki=True), ' ', f.url(1, href=request.page.url(request, querydict, escape=0)), _('full-text search with your search terms'), f.url(0), '.', ]) or ''), "error") Page(request, pagename).send_page() return # This action generates data using the user language request.setContentLanguage(request.lang) request.theme.send_title(title % needle, pagename=pagename) # Start content (important for RTL support) request.write(request.formatter.startContent("content")) # Hints f = request.formatter hints = [] if titlesearch: querydict = dict(wikiutil.parseQueryString(request.query_string)) querydict.update({'titlesearch': 0}) hints.append(''.join([ _( "(!) You're performing a title search that might not include" ' all related results of your search query in this wiki. <<BR>>', wiki=True), ' ', f.url(1, href=request.page.url(request, querydict, escape=0)), f.text( _('Click here to perform a full-text search with your ' 'search terms!')), f.url(0), ])) if advancedsearch and mtime_msg: hints.append(mtime_msg) if hints: request.write(searchHints(f, hints)) # Search stats request.write(results.stats(request, request.formatter, hitsFrom)) # Then search results info = not titlesearch if context: output = results.pageListWithContext(request, request.formatter, info=info, context=context, hitsFrom=hitsFrom, hitsInfo=1) else: output = results.pageList(request, request.formatter, info=info, hitsFrom=hitsFrom, hitsInfo=1) request.write(output) request.write(request.formatter.endContent()) request.theme.send_footer(pagename) request.theme.send_closing_html()
def testMakeQueryString(self): for qstr, in_str, in_unicode in self.tests: assert wikiutil.parseQueryString(wikiutil.makeQueryString(in_unicode)) == MultiDict(in_unicode) assert wikiutil.parseQueryString(wikiutil.makeQueryString(in_str)) == MultiDict(in_unicode)
def testParseQueryString(self): for qstr, expected_str, expected_unicode in self.tests: assert wikiutil.parseQueryString(qstr) == MultiDict(expected_unicode) assert wikiutil.parseQueryString(unicode(qstr)) == MultiDict(expected_unicode)
def formatPageLinks(self, hitsFrom, hitsPerPage, hitsNum): """ Format previous and next page links in page @param hitsFrom: current position in the hits @param hitsPerPage: number of hits per page @param hitsNum: number of hits @rtype: unicode @return: links to previous and next pages (if exist) """ f = self.formatter querydict = dict(wikiutil.parseQueryString(self.request.query_string)) def page_url(n): querydict.update({'from': n * hitsPerPage}) return XXX.page.url(self.request, querydict, escape=0) pages = hitsNum // hitsPerPage remainder = hitsNum % hitsPerPage if remainder: pages += 1 cur_page = hitsFrom // hitsPerPage textlinks = [] # previous page available if cur_page > 0: textlinks.append(''.join([ f.url(1, href=page_url(cur_page-1)), f.text(_('Previous')), f.url(0)])) else: textlinks.append('') # list of pages to be shown page_range = range(*( cur_page - 5 < 0 and (0, pages > 10 and 10 or pages) or (cur_page - 5, cur_page + 6 > pages and pages or cur_page + 6))) textlinks.extend([''.join([ i != cur_page and f.url(1, href=page_url(i)) or '', f.text(str(i+1)), i != cur_page and f.url(0) or '', ]) for i in page_range]) # next page available if cur_page < pages - 1: textlinks.append(''.join([ f.url(1, href=page_url(cur_page+1)), f.text(_('Next')), f.url(0)])) else: textlinks.append('') return ''.join([ f.table(1, attrs={'tableclass': 'searchpages'}), f.table_row(1), f.table_cell(1), # textlinks (f.table_cell(0) + f.table_cell(1)).join(textlinks), f.table_cell(0), f.table_row(0), f.table(0), ])
def testMakeQueryString(self): for qstr, in_str, in_unicode in self.tests: assert wikiutil.parseQueryString(wikiutil.makeQueryString(in_unicode, want_unicode=False), want_unicode=False) == in_str assert wikiutil.parseQueryString(wikiutil.makeQueryString(in_str, want_unicode=False), want_unicode=False) == in_str assert wikiutil.parseQueryString(wikiutil.makeQueryString(in_unicode, want_unicode=True), want_unicode=True) == in_unicode assert wikiutil.parseQueryString(wikiutil.makeQueryString(in_str, want_unicode=True), want_unicode=True) == in_unicode
def testParseQueryString(self): for qstr, expected_str, expected_unicode in self.tests: assert wikiutil.parseQueryString(qstr, want_unicode=False) == expected_str assert wikiutil.parseQueryString(qstr, want_unicode=True) == expected_unicode assert wikiutil.parseQueryString(unicode(qstr), want_unicode=False) == expected_str assert wikiutil.parseQueryString(unicode(qstr), want_unicode=True) == expected_unicode