def search(self, ver=None, register=None):
id = None
try:
params = MultiDict(self._get_search_params(request.params))
if 'q' in params.keys():
id = params['q']
if 'query' in params.keys():
id = params['query']
except ValueError, e:
pass
def search(self, ver=None, register=None):
id = None
try:
params = MultiDict(self._get_search_params(request.params))
if 'q' in params.keys():
id = params['q']
if 'query' in params.keys():
id = params['query']
except ValueError, e:
log.debug(str(e))
pass
def search(self, ver=None, register=None):
id = None
try:
params = MultiDict(self._get_search_params(request.params))
if "q" in list(params.keys()):
id = params["q"]
if "query" in list(params.keys()):
id = params["query"]
except ValueError as e:
log.debug(str(e))
pass
self._post_analytics(c.user, register, "search", id)
return ApiController.search(self, ver, register)
class MimeHeaders(object):
"""Dictionary-like object that preserves the order and
supports multiple values for the same key, knows
whether it has been changed after the creation
"""
def __init__(self, items=()):
self._v = MultiDict([(normalize(key), remove_newlines(val))
for (key, val) in items])
self.changed = False
def __getitem__(self, key):
return self._v.get(normalize(key), None)
def __len__(self):
return len(self._v)
def __iter__(self):
return iter(self._v)
def __contains__(self, key):
return normalize(key) in self._v
def __setitem__(self, key, value):
self._v[normalize(key)] = remove_newlines(value)
self.changed = True
def __delitem__(self, key):
del self._v[normalize(key)]
self.changed = True
def __nonzero__(self):
return len(self._v) > 0
def prepend(self, key, value):
self._v._items.insert(0, (normalize(key), remove_newlines(value)))
self.changed = True
def add(self, key, value):
"""Adds header without changing the
existing headers with same name"""
self._v.add(normalize(key), remove_newlines(value))
self.changed = True
def keys(self):
"""
Returns the keys. (message header names)
It remembers the order in which they were added, what
is really important
"""
return self._v.keys()
def transform(self, fn):
"""Accepts a function, getting a key, val and returning
a new pair of key, val and applies the function to all
header, value pairs in the message.
"""
changed = [False]
def tracking_fn(key, val):
new_key, new_val = fn(key, val)
if new_val != val or new_key != key:
changed[0] = True
return new_key, new_val
v = MultiDict(tracking_fn(key, val) for key, val in self._v.iteritems())
if changed[0]:
self._v = v
self.changed = True
def items(self):
"""
Returns header,val pairs in the preserved order.
"""
return list(self.iteritems())
def iteritems(self):
"""
Returns iterator header,val pairs in the preserved order.
"""
return self._v.iteritems()
def get(self, key, default=None):
"""
Returns header value (case-insensitive).
"""
return self._v.get(normalize(key), default)
def getall(self, key):
"""
Returns all header values by the given header name
(case-insensitive)
"""
return self._v.getall(normalize(key))
def have_changed(self):
"""Tells whether someone has altered the headers
after creation"""
return self.changed
def __str__(self):
return str(self._v)
@classmethod
def from_stream(cls, stream):
"""Takes a stream and reads the headers,
decodes headers to unicode dict like object"""
return cls(parse_stream(stream))
def to_stream(self, stream):
"""Takes a stream and serializes headers
in a mime format"""
for h, v in self._v.iteritems():
try:
h = h.encode('ascii')
except UnicodeDecodeError:
raise EncodingError("Non-ascii header name")
stream.write("{0}: {1}\r\n".format(h, to_mime(h, v)))
class GoalAnalyser(object):
"""Takes a log file and some filters for URL specific stats and generates
CSV files with the result of the DailyStatistics filter"""
def __init__(self, log, location, urls=None, avgs=[1, 7], past_only=False,
config=None):
super(GoalAnalyser, self).__init__()
self.config = config
self.log = log
if urls is None:
if self.config is None:
self.urls = {}
else:
self.urls = config.urls()
else:
self.urls = urls
self.avgs = avgs
self.dir = location
self.past_only = past_only
self.statscounters = MultiDict()
self.outputs = {}
self.files = {}
self.existing_dates = {}
self.parse = logparser()
self.log_entries = {}
if self.config is not None:
for section in self.config.sections():
log = dict(self.config.items(section)).get('log', '').lower()
if log in ('true', 'yes', 'on'):
fn = '%s.log' % section
self.log_entries[section] = fn
self.files[fn] = open(os.path.join(self.dir, fn), 'w')
def _instantiateFilters(self):
for name in self.urls:
self.statscounters[name] = DailyStatistics(self.avgs)
def _instantiateCSVWriters(self):
keys = ['date', ] + sorted(DailyStatistics(self.avgs).stats().keys())
for name in self.urls:
location = os.path.join(self.dir, '%s_stats.csv' % name)
if os.path.exists(location):
# We are going to add to an existing file.
backing = open(location, 'r+')
reader = DictReader(backing)
self.existing_dates[name] = [r['date'] for r in reader]
else:
backing = open(location, 'w')
self.files[name] = backing
writer = DictWriter(backing, keys)
if self.existing_dates.get(name, None) is None:
writer.writerow(dict(zip(keys, keys)))
self.outputs[name] = writer
def filterForLine(self, line):
"""Take a parsed log line and return the rule name that it matches
or None if none match."""
status = int(line.get('status', '200'))
if status >= 400 and status < 600:
return
method = line['method']
url = line['url']
qs = line['querystring']
if qs is not None:
url += qs
for name in self.statscounters.keys():
method_name, url_pattern = self.urls[name]
if method_name != method:
continue
if url_pattern.match(url):
return name
warnings.warn("%s for %s is not classified" % (method, url))
def __call__(self):
self._instantiateFilters()
self._instantiateCSVWriters()
# We know the dates are in order, so parse them and groupby their date
iterable = itertools.imap(self.parse, self.log)
iterable = itertools.ifilter(lambda x: x is not None, iterable)
days = itertools.groupby(iterable, getDateForLine)
existing = self.existing_dates
# if avgs goes back several days we have to gather statistics
# for days in existing
today = datetime.date.today()
statsdays = set([str(today-datetime.timedelta(days=x)) for x in range(int(self.past_only),max(self.avgs)+int(self.past_only))])
for day, iterable in days:
if self.past_only and day == today:
continue
# Duplicate the iterator for each day, find the responsible rule
# name and turn it into a dictionary.iteritems() style iterator.
parsed, destination = itertools.tee(iterable)
destination = itertools.imap(self.filterForLine, destination)
classified = itertools.izip(destination, parsed)
for destination, entry in classified:
# Pass the line onto the underlying stats class
if day.isoformat() not in statsdays and day.isoformat() in existing.get(destination, []):
continue
if destination is None:
continue
self.statscounters[destination].process(entry, day)
fn = self.log_entries.get(destination)
if fn is not None:
entry_info = dict(entry)
entry_info['querystring'] = entry_info['querystring'] or ''
self.files[fn].write(LOG_FORMAT.format(**entry_info))
for name in self.urls:
# Don't duplicate dates in csv file
if day.isoformat() in set(sum(existing.values(), [])):
continue
stats = self.statscounters[name].stats()
if not any(stats.values()):
# We have no data at all, skip this
warnings.warn("No data for %s on %s" %
(name, day.isoformat()))
continue
stats['date'] = day.isoformat()
self.outputs[name].writerow(stats)
self.finish()
def finish(self):
for f in self.files.values():
f.flush()
os.fsync(f.fileno())
f.close()
class MimeHeaders(object):
"""Dictionary-like object that preserves the order and
supports multiple values for the same key, knows
whether it has been changed after the creation
"""
def __init__(self, items=()):
self.v = MultiDict([(normalize(key), val) for (key, val) in items])
self.changed = False
def __getitem__(self, key):
return self.v.get(normalize(key), None)
def __len__(self):
return len(self.v)
def __iter__(self):
return iter(self.v)
def __contains__(self, key):
return normalize(key) in self.v
def __setitem__(self, key, value):
self.v[normalize(key)] = _remove_newlines(value)
self.changed = True
def __delitem__(self, key):
del self.v[normalize(key)]
self.changed = True
def __nonzero__(self):
return len(self.v) > 0
def prepend(self, key, val):
self.v._items.insert(0, (key, _remove_newlines(val)))
self.changed = True
def add(self, key, value):
"""Adds header without changing the
existing headers with same name"""
self.v.add(normalize(key), _remove_newlines(value))
self.changed = True
def keys(self):
"""
Returns the keys. (message header names)
It remembers the order in which they were added, what
is really important
"""
return self.v.keys()
def transform(self, fn):
"""Accepts a function, getting a key, val and returning
a new pair of key, val and applies the function to all
header, value pairs in the message.
"""
changed = [False]
def tracking_fn(key, val):
new_key, new_val = fn(key, val)
if new_val != val or new_key != key:
changed[0] = True
return new_key, new_val
v = MultiDict(tracking_fn(key, val) for key, val in self.v.iteritems())
if changed[0]:
self.v = v
self.changed = True
def items(self):
"""
Returns header,val pairs in the preserved order.
"""
return list(self.iteritems())
def iteritems(self):
"""
Returns iterator header,val pairs in the preserved order.
"""
return self.v.iteritems()
def get(self, key, default=None):
"""
Returns header value (case-insensitive).
"""
return self.v.get(normalize(key), default)
def getall(self, key):
"""
Returns all header values by the given header name
(case-insensitive)
"""
return self.v.getall(normalize(key))
def have_changed(self):
"""Tells whether someone has altered the headers
after creation"""
return self.changed
def __str__(self):
return str(self.v)
@classmethod
def from_stream(cls, stream):
"""Takes a stream and reads the headers,
decodes headers to unicode dict like object"""
return cls(parse_stream(stream))
def to_stream(self, stream):
"""Takes a stream and serializes headers
in a mime format"""
for h, v in self.v.iteritems():
try:
h = h.encode('ascii')
except UnicodeDecodeError:
raise EncodingError("Non-ascii header name")
stream.write("{0}: {1}\r\n".format(h, to_mime(h, v)))
