def main(): store = models.init_store() fp = bz2.BZ2File('/home/data/dedupped-1-comment-per-line.json.bz2', 'r', 1048576 * 10) #fp = bz2.BZ2File('/home/data/top4e4.json.bz2', 'r', 1048576 * 10) #fp = bz2.BZ2File('/home/data/top5k.json.bz2', 'r', 1048576 * 10) stats = { 'all_comments': 0, 'comments': 0, 'io_error': 0, 'links': 0, 'reddits': 0, 'users': 0, 'files': 0, 'orphans': 0, } more = True while more: more = store.in_txn(lambda: process_set(stats, fp), write=True)
import itertools from time import time as now import models BOOL = [False, True] models.init_store().begin().__enter__() benches = [] def benchmark(**params): def nothing(func): benches.append((list(sorted(params.iteritems())), func)) return func return nothing def run(): for params, func in benches: configs = ([(name, v) for v in values] for name, values in params) for config in itertools.product(*configs): name = func.__name__ if config: name += '<%s>' % ( ', '.join('%s=%s' % (k, v) for (k, v) in config)) benchit(name, lambda: func(**dict(config))) print
import urllib import acid import acid.meta import bottle import wheezy.template.engine import wheezy.template.ext.core import wheezy.template.loader import models templates = wheezy.template.engine.Engine( loader=wheezy.template.loader.FileLoader(['templates']), extensions=[wheezy.template.ext.core.CoreExtension()]) store = models.init_store() # Hack to avoid attempting to create Post collection from RO txn with store.begin(write=True): list(models.Post.iter()) models.Post.find() def getint(name, default=None): try: return int(bottle.request.params.get(name)) except (ValueError, TypeError): return default @bottle.route('/') def index():
import itertools from time import time as now import models BOOL = [False, True] models.init_store().begin().__enter__() benches = [] def benchmark(**params): def nothing(func): benches.append((list(sorted(params.iteritems())), func)) return func return nothing def run(): for params, func in benches: configs = ([(name, v) for v in values] for name, values in params) for config in itertools.product(*configs): name = func.__name__ if config: name += "<%s>" % (", ".join("%s=%s" % (k, v) for (k, v) in config)) benchit(name, lambda: func(**dict(config))) print def benchit(s, func):