def save_a_bunch(count=1000):
import time
db_name = 'abunch'
dabase.init(db_name)
start = time.time()
page_ids = get_dab_page_ids(count=count)
pages = green_call_list(get_articles, page_ids)
dabblets = sum([ get_dabblets(p) for p in pages ], [])
# TODO start transaction
for d in dabblets:
d.save()
all_choices = green_call_list(get_dab_choices, dabblets)
for c in all_choices:
c.save()
# TODO end transaction
end = time.time()
print len(dabblets), 'Dabblets saved to', db_name, 'in', end-start, 'seconds'
print len(set([d.title for d in dabblets])), 'unique titles'
print len(set([d.source_title for d in dabblets])), 'unique source pages'
print len(all_choices), 'dabblet choices fetched and saved.'
print Dabblet.select().count(), 'total records in database'
print len(set([d.title for d in Dabblet.select()])), 'unique titles in database'
return dabblets
choice=choice,
solver_ip=request.get('REMOTE_ADDR'),
solver_index=session.get('cur_index', 0),
date_solved=datetime.now())
sol.save()
# replace?
view_count = DabSolution.select().count()
pass_count = DabSolution.select().where(choice_id=None).count()
return { "view_count": view_count,
"solution_count": view_count-pass_count }
@route('/random/')
def get_random_dabblet():
rdabs = Dabblet.select().order_by("RANDOM()").limit(2)
return { 'dabs': [ d.jsondict for d in rdabs ] }
class SlashMiddleware(object):
def __init__(self, app):
self.app = app
def __call__(self, e, h):
e['PATH_INFO'] = e['PATH_INFO'].rstrip('/')+'/'
return self.app(e,h)
if __name__ == '__main__':
dabase.init('abunch')
ALL_DABBLETS = [ (d.id, d.priority) for d in Dabblet.select(['id','priority']) ]
app = SlashMiddleware(bottle.app())
run(app=app, host='0.0.0.0', port=8080, server='gevent')
help="suppress output (TODO)")
return parser.parse_args()
if __name__ == '__main__':
import time
opts, args = parse_args()
start = time.time()
if opts.get_all:
print 'Getting all Dabblets.',
opts.limit = None
else:
print 'Searching up to',opts.limit,'articles for Dabblets.'
print 'Using', opts.concurrency, 'green threads. Saving to', opts.database
dabase.init(opts.database)
try:
dabblets = save_a_bunch(count=opts.limit,
category=opts.category,
concurrency=opts.concurrency,
per_call=opts.grouping,
db_name=opts.database)
except Exception as e:
#if opts.debug: #TODO
# import pdb;pdb.pm()
raise
end = time.time()
print len(dabblets), 'Dabblets saved in', end-start, 'seconds'
