def save_a_bunch(count=1000):
    import time

    db_name = 'abunch'
    dabase.init(db_name)

    start = time.time()

    page_ids = get_dab_page_ids(count=count)

    pages    = green_call_list(get_articles, page_ids)
    dabblets = sum([ get_dabblets(p) for p in pages ], [])

    # TODO start transaction
    for d in dabblets:
        d.save()

    all_choices = green_call_list(get_dab_choices, dabblets)

    for c in all_choices:
        c.save()
    # TODO end transaction 
    end = time.time()

    print len(dabblets), 'Dabblets saved to', db_name, 'in', end-start, 'seconds'
    print len(set([d.title for d in dabblets])), 'unique titles'
    print len(set([d.source_title for d in dabblets])), 'unique source pages'
    print len(all_choices), 'dabblet choices fetched and saved.'

    print Dabblet.select().count(), 'total records in database'
    print len(set([d.title for d in Dabblet.select()])), 'unique titles in database'

    return dabblets
Exemple #2
0
                      choice=choice,
                      solver_ip=request.get('REMOTE_ADDR'),
                      solver_index=session.get('cur_index', 0),
                      date_solved=datetime.now())
    sol.save()
    # replace?

    view_count = DabSolution.select().count()
    pass_count = DabSolution.select().where(choice_id=None).count()
    return { "view_count": view_count,
             "solution_count": view_count-pass_count }
    
@route('/random/')
def get_random_dabblet():
    rdabs = Dabblet.select().order_by("RANDOM()").limit(2)
    
    return { 'dabs': [ d.jsondict for d in rdabs ] }

class SlashMiddleware(object):
    def __init__(self, app):
        self.app = app
    def __call__(self, e, h):
        e['PATH_INFO'] = e['PATH_INFO'].rstrip('/')+'/'
        return self.app(e,h)

if __name__ == '__main__':
    dabase.init('abunch')
    ALL_DABBLETS = [ (d.id, d.priority) for d in Dabblet.select(['id','priority']) ]
    app = SlashMiddleware(bottle.app())
    run(app=app, host='0.0.0.0', port=8080, server='gevent')
Exemple #3
0
                      help="suppress output (TODO)")
    return parser.parse_args()


if __name__ == '__main__':
    import time
    opts, args = parse_args()
    start = time.time()
    if opts.get_all:
        print 'Getting all Dabblets.',
        opts.limit = None
    else:
        print 'Searching up to',opts.limit,'articles for Dabblets.'
    print 'Using', opts.concurrency, 'green threads. Saving to', opts.database

    dabase.init(opts.database)

    try:
        dabblets = save_a_bunch(count=opts.limit,
                                category=opts.category,
                                concurrency=opts.concurrency,
                                per_call=opts.grouping,
                                db_name=opts.database)
    except Exception as e:
        #if opts.debug: #TODO
        #    import pdb;pdb.pm()
        raise

    end = time.time()
    print len(dabblets), 'Dabblets saved in', end-start, 'seconds'