Exemplo n.º 1
0
def record_runtime(db,spider,start,end):
    duration = (end-start).total_seconds()
    mins = duration / 60
    seconds = duration % 60
    timespent = repr(math.trunc(mins)) + 'mins ' + repr(math.trunc(seconds)) + 'seconds'
    if spider in db['runtime']:
        runtime = db['runtime'][spider]
        if duration > runtime['max_seconds']:
            runtime['max_seconds'] = duration
        if duration < runtime['min_seconds']:
            runtime['min_seconds'] = duration
    else:
        runtime = {'name':spider}
        runtime['max_seconds'] = duration
        runtime['min_seconds'] = duration
    runtime['timespent'] = timespent
    runtime['start'] = repr(start)
    runtime['end'] = repr(end)
    db['runtime'][spider] = runtime
    db.sync()
    viewstates.publishStates(db)
Exemplo n.º 2
0
def on_start_crawl(db,spider):
    print 'starting to crawl ', spider, ' on ', repr(datetime.now())
    db['crawling'] = spider
    db.sync()
    viewstates.publishStates(db)
    return datetime.now()