Exemplo n.º 1
0
def runV2():
    db = shelve.open("./neu.db")

    print len(db)
    raise SystemExit
    # for no in xrange(20130001, 20133333):
    for no in xrange(20135000 - 100, 20136000):
        no = str(no)
        if no in db:
            print no, db[no], "√"
            continue
        try:
            t = LibraryV2(no)
            t.login()
            print t.userno, t.name
            db[t.userno] = t.name
            db.sync()
            time.sleep(0.3)
        except KeyboardInterrupt:
            print "Interrupted!"
            raise SystemExit
        except:
            print no, "......", "×"

    db.close()
Exemplo n.º 2
0
def run(gen):
    hosts = ('bw64node01','bw64node02','bw64node03','bw64node04')
    h = cluster.startZeoClients(hosts, gen)
    assert len(h) == len(hosts)
    r = db.connect()
    g = r[gen]
    while g.gen_num < g.final_gen_num or g.leftToEval():
        time.sleep(5)
        db.sync()
    cluster.stopZeoClients(hosts)
Exemplo n.º 3
0
def dbset(g, k, v):
    db.sync()
    c = 0
    while 1:
        try:
            setattr(g, k, v)
            transaction.commit()
            break
        except:
           c += 1
           if c > 10:
                raise
Exemplo n.º 4
0
def run():
    try:
        main = db.soft_sys.main()
        main.init()
    except:
        util.log_exc()
        sys.exit(1)  # don't sync..

    s_next = 0.0
    t_last = time()

    # t = now
    # t_last = start of last tick
    # next = scheduled start of next tick
    # s_next = scheduled time of next cache sync

    try:
        while 1:
            try:
                main.tick()
            except:
                util.log_exc()

            # Calculate next tick time
            next = t_last + tick_t

            # Wait for next tick, doing I/O

            t = time()
            while t < next:
                comm.idle(next - t)
                t = time()

            # Don't use t; we don't want drift here
            t_last = next

            # Check for cache sync
            if t > s_next:
                s_next = t + sync_t
                db.sync()

    except:
        db.sync()
        raise
Exemplo n.º 5
0
def run():
    try:
        main = db.soft_sys.main()
        main.init()
    except:
        util.log_exc()
        sys.exit(1) # don't sync..

    s_next = 0.0
    t_last = time()

    # t = now
    # t_last = start of last tick
    # next = scheduled start of next tick
    # s_next = scheduled time of next cache sync

    try:
        while 1:        
            try: main.tick()
            except: util.log_exc()
            
            # Calculate next tick time
            next = t_last + tick_t
            
            # Wait for next tick, doing I/O
            
            t = time()
            while t < next:
                comm.idle(next - t)
                t = time()
                
            # Don't use t; we don't want drift here
            t_last = next
            
            # Check for cache sync
            if t > s_next:
                s_next = t + sync_t
                db.sync()

    except:
        db.sync()
        raise
Exemplo n.º 6
0
def runRateTest(gn, single=0):
    # run test with 1,2,4,8,16,32,62 client nodes. The actual hosts are
    # hardcoded as cluster.HOSTS but cluster.MASTER is removed.
    f = open('test/rate.txt', 'w')
    f.write('pcs\teh\n')
    r = db.connect()
    g = r[gn]
    nh = [4,8,15,30,45,59]
    nh.reverse()
    if single:
        nh = [1]
    for n in nh:
        dbset(g, 'pause', 1)
        debug('n=%d',n)
        hosts = list(set(cluster.HOSTS) - set([cluster.MASTER]))
        hosts = hosts[:n]
        if single:
            hosts = 'localhost',
        debug('%d hosts : %s', len(hosts), hosts)
        assert len(hosts) == n
        for x in g:
            dbset(x, 'score', None)
        dbset(g, 'updateRate', 0)
        dbset(g, 'updateInfo', ('reset', time.time(), 0))
        h = cluster.startZeoClients(hosts, gn)
        assert len(h) == len(hosts)
        starttime = time.time()
        dbset(g, 'pause', 0)
        debug('unpaused clients')
        while g.leftToEval():
            time.sleep(10)
            db.sync()
        endtime = time.time()
        runtime = endtime - starttime
        e = len(g)
        debug('%d evals in %d seconds', e, runtime)
        eh = e*(60*60)/runtime
        f.write('%d\t%d\n'%(n, eh))
        f.flush()
        cluster.stopZeoClients(hosts)
    f.close()
Exemplo n.º 7
0
 def persist(self):
     db.set(self.db_key(), self.to_json())
     db.sync()
Exemplo n.º 8
0
def shutdown():
    log('util.shutdown() called')

    db.sync()
    # Exit.
    sys.exit(0)
Exemplo n.º 9
0
 def persist(self):
     db.set(self.db_key(), self.to_json())
     db.sync()
Exemplo n.º 10
0
from flask import Flask
from flask_restful import Resource, Api

import sqlite3

from db import sync
from settings import QUERYSET_COUNT

app = Flask(__name__)
api = Api(app)


class Wind(Resource):
    def get(self, direction):
        conn = sqlite3.connect('winds.db')
        c = conn.cursor()
        c.execute('''SELECT * FROM winds 
                     WHERE direction = '{}'
                     ORDER BY speed DESC 
                     LIMIT {}'''.format(direction, QUERYSET_COUNT))
        return c.fetchall()


api.add_resource(Wind, '/<string:direction>')


if __name__ == '__main__':
    sync()
    app.run()
Exemplo n.º 11
0
 def persist(self):
     self.seen_now()
     db.set(self.db_key(), self.to_json())
     db.sync()
Exemplo n.º 12
0
def shutdown():
    log('util.shutdown() called')

    db.sync()
    # Exit.
    sys.exit(0)
Exemplo n.º 13
0
 def persist(self):
     self.seen_now()
     db.set(self.db_key(), self.to_json())
     db.sync()
Exemplo n.º 14
0
                rate = root[g].updateRate
            print 'Generation: name=%s ga=%s gen=%d/%d fitness=%s evh=%d' % (
                g, root[g].ga, root[g].gen_num, root[g].final_gen_num, fn,
                rate)
            if root[g].updateInfo.updating:
                print 'Update started on %s %d seconds ago' % (
                    root[g].updateInfo.host, root[g].updateInfo.elapsed)

    if client or master:
        h = {(1, 0): 'Master', (0, 1): 'Client', (1, 1): 'Master && Client'}
        mode = h[master, client]
        log.info('%s running on %s', mode, cluster.getHostname())
        while 1:

            # find all generations that aren't finished
            db.sync()
            if g:
                runs = [g]
            else:
                runs = [
                    k for (k, i) in root.iteritems()
                    if isinstance(i, evolve.Generation)
                ]
            done = []
            done += [
                r for r in runs if root[r].ga == 'steadystate'
                and len(root[r].scores) == root[r].final_gen_num
            ]
            done += [
                r for r in runs if root[r].ga != 'steadystate'
                and len(root[r].scores) == root[r].final_gen_num + 1