def runV2():
db = shelve.open("./neu.db")
print len(db)
raise SystemExit
# for no in xrange(20130001, 20133333):
for no in xrange(20135000 - 100, 20136000):
no = str(no)
if no in db:
print no, db[no], "√"
continue
try:
t = LibraryV2(no)
t.login()
print t.userno, t.name
db[t.userno] = t.name
db.sync()
time.sleep(0.3)
except KeyboardInterrupt:
print "Interrupted!"
raise SystemExit
except:
print no, "......", "×"
db.close()
def run(gen):
hosts = ('bw64node01','bw64node02','bw64node03','bw64node04')
h = cluster.startZeoClients(hosts, gen)
assert len(h) == len(hosts)
r = db.connect()
g = r[gen]
while g.gen_num < g.final_gen_num or g.leftToEval():
time.sleep(5)
db.sync()
cluster.stopZeoClients(hosts)
def dbset(g, k, v):
db.sync()
c = 0
while 1:
try:
setattr(g, k, v)
transaction.commit()
break
except:
c += 1
if c > 10:
raise
def run():
try:
main = db.soft_sys.main()
main.init()
except:
util.log_exc()
sys.exit(1) # don't sync..
s_next = 0.0
t_last = time()
# t = now
# t_last = start of last tick
# next = scheduled start of next tick
# s_next = scheduled time of next cache sync
try:
while 1:
try:
main.tick()
except:
util.log_exc()
# Calculate next tick time
next = t_last + tick_t
# Wait for next tick, doing I/O
t = time()
while t < next:
comm.idle(next - t)
t = time()
# Don't use t; we don't want drift here
t_last = next
# Check for cache sync
if t > s_next:
s_next = t + sync_t
db.sync()
except:
db.sync()
raise
def run():
try:
main = db.soft_sys.main()
main.init()
except:
util.log_exc()
sys.exit(1) # don't sync..
s_next = 0.0
t_last = time()
# t = now
# t_last = start of last tick
# next = scheduled start of next tick
# s_next = scheduled time of next cache sync
try:
while 1:
try: main.tick()
except: util.log_exc()
# Calculate next tick time
next = t_last + tick_t
# Wait for next tick, doing I/O
t = time()
while t < next:
comm.idle(next - t)
t = time()
# Don't use t; we don't want drift here
t_last = next
# Check for cache sync
if t > s_next:
s_next = t + sync_t
db.sync()
except:
db.sync()
raise
def runRateTest(gn, single=0):
# run test with 1,2,4,8,16,32,62 client nodes. The actual hosts are
# hardcoded as cluster.HOSTS but cluster.MASTER is removed.
f = open('test/rate.txt', 'w')
f.write('pcs\teh\n')
r = db.connect()
g = r[gn]
nh = [4,8,15,30,45,59]
nh.reverse()
if single:
nh = [1]
for n in nh:
dbset(g, 'pause', 1)
debug('n=%d',n)
hosts = list(set(cluster.HOSTS) - set([cluster.MASTER]))
hosts = hosts[:n]
if single:
hosts = 'localhost',
debug('%d hosts : %s', len(hosts), hosts)
assert len(hosts) == n
for x in g:
dbset(x, 'score', None)
dbset(g, 'updateRate', 0)
dbset(g, 'updateInfo', ('reset', time.time(), 0))
h = cluster.startZeoClients(hosts, gn)
assert len(h) == len(hosts)
starttime = time.time()
dbset(g, 'pause', 0)
debug('unpaused clients')
while g.leftToEval():
time.sleep(10)
db.sync()
endtime = time.time()
runtime = endtime - starttime
e = len(g)
debug('%d evals in %d seconds', e, runtime)
eh = e*(60*60)/runtime
f.write('%d\t%d\n'%(n, eh))
f.flush()
cluster.stopZeoClients(hosts)
f.close()
def persist(self):
db.set(self.db_key(), self.to_json())
db.sync()
def shutdown():
log('util.shutdown() called')
db.sync()
# Exit.
sys.exit(0)
def persist(self):
db.set(self.db_key(), self.to_json())
db.sync()
from flask import Flask
from flask_restful import Resource, Api
import sqlite3
from db import sync
from settings import QUERYSET_COUNT
app = Flask(__name__)
api = Api(app)
class Wind(Resource):
def get(self, direction):
conn = sqlite3.connect('winds.db')
c = conn.cursor()
c.execute('''SELECT * FROM winds
WHERE direction = '{}'
ORDER BY speed DESC
LIMIT {}'''.format(direction, QUERYSET_COUNT))
return c.fetchall()
api.add_resource(Wind, '/<string:direction>')
if __name__ == '__main__':
sync()
app.run()
def persist(self):
self.seen_now()
db.set(self.db_key(), self.to_json())
db.sync()
def shutdown():
log('util.shutdown() called')
db.sync()
# Exit.
sys.exit(0)
def persist(self):
self.seen_now()
db.set(self.db_key(), self.to_json())
db.sync()
rate = root[g].updateRate
print 'Generation: name=%s ga=%s gen=%d/%d fitness=%s evh=%d' % (
g, root[g].ga, root[g].gen_num, root[g].final_gen_num, fn,
rate)
if root[g].updateInfo.updating:
print 'Update started on %s %d seconds ago' % (
root[g].updateInfo.host, root[g].updateInfo.elapsed)
if client or master:
h = {(1, 0): 'Master', (0, 1): 'Client', (1, 1): 'Master && Client'}
mode = h[master, client]
log.info('%s running on %s', mode, cluster.getHostname())
while 1:
# find all generations that aren't finished
db.sync()
if g:
runs = [g]
else:
runs = [
k for (k, i) in root.iteritems()
if isinstance(i, evolve.Generation)
]
done = []
done += [
r for r in runs if root[r].ga == 'steadystate'
and len(root[r].scores) == root[r].final_gen_num
]
done += [
r for r in runs if root[r].ga != 'steadystate'
and len(root[r].scores) == root[r].final_gen_num + 1
