def smtp_session (ip, port, nm, nr):
s = smtp_client()
s.connect ((ip, port))
s.get_response()
sys.stderr.write ('+')
for x in range(nm):
code, text = s.command ('MAIL FROM:<*****@*****.**>')
for y in range(nr):
code, text = s.command ('RCPT TO:<*****@*****.**>' %y)
# we need to check the reply codes...
code, text = s.command ('DATA')
s.send (
"Subject: testing\r\n\r\n"
"This is message #%d\r\n" % x +
"BCNU\r\n"
"\r\n.\r\n"
)
code, text = s.get_response()
sys.stderr.write ('m')
s.command ('QUIT')
sys.stderr.write('-')
global count
count = count - 1
#print 'count =',count
if count == 0:
coro.set_exit()
def watcher(thread_ids):
while True:
if len(thread_ids) == 0:
coro.set_exit()
break
thread_ids = [x for x in thread_ids if x in coro.all_threads]
coro.sleep_relative(0.1)
def smtp_session (ip, port, nm, nr):
s = smtp_client()
s.connect ((ip, port))
s.get_response()
sys.stderr.write ('+')
for x in range(nm):
code, text = s.command ('MAIL FROM:<*****@*****.**>')
for y in range(nr):
code, text = s.command ('RCPT TO:<*****@*****.**>' % y)
# we need to check the reply codes...
code, text = s.command ('DATA')
s.send (
"Subject: testing\r\n\r\n"
"This is message #%d\r\n" % x +
"BCNU\r\n"
"\r\n.\r\n"
)
code, text = s.get_response()
sys.stderr.write ('m')
s.command ('QUIT')
sys.stderr.write('-')
global count
count = count - 1
# print 'count =',count
if count == 0:
coro.set_exit()
def go(G, s):
try:
while 1:
# what are the per-txn size limits?
pktlen = s.recv_exact(4)
if not pktlen:
break
else:
pktlen, = struct.unpack('>I', pktlen)
packet = s.recv_exact(pktlen)
data, size = decode(packet)
assert size == pktlen
[index, block_timestamp, raw_tx, lock_scripts] = data
tx = TX()
tx.unpack(raw_tx)
result = True
for i in range(len(tx.inputs)):
lock_script = lock_scripts[i]
try:
tx.verify(i, lock_script, block_timestamp)
except SystemError:
result = False
pkt = encode((result, index))
s.writev([struct.pack('>I', len(pkt)), pkt])
except EOFError:
pass
coro.set_exit()
def go (G):
global in_flight
addr0 = ('127.0.0.1', 0)
addr1 = parse_addr_arg (G.args.connect)
cp = ChainPuller (G, addr0, addr1)
cp.wait_for ('verack')
if G.args.getblocks:
W ('downloading block names...\n')
names = cp.getblocks()
else:
W ('downloading headers...\n')
names = cp.getheaders()
start = time.time()
W ('got %d names...\n' % (len(names),))
WB ('[BLOCKNUM] ')
WR ('(MB/s)\n')
for i in range (0, len (names), 50):
chunk = names[i:i+50]
cp.getdata ([(OBJ_BLOCK, name) for name in chunk])
in_flight += len(chunk)
in_flight_cv.wait()
# let it finish
while 1:
coro.sleep_relative (1)
if in_flight == 0:
break
stop = time.time()
WB (
'\ntransferred %.2f GB in %d secs (%.2f MB/s)\n' % (
float(total_bytes) / GB,
stop - start,
(float(total_bytes) / MB) / (stop - start)
)
)
coro.set_exit()
def watcher(thread_ids):
while True:
if len(thread_ids) == 0:
coro.set_exit()
break
thread_ids = [x for x in thread_ids if x in coro.all_threads]
coro.sleep_relative(0.1)
def go (G, s):
try:
while 1:
# what are the per-txn size limits?
pktlen = s.recv_exact (4)
if not pktlen:
break
else:
pktlen, = struct.unpack ('>I', pktlen)
packet = s.recv_exact (pktlen)
data, size = decode (packet)
assert size == pktlen
[index, block_timestamp, raw_tx, lock_scripts] = data
tx = TX()
tx.unpack (raw_tx)
result = True
for i in range (len (tx.inputs)):
lock_script = lock_scripts[i]
try:
tx.verify (i, lock_script, block_timestamp)
except SystemError:
result = False
pkt = encode ((result, index))
s.writev ([struct.pack ('>I', len(pkt)), pkt])
except EOFError:
pass
coro.set_exit()
def transport_thread(channel):
stdout = coro.fd_sock(1)
while not channel.eof and not channel.closed:
try:
data = channel.read(1024)
if data:
stdout.send(data)
# os.write(1, data)
except EOFError:
break
coro.set_exit()
def main(self):
try:
try:
self._main()
except SystemExit, e:
if e.code is not None:
if not isinstance(e.code, int):
print e.code
coro.set_exit()
finally:
coro.set_exit()
def transport_thread(channel):
stdout = coro.fd_sock (1)
while not channel.eof and not channel.closed:
try:
data = channel.read(1024)
if data:
stdout.send (data)
#os.write(1, data)
except EOFError:
break
coro.set_exit()
def main():
global exit_code
try:
try:
#p = unittest.TestProgram(runNow=False)
p = unittest.TestProgram()
# This should always be the last test case run.
p.test.addTest(ThreadFailedTest('test_threads_failed'))
p.runTests()
except SystemExit, e:
exit_code = e.code
finally:
coro.set_exit(exit_code)
def main():
global exit_code
try:
try:
# p = unittest.TestProgram(runNow=False)
p = unittest.TestProgram()
# This should always be the last test case run.
p.test.addTest(ThreadFailedTest('test_threads_failed'))
p.runTests()
except SystemExit as e:
exit_code = e.code
finally:
coro.set_exit(exit_code)
def t1():
# sample search
LOG('connect...')
c = client(('127.0.0.1', 389))
LOG('bind...')
c.simple_bind('', '')
LOG('search...')
r = c.search('dc=ldapserver,dc=example,dc=com', SCOPE.SUBTREE, DEREF.NEVER,
0, 0, 0, '(objectclass=*)', [])
LOG('unbinding...')
c.unbind()
from pprint import pprint
pprint(r)
coro.set_exit()
return r
def t1():
# sample search
LOG("connect...")
c = client(("127.0.0.1", 389))
LOG("bind...")
c.simple_bind("", "")
LOG("search...")
r = c.search("dc=ldapserver,dc=example,dc=com", SCOPE.SUBTREE, DEREF.NEVER, 0, 0, 0, "(objectclass=*)", [])
LOG("unbinding...")
c.unbind()
from pprint import pprint
pprint(r)
coro.set_exit()
return r
def client (ip='127.0.0.1', port=9000):
global alive
alive += 1
try:
s = coro.tcp_sock()
s.connect ((ip, port))
for i in range (10):
s.send ('howdy there\r\n')
assert (s.recv_exact (13) == 'howdy there\r\n')
coro.write_stderr ('.')
s.close()
finally:
alive -= 1
if alive == 0:
coro.write_stderr ('\ndone.\n')
coro.set_exit()
def serve (G):
path = os.path.join (G.args.base, G.args.file)
s = coro.sock (coro.AF.UNIX, coro.SOCK.STREAM)
try:
os.unlink (path)
except OSError:
pass
s.bind (path)
s.listen (100)
while 1:
conn, addr = s.accept()
if coro.fork() == 0:
coro.spawn (go, G, conn)
s.close()
return
else:
conn.close()
coro.set_exit()
def serve(G):
path = os.path.join(G.args.base, G.args.file)
s = coro.sock(coro.AF.UNIX, coro.SOCK.STREAM)
try:
os.unlink(path)
except OSError:
pass
s.bind(path)
s.listen(100)
while 1:
conn, addr = s.accept()
if coro.fork() == 0:
coro.spawn(go, G, conn)
s.close()
return
else:
conn.close()
coro.set_exit()
def service_client (conn, addr):
while 1:
try:
data = coro.with_timeout (10, conn.recv, 8192)
except coro.TimeoutError:
conn.send ('too slow, moe. good-bye!\r\n')
data = None
if not data:
conn.close()
break
else:
if data[0] == '!':
# a command
if data == '!quit\r\n':
conn.send ('ok\r\n')
conn.close()
break
elif data == '!shutdown\r\n':
coro.set_exit()
conn.send ('ok\r\n')
conn.close()
break
elif data == '!mark\r\n':
the_timer.mark()
conn.send ('ok\r\n')
elif data == '!bench\r\n':
conn.send (
coro_bench.format_rusage (
the_timer.bench()
) + '\r\n\000'
)
elif data == '!stats\r\n':
conn.send ('ok\r\n')
coro_bench.dump_stats()
else:
conn.send ('huh?\r\n')
else:
conn.send (data)
def service_client(conn, addr):
while 1:
try:
data = coro.with_timeout(10, conn.recv, 8192)
except coro.TimeoutError:
conn.send('too slow, moe. good-bye!\r\n')
data = None
if not data:
conn.close()
break
else:
if data[0] == '!':
# a command
if data == '!quit\r\n':
conn.send('ok\r\n')
conn.close()
break
elif data == '!shutdown\r\n':
coro.set_exit()
conn.send('ok\r\n')
conn.close()
break
elif data == '!mark\r\n':
the_timer.mark()
conn.send('ok\r\n')
elif data == '!bench\r\n':
conn.send(
coro_bench.format_rusage(the_timer.bench()) +
'\r\n\000')
elif data == '!stats\r\n':
conn.send('ok\r\n')
coro_bench.dump_stats()
else:
conn.send('huh?\r\n')
else:
conn.send(data)
def do_shutdown (self):
coro.sleep_relative (5)
coro.set_exit()
def cmd_shutdown (self, request, PUSH, parts):
request.push (H3 ('Shutting down...'))
request.done()
coro.sleep_relative (1)
coro.set_exit()
def do_tests():
global q, finished
do_test(push_pop_test)
do_test(remove_test)
do_test(remove_test2)
coro.set_exit()
def shutdown(signum):
# Unfortunately if the code gets in a tight loop, this doesn't run because
# it is being delievered by kqueue.
coro.set_exit(1)
os._exit(1)
def cmd_shutdown(self, request, parts):
request.push("<h3>Shutting down...</h3>")
request.done()
coro.sleep_relative(1)
coro.set_exit()
def go(ip, username, cmd):
c = client(ip, username)
c.command(cmd)
coro.set_exit()
def interrupt_main():
"""Shrapnel does not have a concept of a "main" thread. As a compromise,
this will exit the process with code 1.
"""
coro.set_exit(1)
def exit_coro():
coro.sleep_relative(0.1)
coro.set_exit()
def catch_up (G):
db = G.block_db
def get_names():
"get the chain of all block names, ignoring forks"
if not db.num_block:
return []
else:
names = list (db.num_block[db.last_block])
# XXX handle this case
assert (len(names) == 1)
b = db[names[0]]
r = []
name = b.name
while 1:
r.append(name)
name = db.prev[name]
if name == ZERO_NAME:
break
r.reverse()
return r
ledger = LedgerState (load=True)
if len(ledger.outpoints) == 0:
W ('no outpoints cache. performing fast scan [15-40 minutes]\n')
ledger.outpoints = UTXO_Scan_Map()
fast_scan = True
else:
fast_scan = False
t0 = timer()
names = get_names()
#if fast_scan:
# # TRIM FOR TESTING ONLY
# names = names[:225430]
# drop back by a 20-block horizon
most_names = names[:-20]
i = 0
fed = 0
# lots of disk i/o leads to multi-second latencies, ignore for now.
# [XXX looking into using a disk i/o thread for this?]
coro.set_latency_warning (0)
for name in most_names:
if i == ledger.height + 1:
if i % 1000 == 0:
W('%d ' % (i,))
block = db[name]
ledger.feed_block (block, i)
fed += 1
elif i <= ledger.height:
pass
else:
W('oops, block too high?\n')
import pdb
pdb.set_trace()
i += 1
coro.yield_slice()
coro.set_latency_warning (1)
W('\n')
W(' total=%20s\n' % bcrepr(ledger.total + ledger.lost))
W(' live=%20s\n' % bcrepr(ledger.total))
W(' lost=%20s\n' % bcrepr(ledger.lost))
W(' fees=%20s\n' % bcrepr(ledger.fees))
W('(%.2fs to scan %d blocks into ledger)\n' % (t0.end(), fed))
if fed > 150:
W ('saving... ledger.block_name = %064x\n' % (ledger.block_name,))
ledger.save_state()
if fast_scan:
W ('done with fast scan, reloading...\n')
ledger.outpoints = None
ledger.outpoints = UTXO_Map()
ledger.load_state()
W ('topping off recent_blocks...\n')
G.recent_blocks = RecentBlocks (ledger, db)
names = db.next (ledger.block_name)
while names:
name = names.pop()
W ('adding %r\n' % (name,))
G.recent_blocks.new_block (db[name])
names += db.next (name)
if __name__ == '__main__':
coro.set_exit()
return G.recent_blocks
def go (ip, username, cmd):
c = client (ip, username)
c.command (cmd)
coro.set_exit()
def do_shutdown(self):
coro.sleep_relative(5)
coro.set_exit()
def interrupt_main():
"""Shrapnel does not have a concept of a "main" thread. As a compromise,
this will exit the process with code 1.
"""
coro.set_exit(1)
def catch_up (G):
db = G.block_db
def get_names():
"get the chain of all block names, ignoring forks"
if not db.num_block:
return []
else:
names = list (db.num_block[db.last_block])
# XXX handle this case
assert (len(names) == 1)
b = db[names[0]]
r = []
name = b.name
while 1:
r.append(name)
name = db.prev[name]
if name == ZERO_NAME:
break
r.reverse()
return r
ledger = LedgerState (load=True)
if len(ledger.outpoints) == 0:
LOG ('no cache')
ledger.outpoints = UTXO_Scan_Map()
fast_scan = True
else:
fast_scan = False
t0 = timer()
names = get_names()
#if fast_scan:
# # TRIM FOR TESTING ONLY
# names = names[:225430]
# drop back by a 20-block horizon
most_names = names[:-20]
i = 0
fed = 0
for name in most_names:
if i == ledger.height + 1:
if i % 1000 == 0:
LOG ('scan', i)
block = db[name]
ledger.feed_block (block, i)
fed += 1
elif i <= ledger.height:
pass
else:
LOG ('block too high?')
import pdb; pdb.set_trace()
i += 1
coro.yield_slice()
LOG ('total/lost/fees', ledger.total, ledger.lost, ledger.fees)
LOG ('scan', t0.end(), fed)
if fed > 150:
LOG ('saving', repr(ledger.block_name))
ledger.save_state()
if fast_scan:
LOG ('fast scan done, reloading')
ledger.outpoints = None
ledger.outpoints = UTXO_Map()
ledger.load_state()
LOG ('topping off recent blocks')
G.recent_blocks = RecentBlocks (ledger, db)
names = db.next (ledger.block_name)
while names:
name = names.pop()
LOG ('add', repr(name))
G.recent_blocks.new_block (db[name])
names += db.next (name)
if __name__ == '__main__':
coro.set_exit()
return G.recent_blocks
def do_tests():
global q, finished
do_test(push_pop_test)
do_test(remove_test)
do_test(remove_test2)
coro.set_exit()
def catch_up(G):
db = G.block_db
def get_names():
"get the chain of all block names, ignoring forks"
if not db.num_block:
return []
else:
names = list(db.num_block[db.last_block])
# XXX handle this case
assert (len(names) == 1)
b = db[names[0]]
r = []
name = b.name
while 1:
r.append(name)
name = db.prev[name]
if name == ZERO_NAME:
break
r.reverse()
return r
ledger = LedgerState(load=True)
if len(ledger.outpoints) == 0:
LOG('no cache')
ledger.outpoints = UTXO_Scan_Map()
fast_scan = True
else:
fast_scan = False
t0 = timer()
names = get_names()
#if fast_scan:
# # TRIM FOR TESTING ONLY
# names = names[:225430]
# drop back by a 20-block horizon
most_names = names[:-20]
i = 0
fed = 0
for name in most_names:
if i == ledger.height + 1:
if i % 1000 == 0:
LOG('scan', i)
block = db[name]
ledger.feed_block(block, i)
fed += 1
elif i <= ledger.height:
pass
else:
LOG('block too high?')
import pdb
pdb.set_trace()
i += 1
coro.yield_slice()
LOG('total/lost/fees', ledger.total, ledger.lost, ledger.fees)
LOG('scan', t0.end(), fed)
if fed > 150:
LOG('saving', repr(ledger.block_name))
ledger.save_state()
if fast_scan:
LOG('fast scan done, reloading')
ledger.outpoints = None
ledger.outpoints = UTXO_Map()
ledger.load_state()
LOG('topping off recent blocks')
G.recent_blocks = RecentBlocks(ledger, db)
names = db.next(ledger.block_name)
while names:
name = names.pop()
LOG('add', repr(name))
G.recent_blocks.new_block(db[name])
names += db.next(name)
if __name__ == '__main__':
coro.set_exit()
return G.recent_blocks
def cmd_shutdown (self, request, parts):
request.push ('<h3>Shutting down...</h3>')
request.done()
coro.sleep_relative (1)
coro.set_exit()
def catch_up(G):
db = G.block_db
def get_names():
"get the chain of all block names, ignoring forks"
if not db.num_block:
return []
else:
names = list(db.num_block[db.last_block])
# XXX handle this case
assert (len(names) == 1)
b = db[names[0]]
r = []
name = b.name
while 1:
r.append(name)
name = db.prev[name]
if name == ZERO_NAME:
break
r.reverse()
return r
ledger = LedgerState(load=True)
if len(ledger.outpoints) == 0:
W('no outpoints cache. performing fast scan [30-45 minutes]\n')
ledger.outpoints = UTXO_Scan_Map()
fast_scan = True
else:
fast_scan = False
t0 = timer()
names = get_names()
#if fast_scan:
# # TRIM FOR TESTING ONLY
# names = names[:225430]
# drop back by a 20-block horizon
most_names = names[:-20]
i = 0
fed = 0
for name in most_names:
if i == ledger.height + 1:
if i % 1000 == 0:
W('%d ' % (i, ))
block = db[name]
ledger.feed_block(block, i)
fed += 1
elif i <= ledger.height:
pass
else:
W('oops, block too high?\n')
import pdb
pdb.set_trace()
i += 1
coro.yield_slice()
W('\n')
W(' total=%20s\n' % bcrepr(ledger.total + ledger.lost))
W(' live=%20s\n' % bcrepr(ledger.total))
W(' lost=%20s\n' % bcrepr(ledger.lost))
W(' fees=%20s\n' % bcrepr(ledger.fees))
W('(%.2fs to scan %d blocks into ledger)\n' % (t0.end(), fed))
if fed > 150:
W('saving... ledger.block_name = %064x\n' % (ledger.block_name, ))
ledger.save_state()
if fast_scan:
W('done with fast scan, reloading...\n')
ledger.outpoints = None
ledger.outpoints = UTXO_Map()
ledger.load_state()
W('topping off recent_blocks...\n')
G.recent_blocks = RecentBlocks(ledger, db)
names = db.next(ledger.block_name)
while names:
name = names.pop()
W('adding %r\n' % (name, ))
G.recent_blocks.new_block(db[name])
names += db.next(name)
if __name__ == '__main__':
coro.set_exit()
return G.recent_blocks
def exit_coro():
coro.sleep_relative(0.1)
coro.set_exit()
