def test_burn():
with stats.record_burn('foo', url='http://example.com/'):
t0 = time.process_time()
while time.process_time() < t0 + 0.001:
pass
assert stats.burners['foo']['count'] == 1
assert stats.burners['foo']['time'] > 0 and stats.burners['foo'][
'time'] < 0.3
assert 'list' not in stats.burners[
'foo'] # first burn never goes on the list
with stats.record_burn('foo', url='http://example.com/'):
t0 = time.process_time()
while time.process_time() < t0 + 0.2:
pass
assert stats.burners['foo']['count'] == 2
assert stats.burners['foo']['time'] > 0 and stats.burners['foo'][
'time'] < 0.3
assert len(stats.burners['foo']['list']) == 1
stats.update_cpu_burn('foo', 3, 3.0, set())
assert stats.burners['foo']['count'] == 5
assert stats.burners['foo']['time'] > 3.0 and stats.burners['foo'][
'time'] < 3.3
assert len(stats.burners['foo']['list']) == 1
stats.report()
def test_burn():
with stats.record_burn('foo', url='http://example.com/'):
t0 = time.process_time()
while time.process_time() < t0 + 0.001:
pass
assert stats.burners['foo']['count'] == 1
assert stats.burners['foo']['time'] > 0 and stats.burners['foo']['time'] < 0.3
assert 'list' not in stats.burners['foo'] # first burn never goes on the list
with stats.record_burn('foo', url='http://example.com/'):
t0 = time.process_time()
while time.process_time() < t0 + 0.2:
pass
assert stats.burners['foo']['count'] == 2
assert stats.burners['foo']['time'] > 0 and stats.burners['foo']['time'] < 0.3
assert len(stats.burners['foo']['list']) == 1
stats.update_cpu_burn('foo', 3, 3.0, set())
assert stats.burners['foo']['count'] == 5
assert stats.burners['foo']['time'] > 3.0 and stats.burners['foo']['time'] < 3.3
assert len(stats.burners['foo']['list']) == 1
stats.report()
def main():
for d in sys.argv[1:]:
if os.path.isfile(d):
queue.put_nowait(d)
continue
for root, _, files in os.walk(d):
for f in files:
if f.endswith('.html') or f.endswith('.htm'):
queue.put_nowait(os.path.join(root, f))
print('Queue size is {}, beginning work.'.format(queue.qsize()))
try:
loop.run_until_complete(crawl())
print('exit run until complete')
except KeyboardInterrupt:
sys.stderr.flush()
print('\nInterrupt. Exiting cleanly.\n')
finally:
loop.stop()
loop.run_forever()
loop.close()
levels = [logging.ERROR, logging.WARN, logging.INFO, logging.DEBUG]
logging.basicConfig(level=levels[3])
stats.report()
def test_latency():
with stats.record_latency('foo', url='http://example.com/'):
t0 = time.time()
while time.time() < t0 + 0.001:
pass
assert stats.latencies['foo']['count'] == 1
assert stats.latencies['foo']['time'] > 0 and stats.latencies['foo']['time'] < 0.3
assert 'list' not in stats.latencies['foo'] # first latency never goes on the list
assert 'hist' in stats.latencies['foo']
with stats.record_latency('foo', url='http://example.com/', elapsedmin=0.1):
time.sleep(0.3)
assert stats.latencies['foo']['count'] == 2
assert stats.latencies['foo']['time'] > 0 and stats.latencies['foo']['time'] < 20.0
assert 'list' in stats.latencies['foo']
stats.report()
def test_latency():
with stats.record_latency('foo', url='http://example.com/'):
t0 = time.time()
while time.time() < t0 + 0.001:
pass
assert stats.latencies['foo']['count'] == 1
assert stats.latencies['foo']['time'] > 0 and stats.latencies['foo']['time'] < 0.3
assert 'list' not in stats.latencies['foo'] # first latency never goes on the list
assert 'hist' in stats.latencies['foo']
with stats.record_latency('foo', url='http://example.com/', elapsedmin=0.1):
time.sleep(0.3)
assert stats.latencies['foo']['count'] == 2
assert stats.latencies['foo']['time'] > 0 and stats.latencies['foo']['time'] < 20.0
assert 'list' in stats.latencies['foo']
stats.report()
workers = [asyncio.Task(work(), loop=loop) for _ in range(test_threadcount)]
print('q count is {}'.format(queue.qsize()))
await queue.join()
print('join is done')
for w in workers:
if not w.done():
w.cancel()
# Main program:
for i in range(10000):
queue.put_nowait('foo')
print('Queue size is {}, beginning work.'.format(queue.qsize()))
try:
loop.run_until_complete(crawl())
print('exit run until complete')
except KeyboardInterrupt:
sys.stderr.flush()
print('\nInterrupt. Exiting cleanly.\n')
finally:
loop.stop()
loop.run_forever()
loop.close()
levels = [logging.ERROR, logging.WARN, logging.INFO, logging.DEBUG]
logging.basicConfig(level=levels[3])
stats.report()
parse.report()
extra1 = v.difference(biggest_links)
extra2 = biggest_links.difference(v)
if extra1:
print(' extra in {}: {!r}'.format(names[i], extra1))
else:
print(' count was {} for {}'.format(len(v), names[i]))
if extra2:
print(' extra in {}: {!r}'.format(names[biggest_index], extra2))
else:
print(' count was {} for {}'.format(len(biggest_links), names[biggest_index]))
LOGGER = logging.getLogger(__name__)
levels = [logging.ERROR, logging.WARN, logging.INFO, logging.DEBUG]
logging.basicConfig(level=levels[3])
for d in sys.argv[1:]:
if os.path.isfile(d):
with open(d, 'r', errors='ignore') as fi:
parse_all(d, fi.read())
continue
for root, _, files in os.walk(d):
for f in files:
if f.endswith('.html') or f.endswith('.htm'):
expanded = os.path.join(root, f)
with open(expanded, 'r', errors='ignore') as fi:
parse_all(expanded, fi.read())
stats.report()