def main(argv):
if len(argv) < 2 or not os.path.exists(argv[1]):
print("Usage: seeder.py [config]")
return 1
# Initialize global conf
init_conf(argv)
# Initialize logger
loglevel = logging.INFO
if CONF['debug']:
loglevel = logging.DEBUG
logformat = ("%(asctime)s,%(msecs)05.1f %(levelname)s (%(funcName)s) "
"%(message)s")
logging.basicConfig(level=loglevel,
format=logformat,
filename=CONF['logfile'],
filemode='w')
print(("Log: {}, press CTRL+C to terminate..".format(CONF['logfile'])))
global REDIS_CONN
REDIS_CONN = new_redis_conn(db=CONF['db'])
cron()
return 0
def main(argv):
if len(argv) < 2 or not os.path.exists(argv[1]):
print("Usage: seeder.py [config]")
return 1
# Initialize global conf
init_conf(argv)
# Initialize logger
loglevel = logging.INFO
if CONF['debug']:
loglevel = logging.DEBUG
logformat = ("%(asctime)s,%(msecs)05.1f %(levelname)s (%(funcName)s) "
"%(message)s")
logging.basicConfig(level=loglevel,
format=logformat,
filename=CONF['logfile'],
filemode='w')
print("Log: {}, press CTRL+C to terminate..".format(CONF['logfile']))
global REDIS_CONN
REDIS_CONN = new_redis_conn(db=CONF['db'])
cron()
return 0
def main(argv):
test_conn()
if len(argv) < 3 or not os.path.exists(argv[1]):
print("Usage: crawl.py [config] [master|slave]")
return 1
# Initialize global conf
init_conf(argv)
# Initialize logger
loglevel = logging.INFO
if CONF['debug']:
loglevel = logging.DEBUG
logformat = ("[%(process)d] %(asctime)s,%(msecs)05.1f %(levelname)s "
"(%(funcName)s) %(message)s")
logging.basicConfig(level=loglevel,
format=logformat,
filename=CONF['logfile'],
filemode='a')
print("Log: {}, press CTRL+C to terminate..".format(CONF['logfile']))
global REDIS_CONN
REDIS_CONN = new_redis_conn(db=CONF['db'])
if CONF['master']:
REDIS_CONN.set('crawl:master:state', "starting")
REDIS_CONN.set('crawl:master:blockchain', CONF['BLOCKCHAIN'])
logging.info("Removing all keys")
redis_pipe = REDIS_CONN.pipeline()
for b in all_chains:
redis_pipe.delete('up-{}'.format(b))
redis_pipe.delete('up')
for key in get_keys(REDIS_CONN, 'node:*'):
redis_pipe.delete(key)
for key in get_keys(REDIS_CONN, 'crawl:cidr:*'):
redis_pipe.delete(key)
redis_pipe.delete('pending')
redis_pipe.execute()
set_pending()
update_excluded_networks()
REDIS_CONN.set('crawl:master:state', "running")
# Spawn workers (greenlets) including one worker reserved for cron tasks
workers = []
if CONF['master']:
workers.append(gevent.spawn(cron))
for _ in xrange(CONF['workers'] - len(workers)):
workers.append(gevent.spawn(task))
logging.info("Workers: %d", len(workers))
gevent.joinall(workers)
return 0
def main(argv):
if len(argv) < 3 or not os.path.exists(argv[1]):
print("Usage: crawl.py [config] [master|slave]")
return 1
# Initialize global conf
init_conf(argv)
# Initialize logger
loglevel = logging.INFO
if CONF['debug']:
loglevel = logging.DEBUG
logformat = ("[%(process)d] %(asctime)s,%(msecs)05.1f %(levelname)s "
"(%(funcName)s) %(message)s")
logging.basicConfig(level=loglevel,
format=logformat,
filename=CONF['logfile'],
filemode='a')
print("Log: {}, press CTRL+C to terminate..".format(CONF['logfile']))
global REDIS_CONN
REDIS_CONN = new_redis_conn(db=CONF['db'])
if CONF['master']:
REDIS_CONN.set('crawl:master:state', "starting")
logging.info("Removing all keys")
redis_pipe = REDIS_CONN.pipeline()
redis_pipe.delete('up')
for key in get_keys(REDIS_CONN, 'node:*'):
redis_pipe.delete(key)
for key in get_keys(REDIS_CONN, 'crawl:cidr:*'):
redis_pipe.delete(key)
redis_pipe.delete('pending')
redis_pipe.execute()
set_pending()
update_excluded_networks()
REDIS_CONN.set('crawl:master:state', "running")
# Spawn workers (greenlets) including one worker reserved for cron tasks
workers = []
if CONF['master']:
workers.append(gevent.spawn(cron))
for _ in xrange(CONF['workers'] - len(workers)):
workers.append(gevent.spawn(task))
logging.info("Workers: %d", len(workers))
gevent.joinall(workers)
return 0
def main(argv):
if len(argv) < 2 or not os.path.exists(argv[1]):
print("Usage: export.py [config]")
return 1
# Initialize global conf
init_conf(argv)
# Initialize logger
loglevel = logging.INFO
if CONF['debug']:
loglevel = logging.DEBUG
logformat = (
"%(filename)s %(asctime)s,%(msecs)05.1f %(levelname)s (%(funcName)s) "
"%(message)s")
logging.basicConfig(level=loglevel,
format=logformat,
filename=CONF['logfile'],
filemode='a')
print("Log: {}, press CTRL+C to terminate..".format(CONF['logfile']))
global REDIS_CONN
REDIS_CONN = new_redis_conn(db=CONF['db'])
subscribe_key = 'resolve:{}'.format(hexlify(CONF['magic_number']))
publish_key = 'export:{}'.format(hexlify(CONF['magic_number']))
pubsub = REDIS_CONN.pubsub()
pubsub.subscribe(subscribe_key)
while True:
msg = pubsub.get_message()
if msg is None:
time.sleep(0.001) # 1 ms artificial intrinsic latency.
continue
# 'resolve' message is published by resolve.py after resolving hostname
# and GeoIP data for all reachable nodes.
if msg['channel'] == subscribe_key and msg['type'] == 'message':
timestamp = int(msg['data']) # From ping.py's 'snapshot' message
logging.info("Timestamp: %d", timestamp)
nodes = REDIS_CONN.smembers('opendata')
logging.info("Nodes: %d", len(nodes))
export_nodes(nodes, timestamp)
REDIS_CONN.publish('export', timestamp)
REDIS_CONN.set('last_export', timestamp)
REDIS_CONN.publish(publish_key, timestamp)
return 0
def main(argv):
if len(argv) < 2 or not os.path.exists(argv[1]):
print("Usage: resolve.py [config]")
return 1
# Initialize global conf
init_conf(argv)
# Initialize logger
loglevel = logging.INFO
if CONF['debug']:
loglevel = logging.DEBUG
logformat = ("%(asctime)s,%(msecs)05.1f %(levelname)s (%(funcName)s) "
"%(message)s")
logging.basicConfig(level=loglevel,
format=logformat,
filename=CONF['logfile'],
filemode='w')
print("Log: {}, press CTRL+C to terminate..".format(CONF['logfile']))
global REDIS_CONN
REDIS_CONN = new_redis_conn(db=CONF['db'])
subscribe_key = 'snapshot:{}'.format(hexlify(CONF['magic_number']))
publish_key = 'resolve:{}'.format(hexlify(CONF['magic_number']))
pubsub = REDIS_CONN.pubsub()
pubsub.subscribe(subscribe_key)
while True:
msg = pubsub.get_message()
if msg is None:
time.sleep(0.001) # 1 ms artificial intrinsic latency.
continue
# 'snapshot' message is published by ping.py after establishing
# connection with nodes from a new snapshot.
if msg['channel'] == subscribe_key and msg['type'] == 'message':
timestamp = int(msg['data'])
logging.info("Timestamp: %d", timestamp)
nodes = REDIS_CONN.smembers('opendata')
logging.info("Nodes: %d", len(nodes))
addresses = set([eval(node)[0] for node in nodes])
resolve = Resolve(addresses=addresses)
resolve.resolve_addresses()
REDIS_CONN.publish(publish_key, timestamp)
return 0
def main(argv):
if len(argv) < 2 or not os.path.exists(argv[1]):
print("Usage: resolve.py [config]")
return 1
# Initialize global conf
init_conf(argv)
# Initialize logger
loglevel = logging.INFO
if CONF['debug']:
loglevel = logging.DEBUG
logformat = ("%(asctime)s,%(msecs)05.1f %(levelname)s (%(funcName)s) "
"%(message)s")
logging.basicConfig(level=loglevel,
format=logformat,
filename=CONF['logfile'],
filemode='w')
print("Log: {}, press CTRL+C to terminate..".format(CONF['logfile']))
global REDIS_CONN
REDIS_CONN = new_redis_conn(db=CONF['db'])
subscribe_key = 'snapshot:{}'.format(hexlify(CONF['magic_number']))
publish_key = 'resolve:{}'.format(hexlify(CONF['magic_number']))
pubsub = REDIS_CONN.pubsub()
pubsub.subscribe(subscribe_key)
while True:
msg = pubsub.get_message()
if msg is None:
time.sleep(0.001) # 1 ms artificial intrinsic latency.
continue
# 'snapshot' message is published by ping.py after establishing
# connection with nodes from a new snapshot.
if msg['channel'] == subscribe_key and msg['type'] == 'message':
timestamp = int(msg['data'])
logging.info("Timestamp: %d", timestamp)
nodes = REDIS_CONN.smembers('opendata')
logging.info("Nodes: %d", len(nodes))
addresses = set([eval(node)[0] for node in nodes])
resolve = Resolve(addresses=addresses)
resolve.resolve_addresses()
REDIS_CONN.publish(publish_key, timestamp)
return 0
def main(argv):
if len(argv) < 3 or not os.path.exists(argv[1]):
print("Usage: ping.py [config] [master|slave]")
return 1
# Initialize global conf
init_conf(argv)
# Initialize logger
loglevel = logging.INFO
if CONF['debug']:
loglevel = logging.DEBUG
logformat = ("%(filename)s %(lineno)d %(levelname)s "
"(%(funcName)s) %(message)s")
logging.basicConfig(level=loglevel,
format=logformat,
filename=CONF['logfile'],
filemode='a')
print("Log: {}, press CTRL+C to terminate..".format(CONF['logfile']))
global REDIS_CONN
REDIS_CONN = new_redis_conn(db=CONF['db'])
if CONF['master']:
redis_pipe = REDIS_CONN.pipeline()
logging.info("Removing all keys")
redis_pipe.delete('reachable')
redis_pipe.delete('open')
redis_pipe.delete('opendata')
for key in get_keys(REDIS_CONN, 'ping:cidr:*'):
logging.info("Deleting %s", key)
redis_pipe.delete(key)
redis_pipe.execute()
# Initialize a pool of workers (greenlets)
pool = gevent.pool.Pool(CONF['workers'])
pool.spawn(cron, pool)
pool.join()
return 0
def main(argv):
if len(argv) < 3 or not os.path.exists(argv[1]):
print("Usage: ping.py [config] [master|slave]")
return 1
# Initialize global conf
init_conf(argv)
# Initialize logger
loglevel = logging.INFO
if CONF['debug']:
loglevel = logging.DEBUG
logformat = ("[%(process)d] %(asctime)s,%(msecs)05.1f %(levelname)s "
"(%(funcName)s) %(message)s")
logging.basicConfig(level=loglevel,
format=logformat,
filename=CONF['logfile'],
filemode='a')
print("Log: {}, press CTRL+C to terminate..".format(CONF['logfile']))
global REDIS_CONN
REDIS_CONN = new_redis_conn(db=CONF['db'])
if CONF['master']:
redis_pipe = REDIS_CONN.pipeline()
logging.info("Removing all keys")
redis_pipe.delete('reachable')
redis_pipe.delete('open')
redis_pipe.delete('opendata')
for key in get_keys(REDIS_CONN, 'ping:cidr:*'):
redis_pipe.delete(key)
redis_pipe.execute()
# Initialize a pool of workers (greenlets)
pool = gevent.pool.Pool(CONF['workers'])
pool.spawn(cron, pool)
pool.join()
return 0
def task():
"""
Assigned to a worker to retrieve (pop) a node from the crawl set and
attempt to establish connection with a new node.
"""
redis_conn = new_redis_conn(db=CONF['db'])
while True:
if not CONF['master']:
if CONF['BLOCKCHAIN'] != REDIS_CONN.get('crawl:master:blockchain'):
CONF['BLOCKCHAIN'] = REDIS_CONN.get('crawl:master:blockchain')
set_bchain_params()
while REDIS_CONN.get('crawl:master:state') != "running":
gevent.sleep(CONF['socket_timeout'])
node = redis_conn.spop('pending') # Pop random node from set
if node is None:
gevent.sleep(1)
continue
node = eval(node) # Convert string from Redis to tuple
# Skip IPv6 node
if ":" in node[0] and not CONF['ipv6']:
continue
key = "node:{}-{}-{}".format(node[0], node[1], node[2])
if redis_conn.exists(key):
continue
# Check if prefix has hit its limit
if ":" in node[0] and CONF['ipv6_prefix'] < 128:
cidr = ip_to_network(node[0], CONF['ipv6_prefix'])
nodes = redis_conn.incr('crawl:cidr:{}'.format(cidr))
if nodes > CONF['nodes_per_ipv6_prefix']:
logging.debug("CIDR %s: %d", cidr, nodes)
continue
connect(redis_conn, key)
def task():
"""
Assigned to a worker to retrieve (pop) a node from the crawl set and
attempt to establish connection with a new node.
"""
redis_conn = new_redis_conn(network=SETTINGS['network'])
while True:
if not SETTINGS['master']:
while REDIS_CONN.get('crawl:master:state') != "running":
gevent.sleep(SETTINGS['socket_timeout'])
node = redis_conn.spop('pending') # Pop random node from set
if node is None:
gevent.sleep(1)
continue
node = eval(node) # Convert string from Redis to tuple
# Skip IPv6 node
if ":" in node[0] and not SETTINGS['ipv6']:
continue
key = "node:{}-{}-{}".format(node[0], node[1], node[2])
if redis_conn.exists(key):
continue
# Check if prefix has hit its limit
if ":" in node[0] and SETTINGS['ipv6_prefix'] < 128:
cidr = ip_to_network(node[0], SETTINGS['ipv6_prefix'])
nodes = redis_conn.incr('crawl:cidr:{}'.format(cidr))
if nodes > SETTINGS['nodes_per_ipv6_prefix']:
logging.debug("CIDR %s: %d", cidr, nodes)
continue
connect(redis_conn, key)