def task():
"""
Assigned to a worker to retrieve (pop) a node from the crawl set and
attempt to establish connection with a new node.
"""
redis_conn = new_redis_conn(db=CONF['db'])
while True:
if not CONF['master']:
if CONF['BLOCKCHAIN'] != REDIS_CONN.get('crawl:master:blockchain'):
CONF['BLOCKCHAIN'] = REDIS_CONN.get('crawl:master:blockchain')
set_bchain_params()
while REDIS_CONN.get('crawl:master:state') != "running":
gevent.sleep(CONF['socket_timeout'])
node = redis_conn.spop('pending') # Pop random node from set
if node is None:
gevent.sleep(1)
continue
node = eval(node) # Convert string from Redis to tuple
# Skip IPv6 node
if ":" in node[0] and not CONF['ipv6']:
continue
key = "node:{}-{}-{}".format(node[0], node[1], node[2])
if redis_conn.exists(key):
continue
# Check if prefix has hit its limit
if ":" in node[0] and CONF['ipv6_prefix'] < 128:
cidr = ip_to_network(node[0], CONF['ipv6_prefix'])
nodes = redis_conn.incr('crawl:cidr:{}'.format(cidr))
if nodes > CONF['nodes_per_ipv6_prefix']:
logging.debug("CIDR %s: %d", cidr, nodes)
continue
connect(redis_conn, key)
def task():
"""
Assigned to a worker to retrieve (pop) a node from the crawl set and
attempt to establish connection with a new node.
"""
redis_conn = redis.StrictRedis(unix_socket_path=REDIS_SOCKET,
password=REDIS_PASSWORD)
while True:
if not SETTINGS['master']:
while REDIS_CONN.get('crawl:master:state') != "running":
gevent.sleep(SETTINGS['socket_timeout'])
node = redis_conn.spop('pending') # Pop random node from set
if node is None:
gevent.sleep(1)
continue
node = eval(node) # Convert string from Redis to tuple
# Skip IPv6 node
if ":" in node[0] and not SETTINGS['ipv6']:
continue
key = "node:{}-{}-{}".format(node[0], node[1], node[2])
if redis_conn.exists(key):
continue
# Check if prefix has hit its limit
if ":" in node[0] and SETTINGS['ipv6_prefix'] < 128:
cidr = ip_to_network(node[0], SETTINGS['ipv6_prefix'])
nodes = redis_conn.incr('crawl:cidr:{}'.format(cidr))
if nodes > SETTINGS['nodes_per_ipv6_prefix']:
logging.debug("CIDR %s: %d", cidr, nodes)
continue
connect(redis_conn, key)
def task():
"""
Assigned to a worker to retrieve (pop) a node from the crawl set and
attempt to establish connection with a new node.
"""
redis_conn = redis.StrictRedis(unix_socket_path=REDIS_SOCKET,
password=REDIS_PASSWORD)
while True:
if not SETTINGS['master']:
while REDIS_CONN.get('crawl:master:state') != "running":
gevent.sleep(SETTINGS['socket_timeout'])
node = redis_conn.spop('pending') # Pop random node from set
if node is None:
gevent.sleep(1)
continue
node = eval(node) # Convert string from Redis to tuple
# Skip IPv6 node
if ":" in node[0] and not SETTINGS['ipv6']:
continue
key = "node:{}-{}-{}".format(node[0], node[1], node[2])
if redis_conn.exists(key):
continue
# Check if prefix has hit its limit
if ":" in node[0] and SETTINGS['ipv6_prefix'] < 128:
cidr = ip_to_network(node[0], SETTINGS['ipv6_prefix'])
nodes = redis_conn.incr('crawl:cidr:{}'.format(cidr))
if nodes > SETTINGS['nodes_per_ipv6_prefix']:
logging.debug("CIDR %s: %d", cidr, nodes)
continue
connect(redis_conn, key)
def task():
"""
Assigned to a worker to retrieve (pop) a node from the reachable set and
attempt to establish and maintain connection with the node.
"""
node = REDIS_CONN.spop('reachable')
if node is None:
return
(address, port, services, height) = eval(node)
node = (address, port)
# Check if prefix has hit its limit
cidr_key = None
if ":" in address and CONF['ipv6_prefix'] < 128:
cidr = ip_to_network(address, CONF['ipv6_prefix'])
cidr_key = 'ping:cidr:{}'.format(cidr)
nodes = REDIS_CONN.incr(cidr_key)
logging.info("+CIDR %s: %d", cidr, nodes)
if nodes > CONF['nodes_per_ipv6_prefix']:
logging.info("CIDR limit reached: %s", cidr)
nodes = REDIS_CONN.decr(cidr_key)
logging.info("-CIDR %s: %d", cidr, nodes)
return
if REDIS_CONN.sadd('open', node) == 0:
logging.info("Connection exists: %s", node)
if cidr_key:
nodes = REDIS_CONN.decr(cidr_key)
logging.info("-CIDR %s: %d", cidr, nodes)
return
proxy = None
if address.endswith(".onion"):
proxy = random.choice(CONF['tor_proxies'])
handshake_msgs = []
conn = Connection(node,
(CONF['source_address'], 0),
magic_number=CONF['magic_number'],
socket_timeout=CONF['socket_timeout'],
proxy=proxy,
protocol_version=CONF['protocol_version'],
to_services=services,
from_services=CONF['services'],
user_agent=CONF['user_agent'],
height=height,
relay=CONF['relay'])
try:
logging.debug("Connecting to %s", conn.to_addr)
conn.open()
handshake_msgs = conn.handshake()
except (ProtocolError, ConnectionError, socket.error) as err:
logging.debug("Closing %s (%s)", node, err)
conn.close()
if len(handshake_msgs) == 0:
if cidr_key:
nodes = REDIS_CONN.decr(cidr_key)
logging.info("-CIDR %s: %d", cidr, nodes)
REDIS_CONN.srem('open', node)
return
if address.endswith(".onion"):
# Map local port to .onion node
local_port = conn.socket.getsockname()[1]
logging.info("Local port %s: %d", conn.to_addr, local_port)
REDIS_CONN.set('onion:{}'.format(local_port), conn.to_addr)
Keepalive(conn=conn, version_msg=handshake_msgs[0]).keepalive()
conn.close()
if cidr_key:
nodes = REDIS_CONN.decr(cidr_key)
logging.info("-CIDR %s: %d", cidr, nodes)
REDIS_CONN.srem('open', node)
def task():
"""
Assigned to a worker to retrieve (pop) a node from the reachable set and
attempt to establish and maintain connection with the node.
"""
node = REDIS_CONN.spop('reachable')
if node is None:
return
(address, port, services, height) = eval(node)
node = (address, port)
# Check if prefix has hit its limit
cidr_key = None
if ":" in address and SETTINGS['ipv6_prefix'] < 128:
cidr = ip_to_network(address, SETTINGS['ipv6_prefix'])
cidr_key = 'ping:cidr:{}'.format(cidr)
nodes = REDIS_CONN.incr(cidr_key)
logging.info("+CIDR %s: %d", cidr, nodes)
if nodes > SETTINGS['nodes_per_ipv6_prefix']:
logging.info("CIDR limit reached: %s", cidr)
nodes = REDIS_CONN.decr(cidr_key)
logging.info("-CIDR %s: %d", cidr, nodes)
return
if REDIS_CONN.sadd('open', node) == 0:
logging.info("Connection exists: %s", node)
if cidr_key:
nodes = REDIS_CONN.decr(cidr_key)
logging.info("-CIDR %s: %d", cidr, nodes)
return
proxy = None
if address.endswith(".onion"):
proxy = SETTINGS['tor_proxy']
handshake_msgs = []
conn = Connection(node, (SETTINGS['source_address'], 0),
socket_timeout=SETTINGS['socket_timeout'],
proxy=proxy,
protocol_version=SETTINGS['protocol_version'],
to_services=services,
from_services=SETTINGS['services'],
user_agent=SETTINGS['user_agent'],
height=height,
relay=SETTINGS['relay'])
try:
logging.debug("Connecting to %s", conn.to_addr)
conn.open()
handshake_msgs = conn.handshake()
except (ProtocolError, ConnectionError, socket.error) as err:
logging.debug("Closing %s (%s)", node, err)
conn.close()
if len(handshake_msgs) == 0:
if cidr_key:
nodes = REDIS_CONN.decr(cidr_key)
logging.info("-CIDR %s: %d", cidr, nodes)
REDIS_CONN.srem('open', node)
return
if address.endswith(".onion"):
# Map local port to .onion node
local_port = conn.socket.getsockname()[1]
logging.info("%s: 127.0.0.1:%d", conn.to_addr, local_port)
REDIS_CONN.set('onion:{}'.format(local_port), conn.to_addr)
Keepalive(conn=conn, version_msg=handshake_msgs[0]).keepalive()
conn.close()
if cidr_key:
nodes = REDIS_CONN.decr(cidr_key)
logging.info("-CIDR %s: %d", cidr, nodes)
REDIS_CONN.srem('open', node)
