def task(): """ Assigned to a worker to retrieve (pop) a node from the crawl set and attempt to establish connection with a new node. """ redis_conn = new_redis_conn(db=CONF['db']) while True: if not CONF['master']: if CONF['BLOCKCHAIN'] != REDIS_CONN.get('crawl:master:blockchain'): CONF['BLOCKCHAIN'] = REDIS_CONN.get('crawl:master:blockchain') set_bchain_params() while REDIS_CONN.get('crawl:master:state') != "running": gevent.sleep(CONF['socket_timeout']) node = redis_conn.spop('pending') # Pop random node from set if node is None: gevent.sleep(1) continue node = eval(node) # Convert string from Redis to tuple # Skip IPv6 node if ":" in node[0] and not CONF['ipv6']: continue key = "node:{}-{}-{}".format(node[0], node[1], node[2]) if redis_conn.exists(key): continue # Check if prefix has hit its limit if ":" in node[0] and CONF['ipv6_prefix'] < 128: cidr = ip_to_network(node[0], CONF['ipv6_prefix']) nodes = redis_conn.incr('crawl:cidr:{}'.format(cidr)) if nodes > CONF['nodes_per_ipv6_prefix']: logging.debug("CIDR %s: %d", cidr, nodes) continue connect(redis_conn, key)
def task(): """ Assigned to a worker to retrieve (pop) a node from the crawl set and attempt to establish connection with a new node. """ redis_conn = redis.StrictRedis(unix_socket_path=REDIS_SOCKET, password=REDIS_PASSWORD) while True: if not SETTINGS['master']: while REDIS_CONN.get('crawl:master:state') != "running": gevent.sleep(SETTINGS['socket_timeout']) node = redis_conn.spop('pending') # Pop random node from set if node is None: gevent.sleep(1) continue node = eval(node) # Convert string from Redis to tuple # Skip IPv6 node if ":" in node[0] and not SETTINGS['ipv6']: continue key = "node:{}-{}-{}".format(node[0], node[1], node[2]) if redis_conn.exists(key): continue # Check if prefix has hit its limit if ":" in node[0] and SETTINGS['ipv6_prefix'] < 128: cidr = ip_to_network(node[0], SETTINGS['ipv6_prefix']) nodes = redis_conn.incr('crawl:cidr:{}'.format(cidr)) if nodes > SETTINGS['nodes_per_ipv6_prefix']: logging.debug("CIDR %s: %d", cidr, nodes) continue connect(redis_conn, key)
def task(): """ Assigned to a worker to retrieve (pop) a node from the crawl set and attempt to establish connection with a new node. """ redis_conn = redis.StrictRedis(unix_socket_path=REDIS_SOCKET, password=REDIS_PASSWORD) while True: if not SETTINGS['master']: while REDIS_CONN.get('crawl:master:state') != "running": gevent.sleep(SETTINGS['socket_timeout']) node = redis_conn.spop('pending') # Pop random node from set if node is None: gevent.sleep(1) continue node = eval(node) # Convert string from Redis to tuple # Skip IPv6 node if ":" in node[0] and not SETTINGS['ipv6']: continue key = "node:{}-{}-{}".format(node[0], node[1], node[2]) if redis_conn.exists(key): continue # Check if prefix has hit its limit if ":" in node[0] and SETTINGS['ipv6_prefix'] < 128: cidr = ip_to_network(node[0], SETTINGS['ipv6_prefix']) nodes = redis_conn.incr('crawl:cidr:{}'.format(cidr)) if nodes > SETTINGS['nodes_per_ipv6_prefix']: logging.debug("CIDR %s: %d", cidr, nodes) continue connect(redis_conn, key)
def task(): """ Assigned to a worker to retrieve (pop) a node from the reachable set and attempt to establish and maintain connection with the node. """ node = REDIS_CONN.spop('reachable') if node is None: return (address, port, services, height) = eval(node) node = (address, port) # Check if prefix has hit its limit cidr_key = None if ":" in address and CONF['ipv6_prefix'] < 128: cidr = ip_to_network(address, CONF['ipv6_prefix']) cidr_key = 'ping:cidr:{}'.format(cidr) nodes = REDIS_CONN.incr(cidr_key) logging.info("+CIDR %s: %d", cidr, nodes) if nodes > CONF['nodes_per_ipv6_prefix']: logging.info("CIDR limit reached: %s", cidr) nodes = REDIS_CONN.decr(cidr_key) logging.info("-CIDR %s: %d", cidr, nodes) return if REDIS_CONN.sadd('open', node) == 0: logging.info("Connection exists: %s", node) if cidr_key: nodes = REDIS_CONN.decr(cidr_key) logging.info("-CIDR %s: %d", cidr, nodes) return proxy = None if address.endswith(".onion"): proxy = random.choice(CONF['tor_proxies']) handshake_msgs = [] conn = Connection(node, (CONF['source_address'], 0), magic_number=CONF['magic_number'], socket_timeout=CONF['socket_timeout'], proxy=proxy, protocol_version=CONF['protocol_version'], to_services=services, from_services=CONF['services'], user_agent=CONF['user_agent'], height=height, relay=CONF['relay']) try: logging.debug("Connecting to %s", conn.to_addr) conn.open() handshake_msgs = conn.handshake() except (ProtocolError, ConnectionError, socket.error) as err: logging.debug("Closing %s (%s)", node, err) conn.close() if len(handshake_msgs) == 0: if cidr_key: nodes = REDIS_CONN.decr(cidr_key) logging.info("-CIDR %s: %d", cidr, nodes) REDIS_CONN.srem('open', node) return if address.endswith(".onion"): # Map local port to .onion node local_port = conn.socket.getsockname()[1] logging.info("Local port %s: %d", conn.to_addr, local_port) REDIS_CONN.set('onion:{}'.format(local_port), conn.to_addr) Keepalive(conn=conn, version_msg=handshake_msgs[0]).keepalive() conn.close() if cidr_key: nodes = REDIS_CONN.decr(cidr_key) logging.info("-CIDR %s: %d", cidr, nodes) REDIS_CONN.srem('open', node)
def task(): """ Assigned to a worker to retrieve (pop) a node from the reachable set and attempt to establish and maintain connection with the node. """ node = REDIS_CONN.spop('reachable') if node is None: return (address, port, services, height) = eval(node) node = (address, port) # Check if prefix has hit its limit cidr_key = None if ":" in address and SETTINGS['ipv6_prefix'] < 128: cidr = ip_to_network(address, SETTINGS['ipv6_prefix']) cidr_key = 'ping:cidr:{}'.format(cidr) nodes = REDIS_CONN.incr(cidr_key) logging.info("+CIDR %s: %d", cidr, nodes) if nodes > SETTINGS['nodes_per_ipv6_prefix']: logging.info("CIDR limit reached: %s", cidr) nodes = REDIS_CONN.decr(cidr_key) logging.info("-CIDR %s: %d", cidr, nodes) return if REDIS_CONN.sadd('open', node) == 0: logging.info("Connection exists: %s", node) if cidr_key: nodes = REDIS_CONN.decr(cidr_key) logging.info("-CIDR %s: %d", cidr, nodes) return proxy = None if address.endswith(".onion"): proxy = SETTINGS['tor_proxy'] handshake_msgs = [] conn = Connection(node, (SETTINGS['source_address'], 0), socket_timeout=SETTINGS['socket_timeout'], proxy=proxy, protocol_version=SETTINGS['protocol_version'], to_services=services, from_services=SETTINGS['services'], user_agent=SETTINGS['user_agent'], height=height, relay=SETTINGS['relay']) try: logging.debug("Connecting to %s", conn.to_addr) conn.open() handshake_msgs = conn.handshake() except (ProtocolError, ConnectionError, socket.error) as err: logging.debug("Closing %s (%s)", node, err) conn.close() if len(handshake_msgs) == 0: if cidr_key: nodes = REDIS_CONN.decr(cidr_key) logging.info("-CIDR %s: %d", cidr, nodes) REDIS_CONN.srem('open', node) return if address.endswith(".onion"): # Map local port to .onion node local_port = conn.socket.getsockname()[1] logging.info("%s: 127.0.0.1:%d", conn.to_addr, local_port) REDIS_CONN.set('onion:{}'.format(local_port), conn.to_addr) Keepalive(conn=conn, version_msg=handshake_msgs[0]).keepalive() conn.close() if cidr_key: nodes = REDIS_CONN.decr(cidr_key) logging.info("-CIDR %s: %d", cidr, nodes) REDIS_CONN.srem('open', node)