def actionFindHashIds(self, params):
site = self.sites.get(params["site"])
if not site or not site.settings[
"serving"]: # Site unknown or not serving
self.response({"error": "Unknown site"})
return False
found = site.worker_manager.findOptionalHashIds(params["hash_ids"])
back = {}
for hash_id, peers in found.iteritems():
back[hash_id] = [
helper.packAddress(peer.ip, peer.port) for peer in peers
]
# Check my hashfield
if config.ip_external:
my_ip = config.ip_external
else:
my_ip = self.server.ip
for hash_id in params["hash_ids"]:
if hash_id in site.content_manager.hashfield:
if hash_id not in back:
back[hash_id] = []
back[hash_id].append(
helper.packAddress(my_ip, self.server.port)) # Add myself
if config.verbose:
self.log.debug("Found: %s/%s" %
(len(back), len(params["hash_ids"])))
self.response({"peers": back})
def actionFindHashIds(self, params):
site = self.sites.get(params["site"])
if not site or not site.settings["serving"]: # Site unknown or not serving
self.response({"error": "Unknown site"})
return False
found = site.worker_manager.findOptionalHashIds(params["hash_ids"])
back = {}
for hash_id, peers in found.iteritems():
back[hash_id] = [helper.packAddress(peer.ip, peer.port) for peer in peers]
# Check my hashfield
if config.ip_external:
my_ip = config.ip_external
else:
my_ip = self.server.ip
for hash_id in params["hash_ids"]:
if hash_id in site.content_manager.hashfield:
if hash_id not in back:
back[hash_id] = []
back[hash_id].append(helper.packAddress(my_ip, self.server.port)) # Add myself
if config.verbose:
self.log.debug(
"Found: %s/%s" %
(len(back), len(params["hash_ids"]))
)
self.response({"peers": back})
def testPackAddress(self):
assert len(helper.packAddress("1.1.1.1", 1)) == 6
assert helper.unpackAddress(helper.packAddress("1.1.1.1", 1)) == ("1.1.1.1", 1)
with pytest.raises(socket.error):
helper.packAddress("999.1.1.1", 1)
with pytest.raises(AssertionError):
helper.unpackAddress("X")
def testPackAddress(self):
assert len(helper.packAddress("1.1.1.1", 1)) == 6
assert helper.unpackAddress(helper.packAddress("1.1.1.1", 1)) == ("1.1.1.1", 1)
with pytest.raises(socket.error):
helper.packAddress("999.1.1.1", 1)
with pytest.raises(AssertionError):
helper.unpackAddress("X")
def actionFindHashIds(self, params):
site = self.sites.get(params["site"])
s = time.time()
if not site or not site.settings[
"serving"]: # Site unknown or not serving
self.response({"error": "Unknown site"})
self.connection.badAction(5)
return False
event_key = "%s_findHashIds_%s_%s" % (
self.connection.ip, params["site"], len(params["hash_ids"]))
if self.connection.cpu_time > 0.5 or not RateLimit.isAllowed(
event_key, 60 * 5):
time.sleep(0.1)
back = self.findHashIds(site, params["hash_ids"], limit=10)
else:
back = self.findHashIds(site, params["hash_ids"])
RateLimit.called(event_key)
# Check my hashfield
if self.server.tor_manager and self.server.tor_manager.getOnion(
site.address): # Running onion
my_ip = helper.packOnionAddress(
self.server.tor_manager.getOnion(site.address),
self.server.port)
my_ip_type = "onion"
elif config.ip_external: # External ip defined
my_ip = helper.packAddress(config.ip_external, self.server.port)
my_ip_type = helper.getIpType(config.ip_external)
elif self.server.ip and self.server.ip != "*": # No external ip defined
my_ip = helper.packAddress(self.server.ip, self.server.port)
my_ip_type = helper.getIpType(self.server.ip)
else:
my_ip = None
my_ip_type = "ipv4"
my_hashfield_set = set(site.content_manager.hashfield)
for hash_id in params["hash_ids"]:
if hash_id in my_hashfield_set:
if hash_id not in back[my_ip_type]:
back[my_ip_type][hash_id] = []
if my_ip:
back[my_ip_type][hash_id].append(my_ip) # Add myself
if config.verbose:
self.log.debug("Found: %s for %s hashids in %.3fs" %
({key: len(val)
for key, val in back.iteritems()
}, len(params["hash_ids"]), time.time() - s))
self.response({
"peers": back["ipv4"],
"peers_onion": back["onion"],
"peers_ipv6": back["ipv6"]
})
def testPackAddress(self):
for port in [1, 1000, 65535]:
for ip in [
"1.1.1.1", "127.0.0.1", "0.0.0.0", "255.255.255.255",
"192.168.1.1"
]:
assert len(helper.packAddress(ip, port)) == 6
assert helper.unpackAddress(helper.packAddress(ip,
port)) == (ip,
port)
for ip in [
"1:2:3:4:5:6:7:8", "::1",
"2001:19f0:6c01:e76:5400:1ff:fed6:3eca",
"2001:4860:4860::8888"
]:
assert len(helper.packAddress(ip, port)) == 18
assert helper.unpackAddress(helper.packAddress(ip,
port)) == (ip,
port)
with pytest.raises(struct.error) as err:
helper.packAddress("1.1.1.1", 100000)
with pytest.raises(socket.error):
helper.packAddress("999.1.1.1", 1)
with pytest.raises(AssertionError):
helper.unpackAddress("X")
def actionFindHashIds(self, params):
site = self.sites.get(params["site"])
s = time.time()
if not site or not site.settings[
"serving"]: # Site unknown or not serving
self.response({"error": "Unknown site"})
self.connection.badAction(5)
return False
event_key = "%s_findHashIds_%s_%s" % (
self.connection.ip, params["site"], len(params["hash_ids"]))
if self.connection.cpu_time > 0.5 or not RateLimit.isAllowed(
event_key, 60 * 5):
time.sleep(0.1)
back_ip4, back_onion = self.findHashIds(site,
params["hash_ids"],
limit=10)
else:
back_ip4, back_onion = self.findHashIds(site, params["hash_ids"])
RateLimit.called(event_key)
# Check my hashfield
if self.server.tor_manager and self.server.tor_manager.site_onions.get(
site.address): # Running onion
my_ip = helper.packOnionAddress(
self.server.tor_manager.site_onions[site.address],
self.server.port)
my_back = back_onion
elif config.ip_external: # External ip defined
my_ip = helper.packAddress(config.ip_external, self.server.port)
my_back = back_ip4
else: # No external ip defined
my_ip = my_ip = helper.packAddress(self.server.ip,
self.server.port)
my_back = back_ip4
my_hashfield_set = set(site.content_manager.hashfield)
for hash_id in params["hash_ids"]:
if hash_id in my_hashfield_set:
if hash_id not in my_back:
my_back[hash_id] = []
my_back[hash_id].append(my_ip) # Add myself
if config.verbose:
self.log.debug(
"Found: IP4: %s, Onion: %s for %s hashids in %.3fs" %
(len(back_ip4), len(back_onion), len(
params["hash_ids"]), time.time() - s))
self.response({"peers": back_ip4, "peers_onion": back_onion})
def actionFindHashIds(self, params):
site = self.sites.get(params["site"])
if not site or not site.settings[
"serving"]: # Site unknown or not serving
self.response({"error": "Unknown site"})
self.connection.badAction(5)
return False
found = site.worker_manager.findOptionalHashIds(params["hash_ids"])
back_ip4 = {}
back_onion = {}
for hash_id, peers in found.iteritems():
back_onion[hash_id] = [
helper.packOnionAddress(peer.ip, peer.port) for peer in peers
if peer.ip.endswith("onion")
]
back_ip4[hash_id] = [
helper.packAddress(peer.ip, peer.port) for peer in peers
if not peer.ip.endswith("onion")
]
# Check my hashfield
if self.server.tor_manager and self.server.tor_manager.site_onions.get(
site.address): # Running onion
my_ip = helper.packOnionAddress(
self.server.tor_manager.site_onions[site.address],
self.server.port)
my_back = back_onion
elif config.ip_external: # External ip defined
my_ip = helper.packAddress(config.ip_external, self.server.port)
my_back = back_ip4
else: # No external ip defined
my_ip = my_ip = helper.packAddress(self.server.ip,
self.server.port)
my_back = back_ip4
for hash_id in params["hash_ids"]:
if hash_id in site.content_manager.hashfield:
if hash_id not in my_back:
my_back[hash_id] = []
my_back[hash_id].append(my_ip) # Add myself
if config.verbose:
self.log.debug(
"Found: IP4: %s, Onion: %s for %s hashids" %
(len(back_ip4), len(back_onion), len(params["hash_ids"])))
self.response({"peers": back_ip4, "peers_onion": back_onion})
def packMyAddress(self):
if self.ip.endswith(".onion"):
return helper.packOnionAddress(self.ip, self.port)
elif self.ip.endswith(".i2p"):
return helper.packI2PAddress(self.ip, self.port)
else:
return helper.packAddress(self.ip, self.port)
def peerList(self, hash, ip4=None, onions=[], port=None, limit=30, need_types=["ip4", "onion"]):
hash_peers = {"ip4": [], "onion": []}
if limit == 0:
return hash_peers
hashid = self.getHashId(hash)
where = "hash_id = :hashid"
if onions:
onions_escaped = ["'%s'" % re.sub("[^a-z0-9,]", "", onion) for onion in onions]
where += " AND (onion NOT IN (%s) OR onion IS NULL)" % ",".join(onions_escaped)
elif ip4:
where += " AND (NOT (ip4 = :ip4 AND port = :port) OR ip4 IS NULL)"
query = """
SELECT ip4, port, onion
FROM peer_to_hash
LEFT JOIN peer USING (peer_id)
WHERE %s
LIMIT :limit
""" % where
res = self.execute(query, {"hashid": hashid, "ip4": ip4, "onions": onions, "port": port, "limit": limit})
for row in res:
if row["ip4"] and "ip4" in need_types:
hash_peers["ip4"].append(
helper.packAddress(row["ip4"], row["port"])
)
if row["onion"] and "onion" in need_types:
hash_peers["onion"].append(
helper.packOnionAddress(row["onion"], row["port"])
)
return hash_peers
def peerList(self, hash, ip4=None, onions=[], port=None, limit=30, need_types=["ip4", "onion"]):
hash_peers = {"ip4": [], "onion": []}
if limit == 0:
return hash_peers
hashid = self.getHashId(hash)
where = "hash_id = :hashid"
if onions:
onions_escaped = ["'%s'" % re.sub("[^a-z0-9,]", "", onion) for onion in onions if type(onion) is str]
where += " AND (onion NOT IN (%s) OR onion IS NULL)" % ",".join(onions_escaped)
elif ip4:
where += " AND (NOT (ip4 = :ip4 AND port = :port) OR ip4 IS NULL)"
query = """
SELECT ip4, port, onion
FROM peer_to_hash
LEFT JOIN peer USING (peer_id)
WHERE %s
ORDER BY date_announced DESC
LIMIT :limit
""" % where
res = self.execute(query, {"hashid": hashid, "ip4": ip4, "onions": onions, "port": port, "limit": limit})
for row in res:
if row["ip4"] and "ip4" in need_types:
hash_peers["ip4"].append(
helper.packAddress(row["ip4"], row["port"])
)
if row["onion"] and "onion" in need_types:
hash_peers["onion"].append(
helper.packOnionAddress(row["onion"], row["port"])
)
return hash_peers
def actionFindHashIds(self, params):
site = self.sites.get(params["site"])
s = time.time()
if not site or not site.settings["serving"]: # Site unknown or not serving
self.response({"error": "Unknown site"})
self.connection.badAction(5)
return False
event_key = "%s_findHashIds_%s_%s" % (self.connection.ip, params["site"], len(params["hash_ids"]))
if self.connection.cpu_time > 0.5 or not RateLimit.isAllowed(event_key, 60 * 5):
time.sleep(0.1)
back_ip4, back_onion = self.findHashIds(site, params["hash_ids"], limit=10)
else:
back_ip4, back_onion = self.findHashIds(site, params["hash_ids"])
RateLimit.called(event_key)
# Check my hashfield
if self.server.tor_manager and self.server.tor_manager.site_onions.get(site.address): # Running onion
my_ip = helper.packOnionAddress(self.server.tor_manager.site_onions[site.address], self.server.port)
my_back = back_onion
elif config.ip_external: # External ip defined
my_ip = helper.packAddress(config.ip_external, self.server.port)
my_back = back_ip4
elif self.server.ip and self.server.ip != "*": # No external ip defined
my_ip = helper.packAddress(self.server.ip, self.server.port)
my_back = back_ip4
else:
my_ip = None
my_back = back_ip4
my_hashfield_set = set(site.content_manager.hashfield)
for hash_id in params["hash_ids"]:
if hash_id in my_hashfield_set:
if hash_id not in my_back:
my_back[hash_id] = []
if my_ip:
my_back[hash_id].append(my_ip) # Add myself
if config.verbose:
self.log.debug(
"Found: IP4: %s, Onion: %s for %s hashids in %.3fs" %
(len(back_ip4), len(back_onion), len(params["hash_ids"]), time.time() - s)
)
self.response({"peers": back_ip4, "peers_onion": back_onion})
def actionFindHashIds(self, params):
site = self.sites.get(params["site"])
if not site or not site.settings["serving"]: # Site unknown or not serving
self.response({"error": "Unknown site"})
return False
found = site.worker_manager.findOptionalHashIds(params["hash_ids"])
back = {}
for hash_id, peers in found.iteritems():
back[hash_id] = [helper.packAddress(peer.ip, peer.port) for peer in peers]
self.response({"peers": back})
def actionFindHashIds(self, params):
site = self.sites.get(params["site"])
if not site or not site.settings["serving"]: # Site unknown or not serving
self.response({"error": "Unknown site"})
self.connection.badAction(5)
return False
found = site.worker_manager.findOptionalHashIds(params["hash_ids"])
back_ip4 = {}
back_onion = {}
for hash_id, peers in found.iteritems():
back_onion[hash_id] = [
helper.packOnionAddress(peer.ip, peer.port) for peer in peers if peer.ip.endswith("onion")
]
back_ip4[hash_id] = [
helper.packAddress(peer.ip, peer.port) for peer in peers if not peer.ip.endswith("onion")
]
# Check my hashfield
if self.server.tor_manager and self.server.tor_manager.site_onions.get(site.address): # Running onion
my_ip = helper.packOnionAddress(self.server.tor_manager.site_onions[site.address], self.server.port)
my_back = back_onion
elif config.ip_external: # External ip defined
my_ip = helper.packAddress(config.ip_external, self.server.port)
my_back = back_ip4
else: # No external ip defined
my_ip = my_ip = helper.packAddress(self.server.ip, self.server.port)
my_back = back_ip4
for hash_id in params["hash_ids"]:
if hash_id in site.content_manager.hashfield:
if hash_id not in my_back:
my_back[hash_id] = []
my_back[hash_id].append(my_ip) # Add myself
if config.verbose:
self.log.debug(
"Found: IP4: %s, Onion: %s for %s hashids" % (len(back_ip4), len(back_onion), len(params["hash_ids"]))
)
self.response({"peers": back_ip4, "peers_onion": back_onion})
def peerList(self,
hash,
address=None,
onions=[],
port=None,
limit=30,
need_types=["ipv4", "onion"],
order=True):
back = {"ipv4": [], "ipv6": [], "onion": []}
if limit == 0:
return back
hashid = self.getHashId(hash)
if order:
order_sql = "ORDER BY date_announced DESC"
else:
order_sql = ""
where_sql = "hash_id = :hashid"
if onions:
onions_escaped = [
"'%s'" % re.sub("[^a-z0-9,]", "", onion) for onion in onions
if type(onion) is str
]
where_sql += " AND address NOT IN (%s)" % ",".join(onions_escaped)
elif address:
where_sql += " AND NOT (address = :address AND port = :port)"
query = """
SELECT type, address, port
FROM peer_to_hash
LEFT JOIN peer USING (peer_id)
WHERE %s
%s
LIMIT :limit
""" % (where_sql, order_sql)
res = self.execute(query, {
"hashid": hashid,
"address": address,
"port": port,
"limit": limit
})
for row in res:
if row["type"] in need_types:
if row["type"] == "onion":
packed = helper.packOnionAddress(row["address"],
row["port"])
else:
packed = helper.packAddress(str(row["address"]),
row["port"])
back[row["type"]].append(packed)
return back
def actionFindHashIds(self, params):
site = self.sites.get(params["site"])
if not site or not site.settings[
"serving"]: # Site unknown or not serving
self.response({"error": "Unknown site"})
return False
found = site.worker_manager.findOptionalHashIds(params["hash_ids"])
back = {}
for hash_id, peers in found.iteritems():
back[hash_id] = [
helper.packAddress(peer.ip, peer.port) for peer in peers
]
self.response({"peers": back})
def findHashIds(self, site, hash_ids, limit=100):
back_ip4 = {}
back_onion = {}
found = site.worker_manager.findOptionalHashIds(hash_ids, limit=limit)
for hash_id, peers in found.iteritems():
back_onion[hash_id] = list(
itertools.islice(
(helper.packOnionAddress(peer.ip, peer.port)
for peer in peers if peer.ip.endswith("onion")), 50))
back_ip4[hash_id] = list(
itertools.islice(
(helper.packAddress(peer.ip, peer.port)
for peer in peers if not peer.ip.endswith("onion")), 50))
return back_ip4, back_onion
def findHashIds(self, site, hash_ids, limit=100):
back_ip4 = {}
back_onion = {}
found = site.worker_manager.findOptionalHashIds(hash_ids, limit=limit)
for hash_id, peers in found.iteritems():
back_onion[hash_id] = list(itertools.islice((
helper.packOnionAddress(peer.ip, peer.port)
for peer in peers
if peer.ip.endswith("onion")
), 50))
back_ip4[hash_id] = list(itertools.islice((
helper.packAddress(peer.ip, peer.port)
for peer in peers
if not peer.ip.endswith("onion")
), 50))
return back_ip4, back_onion
def packMyAddress(self):
if self.ip.endswith(".onion"):
return helper.packOnionAddress(self.ip, self.port)
else:
return helper.packAddress(self.ip, self.port)
def packMyAddress(self):
return helper.packAddress(self.ip, self.port)
def packMyAddress(self):
return helper.packAddress(self.ip, self.port)
