async def start_ipv8(self, statedir, listen_port, statistics, no_rest_api):
"""
Main method to startup IPv8.
"""
configuration = get_default_configuration()
configuration['port'] = listen_port
# Open the database
self.tc_persistence = TrustChainDB(statedir, 'trustchain')
if statedir:
# If we use a custom state directory, update various variables
for key in configuration["keys"]:
key["file"] = os.path.join(statedir, key["file"])
for community in configuration["overlays"]:
if community["class"] == "TrustChainCommunity":
community["initialize"]["persistence"] = self.tc_persistence
allowed_overlays = ['DHTDiscoveryCommunity', 'DiscoveryCommunity', 'HiddenTunnelCommunity',
'TrustChainCommunity']
configuration['overlays'] = [overlay for overlay in configuration['overlays']
if overlay['class'] in allowed_overlays]
for overlay in configuration['overlays']:
if overlay['class'] == 'HiddenTunnelCommunity':
overlay['initialize']['settings']['min_circuits'] = 0
overlay['initialize']['settings']['max_circuits'] = 0
overlay['initialize']['settings']['max_relays_or_exits'] = 1000
overlay['initialize']['settings']['peer_flags'] = {PEER_FLAG_EXIT_IPV8}
self.ipv8 = IPv8(configuration, enable_statistics=statistics)
await self.ipv8.start()
async def signal_handler(sig):
print("Received shut down signal %s" % sig)
if not self._stopping:
self._stopping = True
if self.restapi:
await self.restapi.stop()
await self.ipv8.stop()
await gather(*all_tasks())
get_event_loop().stop()
signal.signal(signal.SIGINT, lambda sig, _: ensure_future(signal_handler(sig)))
signal.signal(signal.SIGTERM, lambda sig, _: ensure_future(signal_handler(sig)))
print("Starting IPv8")
if not no_rest_api:
self.restapi = RESTManager(self.ipv8)
await self.restapi.start()
"""
This script outputs a CSV file with the number of confirmed and unconfirmed blocks per day.
"""
from ipv8.attestation.trustchain.database import TrustChainDB
from tc_analysis import DB_PATH
db = TrustChainDB(DB_PATH, "trustchain")
print("Database opened!")
query = "SELECT strftime('%d-%m-%Y', block_timestamp/1000, 'unixepoch'), COUNT(*) FROM blocks GROUP BY strftime('%d-%m-%Y', block_timestamp/1000, 'unixepoch') ORDER BY block_timestamp"
res = list(db.execute(query))
creation_info = []
for day_info in res:
creation_info.append(day_info)
print("Writing statistics")
with open("creation_stats.csv", "w") as output_file:
output_file.write("day,blocks\n")
for day, num_blocks in creation_info:
output_file.write("%s,%d\n" % (day.decode(), num_blocks))
"""
This script outputs a CSV file with the number of confirmed and unconfirmed blocks per day.
"""
import datetime
from ipv8.attestation.trustchain.database import TrustChainDB
from tc_analysis import DB_PATH
day_stats = {}
db = TrustChainDB(DB_PATH, "trustchain")
print("Database opened!")
BATCH_SIZE = 500000
MAX_BLOCKS = -1
unconfirmed_txs = set()
unconfirmed_links = set()
tx_times = {}
def write_results():
with open("creation_stats_detailled.csv", "w") as output_file:
output_file.write("day,confirmed,unconfirmed\n")
for day, info in day_stats.items():
output_file.write("%s,%d,%d\n" % (day, info["confirmed"], info["unconfirmed"]))
parsed_blocks = 0
while True:
"""
This script outputs a CSV file with identities and the number of blocks created for each identity.
"""
from binascii import hexlify
from ipv8.attestation.trustchain.database import TrustChainDB
from tc_analysis import DB_PATH
day_stats = {}
db = TrustChainDB(DB_PATH, "trustchain")
print("Database opened!")
users_info = db.execute(
"SELECT DISTINCT public_key, COUNT(*) FROM blocks GROUP BY public_key")
print("Fetched user info!")
user_id = 1
with open("identities.csv", "w") as out_file:
out_file.write("id,public_key,blocks\n")
for user_info in users_info:
out_file.write("%d,%s,%d\n" %
(user_id, hexlify(user_info[0]).decode(), user_info[1]))
user_id += 1
async def start_crawler(self, statedir, apiport, no_rest_api, testnet,
yappi):
"""
Main method to startup the TrustChain crawler.
"""
# Open the database
self.tc_persistence = TrustChainDB(statedir, 'trustchain')
if statedir:
# If we use a custom state directory, update various variables
for key in crawler_config["keys"]:
key["file"] = os.path.join(statedir, key["file"])
for community in crawler_config["overlays"]:
if community[
"class"] == "TrustChainCrawlerCommunity" or community[
"class"] == "TrustChainBackwardsCrawlerCommunity":
community["initialize"][
"persistence"] = self.tc_persistence
if testnet:
for community in crawler_config["overlays"]:
if community["class"] == "TrustChainCommunity":
community["class"] = "TrustChainTestnetCommunity"
extra_communities = {
"TrustChainCrawlerCommunity":
TrustChainCrawlerCommunity,
"TrustChainBackwardsCrawlerCommunity":
TrustChainBackwardsCrawlerCommunity
}
self.ipv8 = IPv8(crawler_config, extra_communities=extra_communities)
await self.ipv8.start()
async def signal_handler(sig):
print("Received shut down signal %s" % sig)
if not self._stopping:
self._stopping = True
if self.restapi:
await self.restapi.stop()
await self.ipv8.stop()
if yappi:
yappi.stop()
print("Yappi has shutdown")
yappi_stats = yappi.get_func_stats()
yappi_stats.sort("tsub")
out_file = 'yappi'
if statedir:
out_file = os.path.join(statedir, out_file)
yappi_stats.save(out_file, type='callgrind')
await gather(*all_tasks())
get_event_loop().stop()
signal.signal(signal.SIGINT,
lambda sig, _: ensure_future(signal_handler(sig)))
signal.signal(signal.SIGTERM,
lambda sig, _: ensure_future(signal_handler(sig)))
print("Starting TrustChain crawler")
if not no_rest_api:
self.restapi = RESTManager(self.ipv8)
await self.restapi.start(apiport)
if yappi:
yappi.start(builtins=True)
from binascii import hexlify, unhexlify
from deprecated.encoding import decode
from ipv8.attestation.trustchain.database import TrustChainDB
database_path = u"/Users/martijndevos/Documents/trustchain-db"
db = TrustChainDB(database_path, "trustchain")
print("Database opened!")
public_keys = []
balances = {}
# Load public keys
with open("identities.csv", "r") as in_file:
did_header = False
for line in in_file.readlines():
if not did_header:
did_header = True
continue
parts = line.split(",")
public_key = unhexlify(parts[0])
public_keys.append(public_key)
processed = 0
for public_key in public_keys:
last_block = list(
db.execute(
"SELECT tx, type FROM blocks WHERE public_key = ? ORDER BY block_timestamp DESC LIMIT 1",
(public_key, )))[0]
from deprecated.encoding import decode
from ipv8.attestation.trustchain.database import TrustChainDB
day_stats = {}
database_path = u"/Users/martijndevos/Documents/trustchain-db"
db = TrustChainDB(database_path, "trustchain")
print("Database opened!")
BATCH_SIZE = 500000
TOTAL_BLOCKS = 103149931
FIVE_MB = 5 * 1024 * 1024
size_frequencies = {}
under_5_mb_size_frequencies = {}
parsed_blocks = 0
while parsed_blocks < TOTAL_BLOCKS:
blocks = list(db.execute("SELECT link_sequence_number, type, tx FROM blocks ORDER BY block_timestamp ASC LIMIT %d OFFSET %d" % (BATCH_SIZE, parsed_blocks)))
for block in blocks:
if block[1] != b"tribler_bandwidth":
continue
if block[0] == 0: # Source block
decoded_tx = decode(block[2])[1]
download_mb = int(decoded_tx[b"down"] / 1024.0 / 1024.0)
if download_mb not in size_frequencies:
size_frequencies[download_mb] = 0
size_frequencies[download_mb] += 1
if decoded_tx[b"down"] <= FIVE_MB: