def __init__(self, shard_size=16, eviction_interval=timedelta(minutes=1)): self.shard_size = shard_size self.shards = [None] * self.shard_size self.eviction_interval = eviction_interval if 2**int(math.log(self.shard_size, 2) + 0.5) != self.shard_size: raise ShardSizeNotPowerOf2() for i in range(0, self.shard_size): self.shards[i] = Shard() self._run_eviction()
def main(): args = parse_args() n = args.n servers_obj_list = [Server(chr(65 + x), n) for x in range(0, n)] distributed_db = DistributedDataBase(servers_obj_list) shards_obj = (Shard(str(i)) for i in range(1, (n * n) // 2 + 1)) for shard in shards_obj: if args.random: distributed_db.add_random_data(shard) elif args.mirror: distributed_db.add_mirror_data(shard) dataloss = DataLossSimulator(servers_obj_list) print('Killing 2 arbitrary servers results in data loss in {}% cases'\ .format(dataloss.probability*100))
def __init__(self, numberOfShard): # Init shards self.listOfShard = {} self.listOfRecoveryInformation = {} for i in range(int(numberOfShard)): createdShard = Shard(i) self.listOfRecoveryInformation[ i] = createdShard.recoveryInformation self.listOfShard[i] = createdShard # Handle the answers from the recovery self.setOfTransactionsToAbort = set() self.setofTransactionsWithoutCommit = set() for recoveryInformation in self.listOfRecoveryInformation: transactionsToAbort = recoveryInformation.transactionsToAbort transactionsWithoutCommit = recoveryInformation.transactionsWithoutCommit self.setOfTransactionsToAbort = self.setOfTransactionsToAbort.union( transactionsToAbort) self.setofTransactionsWithoutCommit = self.setofTransactionsWithoutCommit.union( transactionsWithoutCommit) # Start receiving msg from clients self.receive
import global_constants from build_connection import MySQLConnection from shard import Shard raw_db = MySQLConnection(db=global_constants.RAW_DB) processed_db = MySQLConnection(db=global_constants.PROCESSED_DB) raw_db.cursor.execute('SELECT COUNT(*) FROM raw_ads;') RAW_ADS_COUNT = raw_db.cursor.fetchone()[0] raw_db.cursor.execute('SELECT COUNT(*) FROM raw_ads_images;') RAW_ADS_IMAGES_COUNT = raw_db.cursor.fetchone()[0] RAW_ADS_COUNT = 100 RAW_ADS_IMAGES_COUNT = 100 print RAW_ADS_IMAGES_COUNT RAW_ADS_PER_THREAD = int(RAW_ADS_COUNT / (global_constants.MAX_THREAD - 1)) RAW_ADS_IMAGES_PER_THREAD = int(RAW_ADS_IMAGES_COUNT / (global_constants.MAX_THREAD - 1)) for i in range(global_constants.MAX_THREAD): Shard(i, RAW_ADS_PER_THREAD, connect_database=True).start()
def add_shard(self, shard_name): self.num_shards += 1 self.shards[shard_name] = Shard(shard_name) self.shard_list.append(shard_name)
import discord from shard import Shard client = discord.Client(shard_id=0, shard_count=3) @client.event async def on_ready(): await s.on_ready() @client.event async def on_message(message): await s.on_message(message) @client.event async def on_reaction_add(reaction, user): await s.on_reaction_add(reaction, user) s = Shard(0, client) s.run()
shard_count = system_config["SHARD_COUNT"] logging.info(f"Main : creating prediction") prediction = Prediction() prediction_thread = threading.Thread(target=prediction.run).start() simulator = Simulator() simulator_thread = threading.Thread(target=simulator.run).start() threads = [] shard_signals = [] viz = [] for i in range(shard_count): logging.info(f"Main : creating thread {i}") shard_signals.append(f"SHARD_{i}") shard = Shard(i) threads.append(threading.Thread(target=shard.run)) logging.info(f"Main : shard signals {shard_signals}") for i, thread in enumerate(threads): logging.info(f"Main : starting thread {i}") thread.start() logging.info("Main : creating beacon chain") beacon_chain = Beacon() logging.info("Main : starting beacon chain") beacon_thread = threading.Thread(target=beacon_chain.run).start() logging.info("Main : all initialized...")
def main(): try: parser = argparse.ArgumentParser( description='Ethereum 2.0 Coss-Shard Simulation Commands') parser.add_argument('-sh', '--shards', type=int, default=config['simulation']['shards'], help="shards to simulate") parser.add_argument('-s', '--slot', type=float, default=config['simulation']['slot'], help="seconds per slot (decimal)") parser.add_argument('-b', '--blocksize', type=int, default=config['simulation']['blocksize'], help="size of shard blocks (kb)") parser.add_argument('-ws', '--witnesssize', type=int, default=config['simulation']['witnesssize'], help="size of Eth1 stateless witness (kb)") parser.add_argument( '-css', '--transactionsize', type=int, default=config['simulation']['transactionsize'], help="size of a cross-shard transaction receipt (bytes)") parser.add_argument( '-t', '--tps', type=int, default=config['transactions']['tps'], help= "number of transactions globally per second to added to mempool") parser.add_argument('-d', '--duration', type=int, default=config['transactions']['duration'], help="duration of time to simulate (seconds)") parser.add_argument( '-cs', '--crossshard', type=float, default=config['transactions']['crossshard'], help= "probability a cross-shard call will occur within a transaction") parser.add_argument( '-c', '--collision', type=float, default=config['transactions']['collision'], help= "probability a transaction will experience a mutated state and cause a reversion of the transaction" ) parser.add_argument( '-sw', '--sweep', action='store_true', default=config['transactions']['sweep'], help= "sweeps the probability for the test duration (eg: 0.25, 0 -> 0.25)" ) parser.add_argument( '-g', '--generate', action='store_true', default=False, help= "application will only generate and stores transactions without simulation" ) parser.add_argument("-o", '--output', type=argparse.FileType('w'), default=None, help="path to file for saving output") parser.add_argument("-ot", '--outputtransactions', type=argparse.FileType('wb'), default=None, help="path to file for saving output") parser.add_argument("-i", '--input', type=argparse.FileType('rb'), default=None, help="path to file to transaction inputs") args = parser.parse_args() def generate_random_transaction(shards, probability, collision_probability): transaction = Transaction(uuid.uuid4()) while True: shard = random.randrange(0, shards, 1) if len(transaction) > 1: if shard == transaction[len(transaction) - 1] and shard + 1 < shards: shard += 1 elif shard == transaction[len(transaction) - 1] and shard + 1 >= shards: shard -= 1 transaction_segment = TransactionSegment( shard, numpy.random.choice( [True, False], p=[collision_probability, 1 - collision_probability])) transaction.append(transaction_segment) choices = [True, False] weights = [1 - probability, probability] if (numpy.random.choice(choices, p=weights)): break probability = probability / 2 return transaction def on_shard_block(beacon_chain, shard, block): if (len(beacon_chain) <= block.index): for i in range((block.index + 1) - len(beacon_chain)): beacon_chain.append(list([None] * args.shards)) beacon_chain[block.index][shard] = block def calc_crossshard_probability(probability, transaction_total, index, is_sweep): if is_sweep: return (index / transaction_total) * probability return probability def generate_transaction_set(args): transaction_set = [] transaction_total = args.duration * args.tps transaction_progress_bar = tqdm(total=transaction_total) transaction_progress_bar.desc = "Generating Transactions" for i in range(transaction_total): probability = calc_crossshard_probability( args.crossshard, transaction_total, i, args.sweep) transaction = generate_random_transaction( args.shards, probability, args.collision) transaction_set.append(transaction) transaction_progress_bar.update(1) transaction_progress_bar.close() return transaction_set def update_progress_bar(progress_bar, env, tick): while True: progress_bar.update(tick) yield env.timeout(tick) def new_slot(env, shard, tick): while True: # before slot yield env.timeout(tick) # after slot shard.produceShardBlock() shard.commitShardBlock() def add_tps(env, transaction_set): while True: yield env.timeout(1 / args.tps) index = int(env.now / (1 / args.tps)) - 1 transaction = transaction_set[index] mempool[transaction[0].shard].append(transaction) env.total_generated_transactions += 1 def create_chart(beacon_chain): stats.create_transaction_and_segments_per_slot_chart( beacon_chain).savefig(TRANSACTION_SET_PATH + '/' + transaction_args_hash() + '.transaction_and_segments_per_slot.png') def output_data(beacon_chain, time_elapsed, transaction_log, collision_log, env): data = "" data += stats.transaction_segments_per_block(beacon_chain) data += stats.transactions_per_block(beacon_chain) if args.sweep: data += stats.probability_over_duration( args.crossshard, env.now, calc_crossshard_probability) data += stats.stats(args, time_elapsed, beacon_chain, transaction_log, env.total_generated_transactions, collision_log) data += stats.config(args) return data def transaction_args(): # duration, tps, crossshard, sweep, collision return { "duration": args.duration, "tps": args.tps, "crossshard": args.crossshard, "sweep": args.sweep, "collision": args.collision } def transaction_args_hash(): return sha256(str(transaction_args()).encode()).hexdigest() def has_cached_transactions(): file_to_open = transaction_args_hash() + ".bin" walk = os.walk(os.getcwd() + "/" + TRANSACTION_SET_PATH) for files in walk: for name in files: for file in name: if file == file_to_open: return True return False # init start_time = time.time() beacon_chain = list() # create mempool mempool = list() receipt_queue = list() for i in range(args.shards): mempool.append([]) receipt_queue.append([]) transaction_log = [] transaction_set = [] collision_log = [] gas = [] bandwidth = [] has_cached_transactions = has_cached_transactions() if args.input == None and not has_cached_transactions: transaction_set = generate_transaction_set(args) else: if (args.input == None): args.input = open( TRANSACTION_SET_PATH + "/" + transaction_args_hash() + ".bin", "rb") print("Recovering cached file") transaction_args_pickle = pickle.load(args.input) transaction_set = pickle.load(args.input) args.input.close() # Override the values or transaction based args for arg in vars(args): if arg in transaction_args_pickle: setattr(args, arg, transaction_args_pickle[arg]) if not has_cached_transactions: file = args.outputtransactions if file == None: file = open( TRANSACTION_SET_PATH + "/" + transaction_args_hash() + ".bin", "wb") pickle.dump(transaction_args(), file) pickle.dump(transaction_set, file) if args.generate != True: progress_bar = tqdm(total=args.duration) progress_bar.desc = "Simulation Running" env = simpy.Environment() env.total_generated_transactions = 0 env.progress = 0 env.process(add_tps(env, transaction_set)) blocklimit = (args.blocksize - args.witnesssize) * 1000 / args.transactionsize for i in range(args.shards): shard = Shard(i, on_shard_block, beacon_chain, blocklimit, mempool, receipt_queue, args.collision, collision_log, gas, bandwidth) env.process(new_slot(env, shard, args.slot)) env.process(update_progress_bar(progress_bar, env, 1)) env.run(until=args.duration) progress_bar.close() data = output_data(beacon_chain, (time.time() - start_time), transaction_log, collision_log, env) create_chart(beacon_chain) if args.output == None: file = open( TRANSACTION_SET_PATH + "/" + transaction_args_hash() + ".csv", "w") file.write(data) else: args.output.write(data) except KeyboardInterrupt: data = output_data(beacon_chain, (time.time() - start_time), transaction_log, collision_log, env) create_chart(beacon_chain) if args.output == None: print(data) else: args.output.write(data) except Exception: traceback.print_exc(file=sys.stdout) sys.exit(0)
import discord from shard import Shard client = discord.Client(shard_id=2, shard_count=3) @client.event async def on_ready(): await s.on_ready() @client.event async def on_message(message): await s.on_message(message) @client.event async def on_reaction_add(reaction, user): await s.on_reaction_add(reaction, user) s = Shard(2, client) s.run()
import discord from shard import Shard client = discord.AutoShardedClient() @client.event async def on_ready(): await s.on_ready() @client.event async def on_message(message): await s.on_message(message) @client.event async def on_reaction_add(reaction, user): await s.on_reaction_add(reaction, user) s = Shard(client) s.run()
import discord from shard import Shard client = discord.Client(shard_id=1, shard_count=3) @client.event async def on_ready(): await s.on_ready() @client.event async def on_message(message): await s.on_message(message) @client.event async def on_reaction_add(reaction, user): await s.on_reaction_add(reaction, user) s = Shard(1, client) s.run()
sql += ",".join( map(str,row) ) sql += ')' print sql # TODO: insert into second DB, and then remove rows from original cursor.close() connection.close() if __name__ == '__main__': shardconf = DjangoShardLoader() session = ShardedSession(shardconf) session.adminCursor().executeAll('delete from throwawayshard.user') session.adminCursor().executeAll('delete from throwawayshard.userComment') sc = ShardConf() sc.id = 10000 sc.capacity_MB = 100 sc.current_MB = 0 sc.database = 'throwawayshard' sc.full = False sc.user = '******' sc.password = '******' sc.host = '192.168.0.201' sc.save() sh = Shard(sc.id, sc.user, sc.password, sc.host, sc.database, sc.capacity_MB, sc.full, sc.initialized, (sc,)) rebalanceShardBucket(session.shards[0], sh, (1,19))